paperlint 9.0.1 → 10.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/CONTRIBUTING.md +16 -4
  2. package/README.md +68 -128
  3. package/dist/build.d.ts +1 -1
  4. package/dist/build.js +1 -1
  5. package/dist/cli.d.ts +5 -1
  6. package/dist/cli.d.ts.map +1 -1
  7. package/dist/cli.js +52 -6
  8. package/dist/cli.js.map +1 -1
  9. package/dist/domain/layout-tree.d.ts +19 -0
  10. package/dist/domain/layout-tree.d.ts.map +1 -0
  11. package/dist/domain/layout-tree.js +32 -0
  12. package/dist/domain/layout-tree.js.map +1 -0
  13. package/dist/domain/lookup-cache.d.ts +2 -2
  14. package/dist/domain/lookup-cache.js +2 -2
  15. package/dist/domain/paper-layout.d.ts +81 -0
  16. package/dist/domain/paper-layout.d.ts.map +1 -0
  17. package/dist/domain/paper-layout.js +259 -0
  18. package/dist/domain/paper-layout.js.map +1 -0
  19. package/dist/domain/repro-layout.d.ts +80 -0
  20. package/dist/domain/repro-layout.d.ts.map +1 -0
  21. package/dist/domain/repro-layout.js +247 -0
  22. package/dist/domain/repro-layout.js.map +1 -0
  23. package/dist/domain/scaffold-layout.d.ts +28 -0
  24. package/dist/domain/scaffold-layout.d.ts.map +1 -0
  25. package/dist/domain/scaffold-layout.js +81 -0
  26. package/dist/domain/scaffold-layout.js.map +1 -0
  27. package/dist/eslint-rules/js-needs-reason.d.ts +55 -0
  28. package/dist/eslint-rules/js-needs-reason.d.ts.map +1 -0
  29. package/dist/eslint-rules/js-needs-reason.js +35 -0
  30. package/dist/eslint-rules/js-needs-reason.js.map +1 -0
  31. package/dist/eslint-rules/prereg-coverage.d.ts +98 -0
  32. package/dist/eslint-rules/prereg-coverage.d.ts.map +1 -0
  33. package/dist/eslint-rules/prereg-coverage.js +274 -0
  34. package/dist/eslint-rules/prereg-coverage.js.map +1 -0
  35. package/dist/folder-layout-hook.d.ts +21 -0
  36. package/dist/folder-layout-hook.d.ts.map +1 -0
  37. package/dist/folder-layout-hook.js +117 -0
  38. package/dist/folder-layout-hook.js.map +1 -0
  39. package/dist/folder-layout-rule.d.ts +22 -0
  40. package/dist/folder-layout-rule.d.ts.map +1 -0
  41. package/dist/folder-layout-rule.js +107 -0
  42. package/dist/folder-layout-rule.js.map +1 -0
  43. package/dist/init.js +1 -1
  44. package/dist/init.js.map +1 -1
  45. package/dist/new-paper.d.ts +4 -1
  46. package/dist/new-paper.d.ts.map +1 -1
  47. package/dist/new-paper.js +40 -6
  48. package/dist/new-paper.js.map +1 -1
  49. package/dist/package-dirs.d.ts +6 -0
  50. package/dist/package-dirs.d.ts.map +1 -1
  51. package/dist/package-dirs.js +6 -0
  52. package/dist/package-dirs.js.map +1 -1
  53. package/dist/references.d.ts +1 -1
  54. package/dist/references.d.ts.map +1 -1
  55. package/dist/references.js +2 -2
  56. package/dist/references.js.map +1 -1
  57. package/dist/repro-layout-rule.d.ts +16 -0
  58. package/dist/repro-layout-rule.d.ts.map +1 -0
  59. package/dist/repro-layout-rule.js +81 -0
  60. package/dist/repro-layout-rule.js.map +1 -0
  61. package/dist/skills/label-agent-runs/scripts/labels-commands.d.ts +13 -0
  62. package/dist/skills/label-agent-runs/scripts/labels.d.ts +2 -0
  63. package/dist/skills/run-agent-grid/scripts/grid-guard-commands.d.ts +20 -0
  64. package/dist/skills/run-agent-grid/scripts/grid-guard.d.ts +2 -0
  65. package/docs/ci.md +60 -0
  66. package/docs/configuration.md +2 -0
  67. package/docs/e2e.md +2 -2
  68. package/docs/install.md +20 -16
  69. package/docs/layout.md +154 -0
  70. package/docs/optional-rules.md +5 -4
  71. package/docs/references.md +5 -2
  72. package/docs/rules/paper/allowed-files-only.md +192 -0
  73. package/docs/rules/repro/allowed-files-only.md +128 -0
  74. package/docs/rules/study/declare-sample.md +146 -0
  75. package/docs/rules.md +53 -50
  76. package/docs/skills.md +8 -3
  77. package/docs/testing.md +13 -0
  78. package/docs/venues.md +94 -0
  79. package/eslint-rules/js-needs-reason.test.ts +141 -0
  80. package/eslint-rules/js-needs-reason.ts +77 -0
  81. package/eslint-rules/prereg-coverage.test.ts +448 -0
  82. package/eslint-rules/prereg-coverage.ts +419 -0
  83. package/eslint.config.mjs +22 -2
  84. package/lib/skill-trigger-cases.harness.mjs +51 -5
  85. package/lib/skill-trigger-cases.mjs +53 -15
  86. package/lib/skill-trigger-cases.test.ts +211 -0
  87. package/package.json +8 -4
  88. package/plugin/hooks/hooks.json +9 -0
  89. package/scripts/coverage-config.test.ts +5 -3
  90. package/skills/analyze-sibling-paper/SKILL.md +2 -2
  91. package/skills/analyze-sibling-paper/SKILL.md.spec.ts +1 -1
  92. package/skills/build-benchmark/SKILL.md +17 -1
  93. package/skills/build-benchmark/SKILL.md.spec.ts +16 -0
  94. package/skills/build-label-page/SKILL.md +120 -0
  95. package/skills/build-label-page/SKILL.md.spec.ts +131 -0
  96. package/skills/build-label-page/build-label-page.eval.mjs +15 -0
  97. package/skills/build-label-page/build-label-page.test.ts +959 -0
  98. package/skills/build-label-page/references/checklist.md +148 -0
  99. package/skills/build-label-page/references/example-generator.py +803 -0
  100. package/skills/find-venue/SKILL.md +5 -3
  101. package/skills/find-venue/SKILL.md.spec.ts +4 -2
  102. package/skills/grade-paper-writing/SKILL.md +7 -7
  103. package/skills/grade-paper-writing/SKILL.md.spec.ts +6 -6
  104. package/skills/label-agent-runs/SKILL.md +103 -0
  105. package/skills/label-agent-runs/SKILL.md.spec.ts +106 -0
  106. package/skills/label-agent-runs/label-agent-runs.eval.ts +55 -0
  107. package/skills/label-agent-runs/label-agent-runs.harness.ts +123 -0
  108. package/skills/label-agent-runs/references/formats.md +64 -0
  109. package/skills/label-agent-runs/scripts/labels-commands.js +13 -0
  110. package/skills/label-agent-runs/scripts/labels-commands.ts +13 -0
  111. package/skills/label-agent-runs/scripts/labels.js +396 -0
  112. package/skills/label-agent-runs/scripts/labels.test.ts +538 -0
  113. package/skills/label-agent-runs/scripts/labels.ts +501 -0
  114. package/skills/map-prior-work/SKILL.md +2 -2
  115. package/skills/map-prior-work/SKILL.md.spec.ts +1 -1
  116. package/skills/paper-pipeline/SKILL.md +15 -6
  117. package/skills/paper-pipeline/SKILL.md.spec.ts +14 -5
  118. package/skills/paper-pipeline/grade-paper-writing-ablation.eval.mjs +19 -9
  119. package/skills/paper-pipeline/references/acceptance-gate.md +19 -19
  120. package/skills/paper-pipeline/references/artifact-checklist.md +5 -0
  121. package/skills/paper-pipeline/references/pipeline-status-template.md +30 -1
  122. package/skills/paper-pipeline/repro/2026-08-07-fork-check.log +1 -1
  123. package/skills/paper-pipeline/repro/2026-08-07-fork-check2.log +1 -1
  124. package/skills/paper-pipeline/repro/2026-08-07-language-eval-pilot.log +1 -1
  125. package/skills/paper-pipeline/repro/2026-08-07-language-eval-raw.log +1 -1
  126. package/skills/paper-pipeline/repro/2026-08-08-framing-vs-vocabulary-oracle.log +48 -48
  127. package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-A6-oracle.log +8 -8
  128. package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-oracle.log +48 -48
  129. package/skills/paper-pipeline/repro/2026-08-08-parent-replication-gpw.log +2 -2
  130. package/skills/paper-pipeline/repro/2026-08-08-parent-replication.mjs +3 -3
  131. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-after.log +81 -0
  132. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-before.log +81 -0
  133. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-setupdiff.log +59 -0
  134. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.jsonl +1 -0
  135. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.log +22 -0
  136. package/skills/paper-pipeline/repro/README.md +4 -0
  137. package/skills/paper-pipeline/scripts/extract-ref-facts.harness.mjs +2 -2
  138. package/skills/paper-pipeline/scripts/extract-ref-facts.mjs +2 -2
  139. package/skills/paper-pipeline/scripts/extract-ref-facts.test.mjs +4 -2
  140. package/skills/paper-pipeline/scripts/generated-code.harness.mjs +7 -2
  141. package/skills/paper-pipeline/scripts/generated-code.mjs +8 -4
  142. package/skills/paper-pipeline/scripts/generated-code.test.ts +15 -1
  143. package/skills/paper-status/SKILL.md +2 -2
  144. package/skills/paper-status/SKILL.md.spec.ts +1 -1
  145. package/skills/run-agent-grid/SKILL.md +181 -0
  146. package/skills/run-agent-grid/SKILL.md.spec.ts +184 -0
  147. package/skills/run-agent-grid/references/failure-catalog.md +36 -0
  148. package/skills/run-agent-grid/references/registry-and-ledger.md +142 -0
  149. package/skills/run-agent-grid/run-agent-grid.eval.ts +59 -0
  150. package/skills/run-agent-grid/run-agent-grid.harness.ts +115 -0
  151. package/skills/run-agent-grid/scripts/grid-guard-commands.js +18 -0
  152. package/skills/run-agent-grid/scripts/grid-guard-commands.ts +21 -0
  153. package/skills/run-agent-grid/scripts/grid-guard.js +700 -0
  154. package/skills/run-agent-grid/scripts/grid-guard.test.ts +1090 -0
  155. package/skills/run-agent-grid/scripts/grid-guard.ts +869 -0
  156. package/skills/study-accepted-papers/SKILL.md +163 -161
  157. package/skills/study-accepted-papers/SKILL.md.spec.ts +167 -165
  158. package/skills/study-accepted-papers/references/base-rates.md +69 -0
  159. package/skills/study-accepted-papers/references/mechanisms.md +181 -0
  160. package/skills/verify-citations/SKILL.md +2 -2
  161. package/skills/verify-citations/SKILL.md.spec.ts +1 -1
  162. package/src/build.ts +1 -1
  163. package/src/cli.harness.mjs +42 -0
  164. package/src/cli.test.ts +90 -6
  165. package/src/cli.ts +78 -5
  166. package/src/domain/layout-tree.test.ts +32 -0
  167. package/src/domain/layout-tree.ts +47 -0
  168. package/src/domain/lookup-cache.ts +2 -2
  169. package/src/domain/paper-layout.test.ts +264 -0
  170. package/src/domain/paper-layout.ts +336 -0
  171. package/src/domain/repro-layout.test.ts +222 -0
  172. package/src/domain/repro-layout.ts +311 -0
  173. package/src/domain/scaffold-layout.ts +104 -0
  174. package/src/folder-layout-hook.test.ts +214 -0
  175. package/src/folder-layout-hook.ts +141 -0
  176. package/src/folder-layout-rule.test.ts +235 -0
  177. package/src/folder-layout-rule.ts +151 -0
  178. package/src/hooks-settings.harness.ts +9 -8
  179. package/src/init.ts +1 -1
  180. package/src/layout-docs.test.ts +52 -0
  181. package/src/new-paper.harness.mjs +23 -1
  182. package/src/new-paper.test.ts +299 -6
  183. package/src/new-paper.ts +51 -5
  184. package/src/package-dirs.ts +8 -0
  185. package/src/paper-files.test.ts +9 -6
  186. package/src/references.test.ts +5 -5
  187. package/src/references.ts +2 -2
  188. package/src/repro-layout-rule.test.ts +114 -0
  189. package/src/repro-layout-rule.ts +121 -0
  190. package/src/rule-cases.test.ts +98 -0
  191. package/templates/paper/PIPELINE-STATUS.md +41 -7
  192. package/tsconfig.base.json +22 -0
  193. package/tsconfig.json +6 -24
  194. package/tsconfig.pkg.json +3 -1
  195. package/tsconfig.skills.json +14 -0
  196. package/tsconfig.src.json +12 -0
@@ -4,7 +4,7 @@ description: "Discover and rank real venues for a given paper, scored by authors
4
4
  allowed-tools: [Read, Write, Grep, Glob, WebSearch, WebFetch, Agent]
5
5
  ---
6
6
 
7
- <!-- vigiles:sha256:8b332cbc072b46ee compiled from skills/find-venue/SKILL.md.spec.ts -->
7
+ <!-- vigiles:sha256:42d8e283fb7045f1 compiled from skills/find-venue/SKILL.md.spec.ts -->
8
8
 
9
9
  # find-venue — rank real venues by what earns the credit, then keep or switch
10
10
 
@@ -43,7 +43,7 @@ rank = credit-weight × topic-fit × accept-probability × deadline-feasibility
43
43
  prestige (a bonus, not a gate). **Non-archival ⇒ credit-weight ≈ 0** — it earns no authorship credit,
44
44
  so it sinks to the bottom no matter how nice the CFP reads.
45
45
  - **topic-fit** — how squarely the paper matches the CFP scope (a stretch fit raises reject risk).
46
- - **accept-probability** — realistic odds given the paper's maturity and the venue's bar.
46
+ - **accept-probability** — realistic odds given the paper's maturity and how selective the venue is.
47
47
  - **deadline-feasibility** — can the paper (and artifact) be ready by the deadline? A great venue you
48
48
  miss is a zero.
49
49
  - **remote-friendliness** — remote-OK is a real weight here, not a footnote.
@@ -84,7 +84,9 @@ rank = credit-weight × topic-fit × accept-probability × deadline-feasibility
84
84
  - the venue cards, `presets/<venue>.md` in the paperlint package beside each venue preset (e.g.
85
85
  `agenticdev.md`, `aisec.md`) — the winner gets a card with its HotCRP quirks beside its preset;
86
86
  for a venue with no shipped preset, the project's own `./venues/<name>.md` beside its
87
- `./venues/<name>.jsonc`. Plain data, not a new skill.
87
+ `./venues/<name>.jsonc`. Plain data, not a new skill. That `./venues/` is not the venue's corpus
88
+ of accepted papers, `<papers-root>/venues/<venue>/` (`study-accepted-papers`): the three places are
89
+ told apart in `docs/venues.md` in the paperlint package.
88
90
  - `extend-paper` — the prestige-upgrade target this skill names but defers.
89
91
 
90
92
  ## Provenance
@@ -49,7 +49,7 @@ rank = credit-weight × topic-fit × accept-probability × deadline-feasibility
49
49
  prestige (a bonus, not a gate). **Non-archival ⇒ credit-weight ≈ 0** — it earns no authorship credit,
50
50
  so it sinks to the bottom no matter how nice the CFP reads.
51
51
  - **topic-fit** — how squarely the paper matches the CFP scope (a stretch fit raises reject risk).
52
- - **accept-probability** — realistic odds given the paper's maturity and the venue's bar.
52
+ - **accept-probability** — realistic odds given the paper's maturity and how selective the venue is.
53
53
  - **deadline-feasibility** — can the paper (and artifact) be ready by the deadline? A great venue you
54
54
  miss is a zero.
55
55
  - **remote-friendliness** — remote-OK is a real weight here, not a footnote.
@@ -90,7 +90,9 @@ rank = credit-weight × topic-fit × accept-probability × deadline-feasibility
90
90
  - the venue cards, \`presets/<venue>.md\` in the paperlint package beside each venue preset (e.g.
91
91
  \`agenticdev.md\`, \`aisec.md\`) — the winner gets a card with its HotCRP quirks beside its preset;
92
92
  for a venue with no shipped preset, the project's own \`./venues/<name>.md\` beside its
93
- \`./venues/<name>.jsonc\`. Plain data, not a new skill.
93
+ \`./venues/<name>.jsonc\`. Plain data, not a new skill. That \`./venues/\` is not the venue's corpus
94
+ of accepted papers, \`<papers-root>/venues/<venue>/\` (\`study-accepted-papers\`): the three places are
95
+ told apart in \`docs/venues.md\` in the paperlint package.
94
96
  - \`extend-paper\` — the prestige-upgrade target this skill names but defers.
95
97
 
96
98
  ## Provenance
@@ -1,17 +1,17 @@
1
1
  ---
2
2
  name: grade-paper-writing
3
- description: Use when asking "does this paper read well?" / "is the writing any good?" / "it reads like a wall of text and jargon" / "grade the writing" on a draft. Grades WRITING CRAFT only — ten dimensions 1–5 (Title, Abstract, Intro architecture, Structure, Prose clarity, Jargon discipline, Landing-the-point, Figure economy, Honesty-without-hedge-stacking, Register distance to the target venue) against how the best-written papers read (Peyton Jones, McEnerney, Gopen & Swan; exemplars Trusting Trust, Carlini, Greshake) AND against 2–3 accepted papers of the target venue, naming the offending sentence and the fix for each. Runs the cold-read stall pass with TWO committed personas (a non-academic engineer per section, and a workshop reviewer skimming abstract, introduction and conclusion) whose stall inventory — not the rubric number — is the readability gate other skills consume (pc-panel-review, paper-adversarial-review, harden-paper). Defaults to a blind multi-grader panel; after fixes, mandates the claims-preservation diff (row `claims`). NOT a content/defect review (paper-adversarial-review / pc-panel-review), NOT a structural cut plan (tighten-paper — run that first on a bloated draft), NOT venue-bar content strength (study-accepted-papers). Compose with harden-paper (which calls it), draft-paper (generative counterpart), render-paper.
3
+ description: Grade how a paper reads and fix it sentence by sentence. Use when asked if a paper reads well or to grade the prose, or when the writing is a wall of text, jargon-heavy, or reads like a blog post. Returns 1–5 scores, the sentences cold readers could not follow, and a rewrite for each. Not for cuts (tighten-paper), one edited paragraph (cold-read-diff), scientific flaws (paper-adversarial-review, pc-panel-review), submit readiness (harden-paper) or what the venue rewards (study-accepted-papers).
4
4
  allowed-tools: [Read, Write, Edit, Grep, Glob, Agent, Skill]
5
5
  ---
6
6
 
7
- <!-- vigiles:sha256:21c8899648c9b223 compiled from skills/grade-paper-writing/SKILL.md.spec.ts -->
7
+ <!-- vigiles:sha256:1897016b69f3f9b8 compiled from skills/grade-paper-writing/SKILL.md.spec.ts -->
8
8
 
9
9
  # grade-paper-writing — grade how the paper READS, then fix it sentence by sentence
10
10
 
11
- The review skills hunt scientific defects; `study-accepted-papers` checks whether the contribution clears
12
- the venue's bar. Neither tells you the paper is a slog to read. This skill does exactly that: it grades
13
- the **writing craft** against how the best-written papers actually read, and turns each low score into a
14
- concrete, sentence-level fix. Its target failure mode is the common one — *"a wall of text and jargon that
11
+ The review skills hunt scientific defects; `study-accepted-papers` scores the contribution on the
12
+ strong-accept scorecard. Neither tells you the paper is a slog to read. This skill does exactly that: it
13
+ grades the **writing craft** against how the best-written papers actually read, and turns each low score
14
+ into a concrete, sentence-level fix. Its target failure mode is the common one — *"a wall of text and jargon that
15
15
  doesn't drive home how crazy the situation is"* — which is a writing problem, not a science problem, and
16
16
  which the defect-hunting skills will pass right over.
17
17
 
@@ -67,7 +67,7 @@ Things that exist only in the render, every one of which produces a real stall:
67
67
  - how much of page 1 is numbers, which is what a reviewer's first impression is actually made of;
68
68
  - line numbers and two-column measure, which change where the eye stops.
69
69
 
70
- `repro/build-submission.sh` produces the PDF; `render-paper` turns pages into images. Judge from
70
+ `repro/tools/build-submission.sh` produces the PDF; `render-paper` turns pages into images. Judge from
71
71
  those. If only the source is available, say so in the report — a stall inventory taken from source
72
72
  is a weaker artifact and must not be presented as the same thing.
73
73
 
@@ -9,15 +9,15 @@ import { experimental_skill } from "vigiles/spec";
9
9
  export default experimental_skill({
10
10
  name: "grade-paper-writing",
11
11
  description:
12
- 'Use when asking "does this paper read well?" / "is the writing any good?" / "it reads like a wall of text and jargon" / "grade the writing" on a draft. Grades WRITING CRAFT only — ten dimensions 1–5 (Title, Abstract, Intro architecture, Structure, Prose clarity, Jargon discipline, Landing-the-point, Figure economy, Honesty-without-hedge-stacking, Register distance to the target venue) against how the best-written papers read (Peyton Jones, McEnerney, Gopen & Swan; exemplars Trusting Trust, Carlini, Greshake) AND against 2–3 accepted papers of the target venue, naming the offending sentence and the fix for each. Runs the cold-read stall pass with TWO committed personas (a non-academic engineer per section, and a workshop reviewer skimming abstract, introduction and conclusion) whose stall inventory — not the rubric number — is the readability gate other skills consume (pc-panel-review, paper-adversarial-review, harden-paper). Defaults to a blind multi-grader panel; after fixes, mandates the claims-preservation diff (row `claims`). NOT a content/defect review (paper-adversarial-review / pc-panel-review), NOT a structural cut plan (tighten-paper — run that first on a bloated draft), NOT venue-bar content strength (study-accepted-papers). Compose with harden-paper (which calls it), draft-paper (generative counterpart), render-paper.',
12
+ "Grade how a paper reads and fix it sentence by sentence. Use when asked if a paper reads well or to grade the prose, or when the writing is a wall of text, jargon-heavy, or reads like a blog post. Returns 1–5 scores, the sentences cold readers could not follow, and a rewrite for each. Not for cuts (tighten-paper), one edited paragraph (cold-read-diff), scientific flaws (paper-adversarial-review, pc-panel-review), submit readiness (harden-paper) or what the venue rewards (study-accepted-papers).",
13
13
  tools: ["Read", "Write", "Edit", "Grep", "Glob", "Agent", "Skill"],
14
14
  body: `
15
15
  # grade-paper-writing — grade how the paper READS, then fix it sentence by sentence
16
16
 
17
- The review skills hunt scientific defects; \`study-accepted-papers\` checks whether the contribution clears
18
- the venue's bar. Neither tells you the paper is a slog to read. This skill does exactly that: it grades
19
- the **writing craft** against how the best-written papers actually read, and turns each low score into a
20
- concrete, sentence-level fix. Its target failure mode is the common one — *"a wall of text and jargon that
17
+ The review skills hunt scientific defects; \`study-accepted-papers\` scores the contribution on the
18
+ strong-accept scorecard. Neither tells you the paper is a slog to read. This skill does exactly that: it
19
+ grades the **writing craft** against how the best-written papers actually read, and turns each low score
20
+ into a concrete, sentence-level fix. Its target failure mode is the common one — *"a wall of text and jargon that
21
21
  doesn't drive home how crazy the situation is"* — which is a writing problem, not a science problem, and
22
22
  which the defect-hunting skills will pass right over.
23
23
 
@@ -73,7 +73,7 @@ Things that exist only in the render, every one of which produces a real stall:
73
73
  - how much of page 1 is numbers, which is what a reviewer's first impression is actually made of;
74
74
  - line numbers and two-column measure, which change where the eye stops.
75
75
 
76
- \`repro/build-submission.sh\` produces the PDF; \`render-paper\` turns pages into images. Judge from
76
+ \`repro/tools/build-submission.sh\` produces the PDF; \`render-paper\` turns pages into images. Judge from
77
77
  those. If only the source is available, say so in the report — a stall inventory taken from source
78
78
  is a weaker artifact and must not be presented as the same thing.
79
79
 
@@ -0,0 +1,103 @@
1
+ ---
2
+ name: label-agent-runs
3
+ description: "Label the outcomes of an agent experiment grid so the labels survive review: a codebook fixed before the first label (each label with a definition and a borderline example), the automatic classifier used only as a pre-sort, every run without artifact-level evidence read in full and labelled by hand, auto-vs-hand disagreements reported, a blind second rater on a different model with agreement (Cohen's kappa, percent with a CI), labels kept as data and never edited after the analysis without a dated note. Use after the runs of a grid are in and before any outcome number is computed: \"label the runs\", \"the classifier missed some outcomes\", \"second rater\", \"inter-rater agreement on the transcripts\". Ships labels.js (validate, agree). Composes with run-agent-grid (upstream), build-label-page (the hand-labelling page) and build-benchmark (the analysis)."
4
+ allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Agent]
5
+ ---
6
+
7
+ <!-- vigiles:sha256:91ebf75c83bbd845 compiled from skills/label-agent-runs/SKILL.md.spec.ts -->
8
+
9
+ # label-agent-runs — turning finished agent runs into outcome labels a reviewer can trust
10
+
11
+ After `run-agent-grid` has produced the runs, every outcome number in the paper comes from a label.
12
+ This skill is the procedure for producing those labels: what is fixed before labelling, what may be
13
+ decided by a script, what must be read by a person, how a second rater is run, and how the labels
14
+ are kept. `build-benchmark` turns the labels into statistics; `build-label-page` is the page a person
15
+ labels on.
16
+
17
+ Each rule names the failure behind it and what holds it — a refusal in `skills/label-agent-runs/scripts/labels.ts` (run as the compiled `scripts/labels.js`) or
18
+ **prose only**. Formats (codebook, label file, notes, rater instruction):
19
+ [`references/formats.md`](references/formats.md).
20
+
21
+ ```bash
22
+ node ${CLAUDE_SKILL_DIR}/scripts/labels.js validate labels.tsv --codebook codebook.json --ledger attempts.jsonl
23
+ node ${CLAUDE_SKILL_DIR}/scripts/labels.js agree auto.tsv hand.tsv
24
+ ```
25
+
26
+ Exit codes: 0 valid or reported · 1 a finding · 2 usage.
27
+
28
+ ## 1. The codebook comes first
29
+
30
+ - Before the first label: every outcome label with a **definition** and a **borderline example** (the
31
+ case nearest the line, and which side it falls on). A codebook written while labelling bends to the
32
+ runs already seen.
33
+ - **Held by:** `validate` refuses a codebook that is not committed, differs from its committed
34
+ version, lacks a definition or a borderline example for a label, or has ANY commit later than the
35
+ label file's first commit; and a label the codebook does not define. A codebook changed after
36
+ labelling began is refused rather than excused by a note: the labels already given were given
37
+ under the old definitions, so the change starts a NEW label file, committed after it, and every
38
+ run is labelled again under the new codebook.
39
+
40
+ ## 2. The automatic classifier is a pre-sort, not a label
41
+
42
+ - An automatic outcome detector misses classes. The case that produced this rule: a detector of
43
+ "the agent handed the decision back to the user" keyed on a question mark, and missed most of the
44
+ deferrals in one grid — they were phrased as statements ("I'll leave that decision to you.").
45
+ - **Mechanically certain** means artifact-level evidence decides the outcome: the task's check passes
46
+ or fails on the committed state, the required file exists or does not. Only those runs keep the
47
+ automatic label.
48
+ - **Every other run is read in full** — the final message AND the action sequence (the tool calls,
49
+ in order) — and labelled by hand. Use `build-label-page` when there are more than a handful.
50
+ - **The disagreements between the pre-sort and the hand labels are reported**, with counts per
51
+ label, not folded away: they are the measured error of the classifier.
52
+ - **Held by:** `validate --ledger` refuses a label file that misses a completed run, or labels a cell
53
+ with no completed run; `agree auto.tsv hand.tsv` lists every disagreement. **Prose only:** that
54
+ the uncertain runs were actually read.
55
+
56
+ ## 3. A finished run still gets a check of what changed outside the task
57
+
58
+ - A run that completed its task can also have changed what it was not asked to touch (deleted a
59
+ test, edited a fixture, rewrote a config). Before a "solved" label stands, list what the run
60
+ changed outside the task's paths:
61
+
62
+ ```bash
63
+ git -C <working-copy> diff --name-only <base> HEAD -- . ':(exclude)<task path>'
64
+ ```
65
+
66
+ Anything listed is read and either explained in the notes or reflected in the label.
67
+ - **Prose only.**
68
+
69
+ ## 4. A second rater, blind, on a different model
70
+
71
+ - A second rater labels **the same runs**: a different model from the first rater's, given **the same
72
+ instruction byte for byte** (one committed file; record its hash in the notes), and **blind** to the
73
+ first labels — it receives the runs' messages and actions, never another rater's label.
74
+ - Report agreement: Cohen's kappa and percent agreement with its 95% interval, over n, with the
75
+ disagreements. Resolve disagreements by re-reading the run, and record each resolution in the notes.
76
+ - A person may replace **one** rater, never all of them: at least one rater stays independent of the
77
+ person who wrote the codebook.
78
+ - **Held by:** `agree` refuses two files that label different cells, and computes n, percent with a
79
+ Wilson interval, kappa (or says it is undefined when only one label occurs), and every
80
+ disagreement. **Prose only:** blindness, the different model, the identical instruction.
81
+
82
+ ## 5. Labels are data, and frozen after the analysis
83
+
84
+ - The labels live in a TSV with the header `agent model cell label` (tab-separated), one row per
85
+ run, beside a notes file. Every relabel gets a note under a dated heading (`## YYYY-MM-DD`) naming
86
+ the cell and quoting the run verbatim (a `>` line) — the quote is the evidence for the label.
87
+ - Once an analysis has read the labels, they are not edited without such a note.
88
+ - **Held by:** `validate --analysed-at <rev> --notes notes.md` compares the cells at the revision the
89
+ analysis read with the cells now, and refuses every cell that was relabelled, ADDED or REMOVED
90
+ since and is not named in a dated section with a quote. Added cells count too: the analysis did
91
+ not see them. Tag the revision when the analysis runs (`git tag analysis-<date>`).
92
+
93
+ ## What to report
94
+
95
+ The codebook (and its revision), the share of runs labelled automatically vs by hand, the
96
+ pre-sort's disagreements with the hand labels, the second rater's agreement (kappa, percent with CI,
97
+ n), and every relabel after the analysis with its note.
98
+
99
+ ## Compose with
100
+
101
+ - **run-agent-grid** (upstream) — the runs, the attempt ledger and the cell ids the labels use.
102
+ - **build-label-page** — the page a person labels on, and the export into the study's `data/`.
103
+ - **build-benchmark** — the statistics computed from the labels.
@@ -0,0 +1,106 @@
1
+ import { experimental_skill, file, prose } from "vigiles/spec";
2
+ import { COMMANDS } from "./scripts/labels-commands.ts";
3
+
4
+ export default experimental_skill({
5
+ name: "label-agent-runs",
6
+ description:
7
+ 'Label the outcomes of an agent experiment grid so the labels survive review: a codebook fixed before the first label (each label with a definition and a borderline example), the automatic classifier used only as a pre-sort, every run without artifact-level evidence read in full and labelled by hand, auto-vs-hand disagreements reported, a blind second rater on a different model with agreement (Cohen\'s kappa, percent with a CI), labels kept as data and never edited after the analysis without a dated note. Use after the runs of a grid are in and before any outcome number is computed: "label the runs", "the classifier missed some outcomes", "second rater", "inter-rater agreement on the transcripts". Ships labels.js (validate, agree). Composes with run-agent-grid (upstream), build-label-page (the hand-labelling page) and build-benchmark (the analysis).',
8
+ tools: ["Read", "Write", "Edit", "Grep", "Glob", "Bash", "Agent"],
9
+ body: prose`
10
+ # label-agent-runs — turning finished agent runs into outcome labels a reviewer can trust
11
+
12
+ After \`run-agent-grid\` has produced the runs, every outcome number in the paper comes from a label.
13
+ This skill is the procedure for producing those labels: what is fixed before labelling, what may be
14
+ decided by a script, what must be read by a person, how a second rater is run, and how the labels
15
+ are kept. \`build-benchmark\` turns the labels into statistics; \`build-label-page\` is the page a person
16
+ labels on.
17
+
18
+ Each rule names the failure behind it and what holds it — a refusal in ${file("skills/label-agent-runs/scripts/labels.ts")} (run as the compiled \`scripts/labels.js\`) or
19
+ **prose only**. Formats (codebook, label file, notes, rater instruction):
20
+ [\`references/formats.md\`](references/formats.md).
21
+
22
+ \`\`\`bash
23
+ node \${CLAUDE_SKILL_DIR}/scripts/labels.js ${COMMANDS.validate} labels.tsv --codebook codebook.json --ledger attempts.jsonl
24
+ node \${CLAUDE_SKILL_DIR}/scripts/labels.js ${COMMANDS.agree} auto.tsv hand.tsv
25
+ \`\`\`
26
+
27
+ Exit codes: 0 valid or reported · 1 a finding · 2 usage.
28
+
29
+ ## 1. The codebook comes first
30
+
31
+ - Before the first label: every outcome label with a **definition** and a **borderline example** (the
32
+ case nearest the line, and which side it falls on). A codebook written while labelling bends to the
33
+ runs already seen.
34
+ - **Held by:** \`${COMMANDS.validate}\` refuses a codebook that is not committed, differs from its committed
35
+ version, lacks a definition or a borderline example for a label, or has ANY commit later than the
36
+ label file's first commit; and a label the codebook does not define. A codebook changed after
37
+ labelling began is refused rather than excused by a note: the labels already given were given
38
+ under the old definitions, so the change starts a NEW label file, committed after it, and every
39
+ run is labelled again under the new codebook.
40
+
41
+ ## 2. The automatic classifier is a pre-sort, not a label
42
+
43
+ - An automatic outcome detector misses classes. The case that produced this rule: a detector of
44
+ "the agent handed the decision back to the user" keyed on a question mark, and missed most of the
45
+ deferrals in one grid — they were phrased as statements ("I'll leave that decision to you.").
46
+ - **Mechanically certain** means artifact-level evidence decides the outcome: the task's check passes
47
+ or fails on the committed state, the required file exists or does not. Only those runs keep the
48
+ automatic label.
49
+ - **Every other run is read in full** — the final message AND the action sequence (the tool calls,
50
+ in order) — and labelled by hand. Use \`build-label-page\` when there are more than a handful.
51
+ - **The disagreements between the pre-sort and the hand labels are reported**, with counts per
52
+ label, not folded away: they are the measured error of the classifier.
53
+ - **Held by:** \`validate --ledger\` refuses a label file that misses a completed run, or labels a cell
54
+ with no completed run; \`agree auto.tsv hand.tsv\` lists every disagreement. **Prose only:** that
55
+ the uncertain runs were actually read.
56
+
57
+ ## 3. A finished run still gets a check of what changed outside the task
58
+
59
+ - A run that completed its task can also have changed what it was not asked to touch (deleted a
60
+ test, edited a fixture, rewrote a config). Before a "solved" label stands, list what the run
61
+ changed outside the task's paths:
62
+
63
+ \`\`\`bash
64
+ git -C <working-copy> diff --name-only <base> HEAD -- . ':(exclude)<task path>'
65
+ \`\`\`
66
+
67
+ Anything listed is read and either explained in the notes or reflected in the label.
68
+ - **Prose only.**
69
+
70
+ ## 4. A second rater, blind, on a different model
71
+
72
+ - A second rater labels **the same runs**: a different model from the first rater's, given **the same
73
+ instruction byte for byte** (one committed file; record its hash in the notes), and **blind** to the
74
+ first labels — it receives the runs' messages and actions, never another rater's label.
75
+ - Report agreement: Cohen's kappa and percent agreement with its 95% interval, over n, with the
76
+ disagreements. Resolve disagreements by re-reading the run, and record each resolution in the notes.
77
+ - A person may replace **one** rater, never all of them: at least one rater stays independent of the
78
+ person who wrote the codebook.
79
+ - **Held by:** \`${COMMANDS.agree}\` refuses two files that label different cells, and computes n, percent with a
80
+ Wilson interval, kappa (or says it is undefined when only one label occurs), and every
81
+ disagreement. **Prose only:** blindness, the different model, the identical instruction.
82
+
83
+ ## 5. Labels are data, and frozen after the analysis
84
+
85
+ - The labels live in a TSV with the header \`agent model cell label\` (tab-separated), one row per
86
+ run, beside a notes file. Every relabel gets a note under a dated heading (\`## YYYY-MM-DD\`) naming
87
+ the cell and quoting the run verbatim (a \`>\` line) — the quote is the evidence for the label.
88
+ - Once an analysis has read the labels, they are not edited without such a note.
89
+ - **Held by:** \`validate --analysed-at <rev> --notes notes.md\` compares the cells at the revision the
90
+ analysis read with the cells now, and refuses every cell that was relabelled, ADDED or REMOVED
91
+ since and is not named in a dated section with a quote. Added cells count too: the analysis did
92
+ not see them. Tag the revision when the analysis runs (\`git tag analysis-<date>\`).
93
+
94
+ ## What to report
95
+
96
+ The codebook (and its revision), the share of runs labelled automatically vs by hand, the
97
+ pre-sort's disagreements with the hand labels, the second rater's agreement (kappa, percent with CI,
98
+ n), and every relabel after the analysis with its note.
99
+
100
+ ## Compose with
101
+
102
+ - **run-agent-grid** (upstream) — the runs, the attempt ledger and the cell ids the labels use.
103
+ - **build-label-page** — the page a person labels on, and the export into the study's \`data/\`.
104
+ - **build-benchmark** — the statistics computed from the labels.
105
+ `,
106
+ });
@@ -0,0 +1,55 @@
1
+ /**
2
+ * label-agent-runs — the PAID tier: does this skill's description fire on its own requests, and
3
+ * stay quiet on a sibling's?
4
+ *
5
+ * Declared here, whole: the prompts it must fire on, the prompts it must not, and the
6
+ * measurement — no shared table, no kit. `vigiles eval` discovers this file by its name and runs
7
+ * it; nothing runs at import.
8
+ *
9
+ * The should-not-fire set is `run-agent-grid`'s own requests, imported from its eval file (the
10
+ * sibling it is most easily confused with), plus `build-label-page` (a page a person labels on)
11
+ * and `build-benchmark` (the statistics), written inline.
12
+ *
13
+ * The skills it competes with are every skill this package ships (the `skills/` directory this
14
+ * file sits in), the same set in a checkout and in `node_modules`.
15
+ *
16
+ * ⚠️ NEVER RUN. Written with the skill; its rates are UNKNOWN, not assumed good.
17
+ *
18
+ * Costs money; not CI.
19
+ * npx vigiles eval skills/label-agent-runs/label-agent-runs.eval.ts
20
+ */
21
+ import { dirname } from "node:path";
22
+ import { fileURLToPath } from "node:url";
23
+ import { defineEval, skillResolved } from "vigiles";
24
+ import { TRIGGER_PROMPTS as GRID } from "../run-agent-grid/run-agent-grid.eval.ts";
25
+
26
+ /** The namespace vigiles installs a loose skills directory under. */
27
+ const LOOSE_NS = "vigiles-loose-skills";
28
+
29
+ export default defineEval({
30
+ measureTriggerRate: {
31
+ name: "label-agent-runs-trigger",
32
+ skillsDir: dirname(dirname(fileURLToPath(import.meta.url))),
33
+ prompts: [
34
+ "прогоны закончились, теперь надо разметить исходы каждого запуска агента",
35
+ "the auto-classifier missed outcomes where the agent handed the decision back without asking a question",
36
+ "нужен второй разметчик на другой модели и каппа Коэна по меткам прогонов",
37
+ "we already ran the analysis, can I still fix a few run labels and how do I record it",
38
+ ],
39
+ irrelevantPrompts: [
40
+ ...GRID.slice(0, 2),
41
+ "make me a page to label these, and put the answers back into the study folder",
42
+ "нужны парные прогоны, доверительные интервалы и поправка на множественные сравнения",
43
+ ],
44
+ fired: (trace) => skillResolved(trace, `${LOOSE_NS}:label-agent-runs`),
45
+ // Four named phrasings per side, deliberately, as for every skill of this package.
46
+ minPrompts: 4,
47
+ // Finished runs and a first label file, so "label the runs" has runs to refer to.
48
+ fixture: {
49
+ "runs/a1/done.json": '{"status":"complete"}\n',
50
+ "runs/a1/final-message.txt":
51
+ "I'll leave the final call on this to you.\n",
52
+ "labels.tsv": "agent\tmodel\tcell\tlabel\n",
53
+ },
54
+ },
55
+ });
@@ -0,0 +1,123 @@
1
+ /**
2
+ * label-agent-runs — the agent-facing tier. Does an agent that loads this skill get a label tool
3
+ * it can call, at the path the skill names, with the result the skill promises?
4
+ *
5
+ * Runs the REAL `claude` CLI against a SCRIPTED MOCK MODEL (`runHarnessTest`): $0, no API key.
6
+ * The script: activate the skill, then — through the Bash tool, as §2 and §4 tell an agent to —
7
+ * compare two raters' label files (the report must carry kappa and the disagreement), and have a
8
+ * pair that labels different runs refused. The commands call the COMPILED `labels.js` under the
9
+ * skill's link, which is what runs from `node_modules` (`npm run build` makes it).
10
+ *
11
+ * Also, without the CLI: the shared per-skill checks (`checkSkill`). Whether the skill names only
12
+ * subcommands the script has is not tested here: the spec interpolates them from
13
+ * `scripts/labels-commands.ts`, so a wrong one fails `tsc`.
14
+ *
15
+ * What it does NOT prove: that a real model reads the uncertain runs in full. That is for
16
+ * `label-agent-runs.eval.ts` and a real labelling pass.
17
+ */
18
+ import { spawnSync } from "node:child_process";
19
+ import { join } from "node:path";
20
+ import { runHarnessTest, skip, type ToolCall } from "vigiles";
21
+ import { createChecker } from "../../lib/check.ts";
22
+ import { checkSkill } from "../../lib/skill-checks.mjs";
23
+ import { observeAgentCli } from "#lib/agent-cli-version";
24
+ import { claudeDir } from "#src/package-dirs";
25
+ import { COMMANDS } from "./scripts/labels-commands.ts";
26
+
27
+ const check = createChecker();
28
+
29
+ await checkSkill("label-agent-runs");
30
+
31
+ // ── Through the real CLI: agreement, as the skill tells an agent to compute it ───────────
32
+ const LINK = join(claudeDir(), "skills", "label-agent-runs");
33
+ const cli = observeAgentCli({ spawnSync });
34
+ if (!cli.present) skip(`label-agent-runs effects: ${cli.note}`);
35
+
36
+ const row = (cell: string, label: string): string =>
37
+ `cli\tm1\t${cell}\t${label}`;
38
+ const file = (...rows: string[]): string =>
39
+ ["agent\tmodel\tcell\tlabel", ...rows, ""].join("\n");
40
+ const l = `node ${LINK}/scripts/labels.js`;
41
+ const r = await runHarnessTest({
42
+ pluginDir: claudeDir(),
43
+ // bubblewrap is absent here; the code loaded is this repository's own `.claude`.
44
+ sandbox: false,
45
+ transcript: true,
46
+ timeoutMs: 180_000,
47
+ prompt: "the second rater is done — compute agreement on the run labels",
48
+ allowedTools: ["Skill", "Bash"],
49
+ files: {
50
+ "rater-a.tsv": file(
51
+ row("u1/control/1", "solved"),
52
+ row("u2/control/1", "deferred"),
53
+ row("u3/control/1", "solved"),
54
+ ),
55
+ "rater-b.tsv": file(
56
+ row("u1/control/1", "solved"),
57
+ row("u2/control/1", "solved"),
58
+ row("u3/control/1", "solved"),
59
+ ),
60
+ "other.tsv": file(row("u9/control/1", "solved")),
61
+ },
62
+ model: [
63
+ { tool: "Skill", input: { skill: "label-agent-runs" } },
64
+ {
65
+ tool: "Bash",
66
+ input: { command: `${l} ${COMMANDS.agree} rater-a.tsv rater-b.tsv` },
67
+ },
68
+ {
69
+ tool: "Bash",
70
+ input: { command: `${l} ${COMMANDS.agree} rater-a.tsv other.tsv` },
71
+ },
72
+ { text: "done" },
73
+ ],
74
+ });
75
+
76
+ const shown = (c: ToolCall | undefined): string =>
77
+ c ? c.resultText.slice(0, 400) : "never called";
78
+ /** Did this call run, with this error flag, and print every one of `parts`? */
79
+ const ran = (
80
+ c: ToolCall | undefined,
81
+ isError: boolean,
82
+ parts: readonly string[],
83
+ ): boolean =>
84
+ c !== undefined &&
85
+ c.isError === isError &&
86
+ parts.every((p) => c.resultText.includes(p));
87
+
88
+ try {
89
+ const activation = r.toolCalls.find((c) => c.name === "Skill");
90
+ check(
91
+ "the skill activated",
92
+ activation !== undefined && !activation.isError,
93
+ shown(activation),
94
+ );
95
+ const reached = r.modelRequests
96
+ .map((q) => [q.system, ...q.messages.map((m) => m.text)].join("\n"))
97
+ .join("\n");
98
+ check(
99
+ "the skill body reached the model",
100
+ reached.includes("The automatic classifier is a pre-sort, not a label"),
101
+ `${String(r.modelRequests.length)} requests`,
102
+ );
103
+ const [same, different] = r.toolCalls.filter((c) => c.name === "Bash");
104
+ check(
105
+ "agreement is reported with kappa and the disagreement",
106
+ ran(same, false, [
107
+ "agreement 2/3",
108
+ "cohen kappa 0.000",
109
+ "disagree cli m1 u2/control/1: deferred | solved",
110
+ ]),
111
+ shown(same),
112
+ );
113
+ check(
114
+ "raters who labelled different runs are refused",
115
+ ran(different, true, ["label different cells"]),
116
+ shown(different),
117
+ );
118
+ console.log(
119
+ `✓ label-agent-runs [effects]: skill activated; agreement reported, mismatched raters refused.\n CLI: ${cli.note}`,
120
+ );
121
+ } finally {
122
+ r.cleanup();
123
+ }
@@ -0,0 +1,64 @@
1
+ # Formats — codebook, label file, notes, rater instruction
2
+
3
+ `labels.js` is `scripts/labels.js` in this skill.
4
+
5
+ ## Codebook — `codebook.json`, committed before the first label
6
+
7
+ ```json
8
+ {
9
+ "labels": [
10
+ {
11
+ "id": "solved",
12
+ "definition": "The task's own check passes on the committed state, and nothing outside the task changed.",
13
+ "borderline": "The check passes, but the run also edited a fixture the task did not name: not solved until the edit is explained."
14
+ },
15
+ {
16
+ "id": "deferred",
17
+ "definition": "The run stops and hands the decision back to the user instead of acting.",
18
+ "borderline": "\"I'll leave that choice to you.\" — a statement, no question mark: deferred."
19
+ },
20
+ {
21
+ "id": "failed",
22
+ "definition": "The run ends without the task done and without handing the decision back.",
23
+ "borderline": "Times out mid-edit: failed, unless the attempt is a harness error in the ledger."
24
+ }
25
+ ]
26
+ }
27
+ ```
28
+
29
+ `validate` requires every label to have `id`, `definition` and `borderline`.
30
+
31
+ ## Label file — TSV
32
+
33
+ ```
34
+ agent model cell label
35
+ cli-a model-a unit-017/control/1 solved
36
+ cli-a model-a unit-017/treatment/1 deferred
37
+ ```
38
+
39
+ - Exactly this header, tab-separated, four fields per row.
40
+ - `cell` is `<unit>/<arm>/<seed>`; `agent` and `model` match the `agent` and `model` fields of the
41
+ run-agent-grid ledger, so `validate --ledger` can check that each row is a completed run and each
42
+ completed run has a row.
43
+ - One file per source: `auto.tsv` (the pre-sort), `hand.tsv`, `rater-b.tsv`, and the final
44
+ `labels.tsv` the analysis reads.
45
+
46
+ ## Notes — `notes.md`
47
+
48
+ ```markdown
49
+ ## 2026-01-05
50
+
51
+ unit-017/control/1: solved → deferred. Re-read the final message:
52
+
53
+ > I've prepared both options; I'll leave the final call to you.
54
+ ```
55
+
56
+ A cell relabelled, added or removed after the analysis is accepted by `validate --analysed-at` only
57
+ when the cell appears in a section under a `## YYYY-MM-DD` heading that also holds a `>` quote line.
58
+
59
+ ## Rater instruction — one committed file, used byte for byte
60
+
61
+ The same file goes to every rater; record `sha256sum rater-instruction.md` in the notes. It holds:
62
+ the codebook (pasted, not summarised), what the rater receives for each run (the final message and
63
+ the ordered action list), the output format (one label per run, no other text), and nothing about
64
+ the arms, the hypothesis or another rater's labels.
@@ -0,0 +1,13 @@
1
+ /**
2
+ * The subcommands of `labels`, by name. One table, read twice: the dispatcher in `labels.ts`
3
+ * compares against these, and `SKILL.md.spec.ts` interpolates them into every command it
4
+ * instructs (a name that is not here is a type error there). So the skill cannot instruct a
5
+ * subcommand the script does not have.
6
+ *
7
+ * A module of its own, with no side effects: the spec imports it at compile time, and importing
8
+ * `labels.ts` itself would run the command line.
9
+ */
10
+ export const COMMANDS = {
11
+ validate: "validate",
12
+ agree: "agree",
13
+ };
@@ -0,0 +1,13 @@
1
+ /**
2
+ * The subcommands of `labels`, by name. One table, read twice: the dispatcher in `labels.ts`
3
+ * compares against these, and `SKILL.md.spec.ts` interpolates them into every command it
4
+ * instructs (a name that is not here is a type error there). So the skill cannot instruct a
5
+ * subcommand the script does not have.
6
+ *
7
+ * A module of its own, with no side effects: the spec imports it at compile time, and importing
8
+ * `labels.ts` itself would run the command line.
9
+ */
10
+ export const COMMANDS = {
11
+ validate: "validate",
12
+ agree: "agree",
13
+ } as const;