paperlint 9.0.1 → 10.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +16 -4
- package/README.md +68 -128
- package/dist/build.d.ts +1 -1
- package/dist/build.js +1 -1
- package/dist/cli.d.ts +5 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +52 -6
- package/dist/cli.js.map +1 -1
- package/dist/domain/layout-tree.d.ts +19 -0
- package/dist/domain/layout-tree.d.ts.map +1 -0
- package/dist/domain/layout-tree.js +32 -0
- package/dist/domain/layout-tree.js.map +1 -0
- package/dist/domain/lookup-cache.d.ts +2 -2
- package/dist/domain/lookup-cache.js +2 -2
- package/dist/domain/paper-layout.d.ts +81 -0
- package/dist/domain/paper-layout.d.ts.map +1 -0
- package/dist/domain/paper-layout.js +259 -0
- package/dist/domain/paper-layout.js.map +1 -0
- package/dist/domain/repro-layout.d.ts +80 -0
- package/dist/domain/repro-layout.d.ts.map +1 -0
- package/dist/domain/repro-layout.js +247 -0
- package/dist/domain/repro-layout.js.map +1 -0
- package/dist/domain/scaffold-layout.d.ts +28 -0
- package/dist/domain/scaffold-layout.d.ts.map +1 -0
- package/dist/domain/scaffold-layout.js +81 -0
- package/dist/domain/scaffold-layout.js.map +1 -0
- package/dist/eslint-rules/js-needs-reason.d.ts +55 -0
- package/dist/eslint-rules/js-needs-reason.d.ts.map +1 -0
- package/dist/eslint-rules/js-needs-reason.js +35 -0
- package/dist/eslint-rules/js-needs-reason.js.map +1 -0
- package/dist/eslint-rules/prereg-coverage.d.ts +98 -0
- package/dist/eslint-rules/prereg-coverage.d.ts.map +1 -0
- package/dist/eslint-rules/prereg-coverage.js +274 -0
- package/dist/eslint-rules/prereg-coverage.js.map +1 -0
- package/dist/folder-layout-hook.d.ts +21 -0
- package/dist/folder-layout-hook.d.ts.map +1 -0
- package/dist/folder-layout-hook.js +117 -0
- package/dist/folder-layout-hook.js.map +1 -0
- package/dist/folder-layout-rule.d.ts +22 -0
- package/dist/folder-layout-rule.d.ts.map +1 -0
- package/dist/folder-layout-rule.js +107 -0
- package/dist/folder-layout-rule.js.map +1 -0
- package/dist/init.js +1 -1
- package/dist/init.js.map +1 -1
- package/dist/new-paper.d.ts +4 -1
- package/dist/new-paper.d.ts.map +1 -1
- package/dist/new-paper.js +40 -6
- package/dist/new-paper.js.map +1 -1
- package/dist/package-dirs.d.ts +6 -0
- package/dist/package-dirs.d.ts.map +1 -1
- package/dist/package-dirs.js +6 -0
- package/dist/package-dirs.js.map +1 -1
- package/dist/references.d.ts +1 -1
- package/dist/references.d.ts.map +1 -1
- package/dist/references.js +2 -2
- package/dist/references.js.map +1 -1
- package/dist/repro-layout-rule.d.ts +16 -0
- package/dist/repro-layout-rule.d.ts.map +1 -0
- package/dist/repro-layout-rule.js +81 -0
- package/dist/repro-layout-rule.js.map +1 -0
- package/dist/skills/label-agent-runs/scripts/labels-commands.d.ts +13 -0
- package/dist/skills/label-agent-runs/scripts/labels.d.ts +2 -0
- package/dist/skills/run-agent-grid/scripts/grid-guard-commands.d.ts +20 -0
- package/dist/skills/run-agent-grid/scripts/grid-guard.d.ts +2 -0
- package/docs/ci.md +60 -0
- package/docs/configuration.md +2 -0
- package/docs/e2e.md +2 -2
- package/docs/install.md +20 -16
- package/docs/layout.md +154 -0
- package/docs/optional-rules.md +5 -4
- package/docs/references.md +5 -2
- package/docs/rules/paper/allowed-files-only.md +192 -0
- package/docs/rules/repro/allowed-files-only.md +128 -0
- package/docs/rules/study/declare-sample.md +146 -0
- package/docs/rules.md +53 -50
- package/docs/skills.md +8 -3
- package/docs/testing.md +13 -0
- package/docs/venues.md +94 -0
- package/eslint-rules/js-needs-reason.test.ts +141 -0
- package/eslint-rules/js-needs-reason.ts +77 -0
- package/eslint-rules/prereg-coverage.test.ts +448 -0
- package/eslint-rules/prereg-coverage.ts +419 -0
- package/eslint.config.mjs +22 -2
- package/lib/skill-trigger-cases.harness.mjs +51 -5
- package/lib/skill-trigger-cases.mjs +53 -15
- package/lib/skill-trigger-cases.test.ts +211 -0
- package/package.json +8 -4
- package/plugin/hooks/hooks.json +9 -0
- package/scripts/coverage-config.test.ts +5 -3
- package/skills/analyze-sibling-paper/SKILL.md +2 -2
- package/skills/analyze-sibling-paper/SKILL.md.spec.ts +1 -1
- package/skills/build-benchmark/SKILL.md +17 -1
- package/skills/build-benchmark/SKILL.md.spec.ts +16 -0
- package/skills/build-label-page/SKILL.md +120 -0
- package/skills/build-label-page/SKILL.md.spec.ts +131 -0
- package/skills/build-label-page/build-label-page.eval.mjs +15 -0
- package/skills/build-label-page/build-label-page.test.ts +959 -0
- package/skills/build-label-page/references/checklist.md +148 -0
- package/skills/build-label-page/references/example-generator.py +803 -0
- package/skills/find-venue/SKILL.md +5 -3
- package/skills/find-venue/SKILL.md.spec.ts +4 -2
- package/skills/grade-paper-writing/SKILL.md +7 -7
- package/skills/grade-paper-writing/SKILL.md.spec.ts +6 -6
- package/skills/label-agent-runs/SKILL.md +103 -0
- package/skills/label-agent-runs/SKILL.md.spec.ts +106 -0
- package/skills/label-agent-runs/label-agent-runs.eval.ts +55 -0
- package/skills/label-agent-runs/label-agent-runs.harness.ts +123 -0
- package/skills/label-agent-runs/references/formats.md +64 -0
- package/skills/label-agent-runs/scripts/labels-commands.js +13 -0
- package/skills/label-agent-runs/scripts/labels-commands.ts +13 -0
- package/skills/label-agent-runs/scripts/labels.js +396 -0
- package/skills/label-agent-runs/scripts/labels.test.ts +538 -0
- package/skills/label-agent-runs/scripts/labels.ts +501 -0
- package/skills/map-prior-work/SKILL.md +2 -2
- package/skills/map-prior-work/SKILL.md.spec.ts +1 -1
- package/skills/paper-pipeline/SKILL.md +15 -6
- package/skills/paper-pipeline/SKILL.md.spec.ts +14 -5
- package/skills/paper-pipeline/grade-paper-writing-ablation.eval.mjs +19 -9
- package/skills/paper-pipeline/references/acceptance-gate.md +19 -19
- package/skills/paper-pipeline/references/artifact-checklist.md +5 -0
- package/skills/paper-pipeline/references/pipeline-status-template.md +30 -1
- package/skills/paper-pipeline/repro/2026-08-07-fork-check.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-fork-check2.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-language-eval-pilot.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-language-eval-raw.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-08-framing-vs-vocabulary-oracle.log +48 -48
- package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-A6-oracle.log +8 -8
- package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-oracle.log +48 -48
- package/skills/paper-pipeline/repro/2026-08-08-parent-replication-gpw.log +2 -2
- package/skills/paper-pipeline/repro/2026-08-08-parent-replication.mjs +3 -3
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-after.log +81 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-before.log +81 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-setupdiff.log +59 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.jsonl +1 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.log +22 -0
- package/skills/paper-pipeline/repro/README.md +4 -0
- package/skills/paper-pipeline/scripts/extract-ref-facts.harness.mjs +2 -2
- package/skills/paper-pipeline/scripts/extract-ref-facts.mjs +2 -2
- package/skills/paper-pipeline/scripts/extract-ref-facts.test.mjs +4 -2
- package/skills/paper-pipeline/scripts/generated-code.harness.mjs +7 -2
- package/skills/paper-pipeline/scripts/generated-code.mjs +8 -4
- package/skills/paper-pipeline/scripts/generated-code.test.ts +15 -1
- package/skills/paper-status/SKILL.md +2 -2
- package/skills/paper-status/SKILL.md.spec.ts +1 -1
- package/skills/run-agent-grid/SKILL.md +181 -0
- package/skills/run-agent-grid/SKILL.md.spec.ts +184 -0
- package/skills/run-agent-grid/references/failure-catalog.md +36 -0
- package/skills/run-agent-grid/references/registry-and-ledger.md +142 -0
- package/skills/run-agent-grid/run-agent-grid.eval.ts +59 -0
- package/skills/run-agent-grid/run-agent-grid.harness.ts +115 -0
- package/skills/run-agent-grid/scripts/grid-guard-commands.js +18 -0
- package/skills/run-agent-grid/scripts/grid-guard-commands.ts +21 -0
- package/skills/run-agent-grid/scripts/grid-guard.js +700 -0
- package/skills/run-agent-grid/scripts/grid-guard.test.ts +1090 -0
- package/skills/run-agent-grid/scripts/grid-guard.ts +869 -0
- package/skills/study-accepted-papers/SKILL.md +163 -161
- package/skills/study-accepted-papers/SKILL.md.spec.ts +167 -165
- package/skills/study-accepted-papers/references/base-rates.md +69 -0
- package/skills/study-accepted-papers/references/mechanisms.md +181 -0
- package/skills/verify-citations/SKILL.md +2 -2
- package/skills/verify-citations/SKILL.md.spec.ts +1 -1
- package/src/build.ts +1 -1
- package/src/cli.harness.mjs +42 -0
- package/src/cli.test.ts +90 -6
- package/src/cli.ts +78 -5
- package/src/domain/layout-tree.test.ts +32 -0
- package/src/domain/layout-tree.ts +47 -0
- package/src/domain/lookup-cache.ts +2 -2
- package/src/domain/paper-layout.test.ts +264 -0
- package/src/domain/paper-layout.ts +336 -0
- package/src/domain/repro-layout.test.ts +222 -0
- package/src/domain/repro-layout.ts +311 -0
- package/src/domain/scaffold-layout.ts +104 -0
- package/src/folder-layout-hook.test.ts +214 -0
- package/src/folder-layout-hook.ts +141 -0
- package/src/folder-layout-rule.test.ts +235 -0
- package/src/folder-layout-rule.ts +151 -0
- package/src/hooks-settings.harness.ts +9 -8
- package/src/init.ts +1 -1
- package/src/layout-docs.test.ts +52 -0
- package/src/new-paper.harness.mjs +23 -1
- package/src/new-paper.test.ts +299 -6
- package/src/new-paper.ts +51 -5
- package/src/package-dirs.ts +8 -0
- package/src/paper-files.test.ts +9 -6
- package/src/references.test.ts +5 -5
- package/src/references.ts +2 -2
- package/src/repro-layout-rule.test.ts +114 -0
- package/src/repro-layout-rule.ts +121 -0
- package/src/rule-cases.test.ts +98 -0
- package/templates/paper/PIPELINE-STATUS.md +41 -7
- package/tsconfig.base.json +22 -0
- package/tsconfig.json +6 -24
- package/tsconfig.pkg.json +3 -1
- package/tsconfig.skills.json +14 -0
- package/tsconfig.src.json +12 -0
|
@@ -4,7 +4,7 @@ description: "Discover and rank real venues for a given paper, scored by authors
|
|
|
4
4
|
allowed-tools: [Read, Write, Grep, Glob, WebSearch, WebFetch, Agent]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:42d8e283fb7045f1 compiled from skills/find-venue/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# find-venue — rank real venues by what earns the credit, then keep or switch
|
|
10
10
|
|
|
@@ -43,7 +43,7 @@ rank = credit-weight × topic-fit × accept-probability × deadline-feasibility
|
|
|
43
43
|
prestige (a bonus, not a gate). **Non-archival ⇒ credit-weight ≈ 0** — it earns no authorship credit,
|
|
44
44
|
so it sinks to the bottom no matter how nice the CFP reads.
|
|
45
45
|
- **topic-fit** — how squarely the paper matches the CFP scope (a stretch fit raises reject risk).
|
|
46
|
-
- **accept-probability** — realistic odds given the paper's maturity and the venue
|
|
46
|
+
- **accept-probability** — realistic odds given the paper's maturity and how selective the venue is.
|
|
47
47
|
- **deadline-feasibility** — can the paper (and artifact) be ready by the deadline? A great venue you
|
|
48
48
|
miss is a zero.
|
|
49
49
|
- **remote-friendliness** — remote-OK is a real weight here, not a footnote.
|
|
@@ -84,7 +84,9 @@ rank = credit-weight × topic-fit × accept-probability × deadline-feasibility
|
|
|
84
84
|
- the venue cards, `presets/<venue>.md` in the paperlint package beside each venue preset (e.g.
|
|
85
85
|
`agenticdev.md`, `aisec.md`) — the winner gets a card with its HotCRP quirks beside its preset;
|
|
86
86
|
for a venue with no shipped preset, the project's own `./venues/<name>.md` beside its
|
|
87
|
-
`./venues/<name>.jsonc`. Plain data, not a new skill.
|
|
87
|
+
`./venues/<name>.jsonc`. Plain data, not a new skill. That `./venues/` is not the venue's corpus
|
|
88
|
+
of accepted papers, `<papers-root>/venues/<venue>/` (`study-accepted-papers`): the three places are
|
|
89
|
+
told apart in `docs/venues.md` in the paperlint package.
|
|
88
90
|
- `extend-paper` — the prestige-upgrade target this skill names but defers.
|
|
89
91
|
|
|
90
92
|
## Provenance
|
|
@@ -49,7 +49,7 @@ rank = credit-weight × topic-fit × accept-probability × deadline-feasibility
|
|
|
49
49
|
prestige (a bonus, not a gate). **Non-archival ⇒ credit-weight ≈ 0** — it earns no authorship credit,
|
|
50
50
|
so it sinks to the bottom no matter how nice the CFP reads.
|
|
51
51
|
- **topic-fit** — how squarely the paper matches the CFP scope (a stretch fit raises reject risk).
|
|
52
|
-
- **accept-probability** — realistic odds given the paper's maturity and the venue
|
|
52
|
+
- **accept-probability** — realistic odds given the paper's maturity and how selective the venue is.
|
|
53
53
|
- **deadline-feasibility** — can the paper (and artifact) be ready by the deadline? A great venue you
|
|
54
54
|
miss is a zero.
|
|
55
55
|
- **remote-friendliness** — remote-OK is a real weight here, not a footnote.
|
|
@@ -90,7 +90,9 @@ rank = credit-weight × topic-fit × accept-probability × deadline-feasibility
|
|
|
90
90
|
- the venue cards, \`presets/<venue>.md\` in the paperlint package beside each venue preset (e.g.
|
|
91
91
|
\`agenticdev.md\`, \`aisec.md\`) — the winner gets a card with its HotCRP quirks beside its preset;
|
|
92
92
|
for a venue with no shipped preset, the project's own \`./venues/<name>.md\` beside its
|
|
93
|
-
\`./venues/<name>.jsonc\`. Plain data, not a new skill.
|
|
93
|
+
\`./venues/<name>.jsonc\`. Plain data, not a new skill. That \`./venues/\` is not the venue's corpus
|
|
94
|
+
of accepted papers, \`<papers-root>/venues/<venue>/\` (\`study-accepted-papers\`): the three places are
|
|
95
|
+
told apart in \`docs/venues.md\` in the paperlint package.
|
|
94
96
|
- \`extend-paper\` — the prestige-upgrade target this skill names but defers.
|
|
95
97
|
|
|
96
98
|
## Provenance
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: grade-paper-writing
|
|
3
|
-
description:
|
|
3
|
+
description: Grade how a paper reads and fix it sentence by sentence. Use when asked if a paper reads well or to grade the prose, or when the writing is a wall of text, jargon-heavy, or reads like a blog post. Returns 1–5 scores, the sentences cold readers could not follow, and a rewrite for each. Not for cuts (tighten-paper), one edited paragraph (cold-read-diff), scientific flaws (paper-adversarial-review, pc-panel-review), submit readiness (harden-paper) or what the venue rewards (study-accepted-papers).
|
|
4
4
|
allowed-tools: [Read, Write, Edit, Grep, Glob, Agent, Skill]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:1897016b69f3f9b8 compiled from skills/grade-paper-writing/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# grade-paper-writing — grade how the paper READS, then fix it sentence by sentence
|
|
10
10
|
|
|
11
|
-
The review skills hunt scientific defects; `study-accepted-papers`
|
|
12
|
-
|
|
13
|
-
the **writing craft** against how the best-written papers actually read, and turns each low score
|
|
14
|
-
concrete, sentence-level fix. Its target failure mode is the common one — *"a wall of text and jargon that
|
|
11
|
+
The review skills hunt scientific defects; `study-accepted-papers` scores the contribution on the
|
|
12
|
+
strong-accept scorecard. Neither tells you the paper is a slog to read. This skill does exactly that: it
|
|
13
|
+
grades the **writing craft** against how the best-written papers actually read, and turns each low score
|
|
14
|
+
into a concrete, sentence-level fix. Its target failure mode is the common one — *"a wall of text and jargon that
|
|
15
15
|
doesn't drive home how crazy the situation is"* — which is a writing problem, not a science problem, and
|
|
16
16
|
which the defect-hunting skills will pass right over.
|
|
17
17
|
|
|
@@ -67,7 +67,7 @@ Things that exist only in the render, every one of which produces a real stall:
|
|
|
67
67
|
- how much of page 1 is numbers, which is what a reviewer's first impression is actually made of;
|
|
68
68
|
- line numbers and two-column measure, which change where the eye stops.
|
|
69
69
|
|
|
70
|
-
`repro/build-submission.sh` produces the PDF; `render-paper` turns pages into images. Judge from
|
|
70
|
+
`repro/tools/build-submission.sh` produces the PDF; `render-paper` turns pages into images. Judge from
|
|
71
71
|
those. If only the source is available, say so in the report — a stall inventory taken from source
|
|
72
72
|
is a weaker artifact and must not be presented as the same thing.
|
|
73
73
|
|
|
@@ -9,15 +9,15 @@ import { experimental_skill } from "vigiles/spec";
|
|
|
9
9
|
export default experimental_skill({
|
|
10
10
|
name: "grade-paper-writing",
|
|
11
11
|
description:
|
|
12
|
-
|
|
12
|
+
"Grade how a paper reads and fix it sentence by sentence. Use when asked if a paper reads well or to grade the prose, or when the writing is a wall of text, jargon-heavy, or reads like a blog post. Returns 1–5 scores, the sentences cold readers could not follow, and a rewrite for each. Not for cuts (tighten-paper), one edited paragraph (cold-read-diff), scientific flaws (paper-adversarial-review, pc-panel-review), submit readiness (harden-paper) or what the venue rewards (study-accepted-papers).",
|
|
13
13
|
tools: ["Read", "Write", "Edit", "Grep", "Glob", "Agent", "Skill"],
|
|
14
14
|
body: `
|
|
15
15
|
# grade-paper-writing — grade how the paper READS, then fix it sentence by sentence
|
|
16
16
|
|
|
17
|
-
The review skills hunt scientific defects; \`study-accepted-papers\`
|
|
18
|
-
|
|
19
|
-
the **writing craft** against how the best-written papers actually read, and turns each low score
|
|
20
|
-
concrete, sentence-level fix. Its target failure mode is the common one — *"a wall of text and jargon that
|
|
17
|
+
The review skills hunt scientific defects; \`study-accepted-papers\` scores the contribution on the
|
|
18
|
+
strong-accept scorecard. Neither tells you the paper is a slog to read. This skill does exactly that: it
|
|
19
|
+
grades the **writing craft** against how the best-written papers actually read, and turns each low score
|
|
20
|
+
into a concrete, sentence-level fix. Its target failure mode is the common one — *"a wall of text and jargon that
|
|
21
21
|
doesn't drive home how crazy the situation is"* — which is a writing problem, not a science problem, and
|
|
22
22
|
which the defect-hunting skills will pass right over.
|
|
23
23
|
|
|
@@ -73,7 +73,7 @@ Things that exist only in the render, every one of which produces a real stall:
|
|
|
73
73
|
- how much of page 1 is numbers, which is what a reviewer's first impression is actually made of;
|
|
74
74
|
- line numbers and two-column measure, which change where the eye stops.
|
|
75
75
|
|
|
76
|
-
\`repro/build-submission.sh\` produces the PDF; \`render-paper\` turns pages into images. Judge from
|
|
76
|
+
\`repro/tools/build-submission.sh\` produces the PDF; \`render-paper\` turns pages into images. Judge from
|
|
77
77
|
those. If only the source is available, say so in the report — a stall inventory taken from source
|
|
78
78
|
is a weaker artifact and must not be presented as the same thing.
|
|
79
79
|
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: label-agent-runs
|
|
3
|
+
description: "Label the outcomes of an agent experiment grid so the labels survive review: a codebook fixed before the first label (each label with a definition and a borderline example), the automatic classifier used only as a pre-sort, every run without artifact-level evidence read in full and labelled by hand, auto-vs-hand disagreements reported, a blind second rater on a different model with agreement (Cohen's kappa, percent with a CI), labels kept as data and never edited after the analysis without a dated note. Use after the runs of a grid are in and before any outcome number is computed: \"label the runs\", \"the classifier missed some outcomes\", \"second rater\", \"inter-rater agreement on the transcripts\". Ships labels.js (validate, agree). Composes with run-agent-grid (upstream), build-label-page (the hand-labelling page) and build-benchmark (the analysis)."
|
|
4
|
+
allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Agent]
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
<!-- vigiles:sha256:91ebf75c83bbd845 compiled from skills/label-agent-runs/SKILL.md.spec.ts -->
|
|
8
|
+
|
|
9
|
+
# label-agent-runs — turning finished agent runs into outcome labels a reviewer can trust
|
|
10
|
+
|
|
11
|
+
After `run-agent-grid` has produced the runs, every outcome number in the paper comes from a label.
|
|
12
|
+
This skill is the procedure for producing those labels: what is fixed before labelling, what may be
|
|
13
|
+
decided by a script, what must be read by a person, how a second rater is run, and how the labels
|
|
14
|
+
are kept. `build-benchmark` turns the labels into statistics; `build-label-page` is the page a person
|
|
15
|
+
labels on.
|
|
16
|
+
|
|
17
|
+
Each rule names the failure behind it and what holds it — a refusal in `skills/label-agent-runs/scripts/labels.ts` (run as the compiled `scripts/labels.js`) or
|
|
18
|
+
**prose only**. Formats (codebook, label file, notes, rater instruction):
|
|
19
|
+
[`references/formats.md`](references/formats.md).
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
node ${CLAUDE_SKILL_DIR}/scripts/labels.js validate labels.tsv --codebook codebook.json --ledger attempts.jsonl
|
|
23
|
+
node ${CLAUDE_SKILL_DIR}/scripts/labels.js agree auto.tsv hand.tsv
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
Exit codes: 0 valid or reported · 1 a finding · 2 usage.
|
|
27
|
+
|
|
28
|
+
## 1. The codebook comes first
|
|
29
|
+
|
|
30
|
+
- Before the first label: every outcome label with a **definition** and a **borderline example** (the
|
|
31
|
+
case nearest the line, and which side it falls on). A codebook written while labelling bends to the
|
|
32
|
+
runs already seen.
|
|
33
|
+
- **Held by:** `validate` refuses a codebook that is not committed, differs from its committed
|
|
34
|
+
version, lacks a definition or a borderline example for a label, or has ANY commit later than the
|
|
35
|
+
label file's first commit; and a label the codebook does not define. A codebook changed after
|
|
36
|
+
labelling began is refused rather than excused by a note: the labels already given were given
|
|
37
|
+
under the old definitions, so the change starts a NEW label file, committed after it, and every
|
|
38
|
+
run is labelled again under the new codebook.
|
|
39
|
+
|
|
40
|
+
## 2. The automatic classifier is a pre-sort, not a label
|
|
41
|
+
|
|
42
|
+
- An automatic outcome detector misses classes. The case that produced this rule: a detector of
|
|
43
|
+
"the agent handed the decision back to the user" keyed on a question mark, and missed most of the
|
|
44
|
+
deferrals in one grid — they were phrased as statements ("I'll leave that decision to you.").
|
|
45
|
+
- **Mechanically certain** means artifact-level evidence decides the outcome: the task's check passes
|
|
46
|
+
or fails on the committed state, the required file exists or does not. Only those runs keep the
|
|
47
|
+
automatic label.
|
|
48
|
+
- **Every other run is read in full** — the final message AND the action sequence (the tool calls,
|
|
49
|
+
in order) — and labelled by hand. Use `build-label-page` when there are more than a handful.
|
|
50
|
+
- **The disagreements between the pre-sort and the hand labels are reported**, with counts per
|
|
51
|
+
label, not folded away: they are the measured error of the classifier.
|
|
52
|
+
- **Held by:** `validate --ledger` refuses a label file that misses a completed run, or labels a cell
|
|
53
|
+
with no completed run; `agree auto.tsv hand.tsv` lists every disagreement. **Prose only:** that
|
|
54
|
+
the uncertain runs were actually read.
|
|
55
|
+
|
|
56
|
+
## 3. A finished run still gets a check of what changed outside the task
|
|
57
|
+
|
|
58
|
+
- A run that completed its task can also have changed what it was not asked to touch (deleted a
|
|
59
|
+
test, edited a fixture, rewrote a config). Before a "solved" label stands, list what the run
|
|
60
|
+
changed outside the task's paths:
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
git -C <working-copy> diff --name-only <base> HEAD -- . ':(exclude)<task path>'
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Anything listed is read and either explained in the notes or reflected in the label.
|
|
67
|
+
- **Prose only.**
|
|
68
|
+
|
|
69
|
+
## 4. A second rater, blind, on a different model
|
|
70
|
+
|
|
71
|
+
- A second rater labels **the same runs**: a different model from the first rater's, given **the same
|
|
72
|
+
instruction byte for byte** (one committed file; record its hash in the notes), and **blind** to the
|
|
73
|
+
first labels — it receives the runs' messages and actions, never another rater's label.
|
|
74
|
+
- Report agreement: Cohen's kappa and percent agreement with its 95% interval, over n, with the
|
|
75
|
+
disagreements. Resolve disagreements by re-reading the run, and record each resolution in the notes.
|
|
76
|
+
- A person may replace **one** rater, never all of them: at least one rater stays independent of the
|
|
77
|
+
person who wrote the codebook.
|
|
78
|
+
- **Held by:** `agree` refuses two files that label different cells, and computes n, percent with a
|
|
79
|
+
Wilson interval, kappa (or says it is undefined when only one label occurs), and every
|
|
80
|
+
disagreement. **Prose only:** blindness, the different model, the identical instruction.
|
|
81
|
+
|
|
82
|
+
## 5. Labels are data, and frozen after the analysis
|
|
83
|
+
|
|
84
|
+
- The labels live in a TSV with the header `agent model cell label` (tab-separated), one row per
|
|
85
|
+
run, beside a notes file. Every relabel gets a note under a dated heading (`## YYYY-MM-DD`) naming
|
|
86
|
+
the cell and quoting the run verbatim (a `>` line) — the quote is the evidence for the label.
|
|
87
|
+
- Once an analysis has read the labels, they are not edited without such a note.
|
|
88
|
+
- **Held by:** `validate --analysed-at <rev> --notes notes.md` compares the cells at the revision the
|
|
89
|
+
analysis read with the cells now, and refuses every cell that was relabelled, ADDED or REMOVED
|
|
90
|
+
since and is not named in a dated section with a quote. Added cells count too: the analysis did
|
|
91
|
+
not see them. Tag the revision when the analysis runs (`git tag analysis-<date>`).
|
|
92
|
+
|
|
93
|
+
## What to report
|
|
94
|
+
|
|
95
|
+
The codebook (and its revision), the share of runs labelled automatically vs by hand, the
|
|
96
|
+
pre-sort's disagreements with the hand labels, the second rater's agreement (kappa, percent with CI,
|
|
97
|
+
n), and every relabel after the analysis with its note.
|
|
98
|
+
|
|
99
|
+
## Compose with
|
|
100
|
+
|
|
101
|
+
- **run-agent-grid** (upstream) — the runs, the attempt ledger and the cell ids the labels use.
|
|
102
|
+
- **build-label-page** — the page a person labels on, and the export into the study's `data/`.
|
|
103
|
+
- **build-benchmark** — the statistics computed from the labels.
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { experimental_skill, file, prose } from "vigiles/spec";
|
|
2
|
+
import { COMMANDS } from "./scripts/labels-commands.ts";
|
|
3
|
+
|
|
4
|
+
export default experimental_skill({
|
|
5
|
+
name: "label-agent-runs",
|
|
6
|
+
description:
|
|
7
|
+
'Label the outcomes of an agent experiment grid so the labels survive review: a codebook fixed before the first label (each label with a definition and a borderline example), the automatic classifier used only as a pre-sort, every run without artifact-level evidence read in full and labelled by hand, auto-vs-hand disagreements reported, a blind second rater on a different model with agreement (Cohen\'s kappa, percent with a CI), labels kept as data and never edited after the analysis without a dated note. Use after the runs of a grid are in and before any outcome number is computed: "label the runs", "the classifier missed some outcomes", "second rater", "inter-rater agreement on the transcripts". Ships labels.js (validate, agree). Composes with run-agent-grid (upstream), build-label-page (the hand-labelling page) and build-benchmark (the analysis).',
|
|
8
|
+
tools: ["Read", "Write", "Edit", "Grep", "Glob", "Bash", "Agent"],
|
|
9
|
+
body: prose`
|
|
10
|
+
# label-agent-runs — turning finished agent runs into outcome labels a reviewer can trust
|
|
11
|
+
|
|
12
|
+
After \`run-agent-grid\` has produced the runs, every outcome number in the paper comes from a label.
|
|
13
|
+
This skill is the procedure for producing those labels: what is fixed before labelling, what may be
|
|
14
|
+
decided by a script, what must be read by a person, how a second rater is run, and how the labels
|
|
15
|
+
are kept. \`build-benchmark\` turns the labels into statistics; \`build-label-page\` is the page a person
|
|
16
|
+
labels on.
|
|
17
|
+
|
|
18
|
+
Each rule names the failure behind it and what holds it — a refusal in ${file("skills/label-agent-runs/scripts/labels.ts")} (run as the compiled \`scripts/labels.js\`) or
|
|
19
|
+
**prose only**. Formats (codebook, label file, notes, rater instruction):
|
|
20
|
+
[\`references/formats.md\`](references/formats.md).
|
|
21
|
+
|
|
22
|
+
\`\`\`bash
|
|
23
|
+
node \${CLAUDE_SKILL_DIR}/scripts/labels.js ${COMMANDS.validate} labels.tsv --codebook codebook.json --ledger attempts.jsonl
|
|
24
|
+
node \${CLAUDE_SKILL_DIR}/scripts/labels.js ${COMMANDS.agree} auto.tsv hand.tsv
|
|
25
|
+
\`\`\`
|
|
26
|
+
|
|
27
|
+
Exit codes: 0 valid or reported · 1 a finding · 2 usage.
|
|
28
|
+
|
|
29
|
+
## 1. The codebook comes first
|
|
30
|
+
|
|
31
|
+
- Before the first label: every outcome label with a **definition** and a **borderline example** (the
|
|
32
|
+
case nearest the line, and which side it falls on). A codebook written while labelling bends to the
|
|
33
|
+
runs already seen.
|
|
34
|
+
- **Held by:** \`${COMMANDS.validate}\` refuses a codebook that is not committed, differs from its committed
|
|
35
|
+
version, lacks a definition or a borderline example for a label, or has ANY commit later than the
|
|
36
|
+
label file's first commit; and a label the codebook does not define. A codebook changed after
|
|
37
|
+
labelling began is refused rather than excused by a note: the labels already given were given
|
|
38
|
+
under the old definitions, so the change starts a NEW label file, committed after it, and every
|
|
39
|
+
run is labelled again under the new codebook.
|
|
40
|
+
|
|
41
|
+
## 2. The automatic classifier is a pre-sort, not a label
|
|
42
|
+
|
|
43
|
+
- An automatic outcome detector misses classes. The case that produced this rule: a detector of
|
|
44
|
+
"the agent handed the decision back to the user" keyed on a question mark, and missed most of the
|
|
45
|
+
deferrals in one grid — they were phrased as statements ("I'll leave that decision to you.").
|
|
46
|
+
- **Mechanically certain** means artifact-level evidence decides the outcome: the task's check passes
|
|
47
|
+
or fails on the committed state, the required file exists or does not. Only those runs keep the
|
|
48
|
+
automatic label.
|
|
49
|
+
- **Every other run is read in full** — the final message AND the action sequence (the tool calls,
|
|
50
|
+
in order) — and labelled by hand. Use \`build-label-page\` when there are more than a handful.
|
|
51
|
+
- **The disagreements between the pre-sort and the hand labels are reported**, with counts per
|
|
52
|
+
label, not folded away: they are the measured error of the classifier.
|
|
53
|
+
- **Held by:** \`validate --ledger\` refuses a label file that misses a completed run, or labels a cell
|
|
54
|
+
with no completed run; \`agree auto.tsv hand.tsv\` lists every disagreement. **Prose only:** that
|
|
55
|
+
the uncertain runs were actually read.
|
|
56
|
+
|
|
57
|
+
## 3. A finished run still gets a check of what changed outside the task
|
|
58
|
+
|
|
59
|
+
- A run that completed its task can also have changed what it was not asked to touch (deleted a
|
|
60
|
+
test, edited a fixture, rewrote a config). Before a "solved" label stands, list what the run
|
|
61
|
+
changed outside the task's paths:
|
|
62
|
+
|
|
63
|
+
\`\`\`bash
|
|
64
|
+
git -C <working-copy> diff --name-only <base> HEAD -- . ':(exclude)<task path>'
|
|
65
|
+
\`\`\`
|
|
66
|
+
|
|
67
|
+
Anything listed is read and either explained in the notes or reflected in the label.
|
|
68
|
+
- **Prose only.**
|
|
69
|
+
|
|
70
|
+
## 4. A second rater, blind, on a different model
|
|
71
|
+
|
|
72
|
+
- A second rater labels **the same runs**: a different model from the first rater's, given **the same
|
|
73
|
+
instruction byte for byte** (one committed file; record its hash in the notes), and **blind** to the
|
|
74
|
+
first labels — it receives the runs' messages and actions, never another rater's label.
|
|
75
|
+
- Report agreement: Cohen's kappa and percent agreement with its 95% interval, over n, with the
|
|
76
|
+
disagreements. Resolve disagreements by re-reading the run, and record each resolution in the notes.
|
|
77
|
+
- A person may replace **one** rater, never all of them: at least one rater stays independent of the
|
|
78
|
+
person who wrote the codebook.
|
|
79
|
+
- **Held by:** \`${COMMANDS.agree}\` refuses two files that label different cells, and computes n, percent with a
|
|
80
|
+
Wilson interval, kappa (or says it is undefined when only one label occurs), and every
|
|
81
|
+
disagreement. **Prose only:** blindness, the different model, the identical instruction.
|
|
82
|
+
|
|
83
|
+
## 5. Labels are data, and frozen after the analysis
|
|
84
|
+
|
|
85
|
+
- The labels live in a TSV with the header \`agent model cell label\` (tab-separated), one row per
|
|
86
|
+
run, beside a notes file. Every relabel gets a note under a dated heading (\`## YYYY-MM-DD\`) naming
|
|
87
|
+
the cell and quoting the run verbatim (a \`>\` line) — the quote is the evidence for the label.
|
|
88
|
+
- Once an analysis has read the labels, they are not edited without such a note.
|
|
89
|
+
- **Held by:** \`validate --analysed-at <rev> --notes notes.md\` compares the cells at the revision the
|
|
90
|
+
analysis read with the cells now, and refuses every cell that was relabelled, ADDED or REMOVED
|
|
91
|
+
since and is not named in a dated section with a quote. Added cells count too: the analysis did
|
|
92
|
+
not see them. Tag the revision when the analysis runs (\`git tag analysis-<date>\`).
|
|
93
|
+
|
|
94
|
+
## What to report
|
|
95
|
+
|
|
96
|
+
The codebook (and its revision), the share of runs labelled automatically vs by hand, the
|
|
97
|
+
pre-sort's disagreements with the hand labels, the second rater's agreement (kappa, percent with CI,
|
|
98
|
+
n), and every relabel after the analysis with its note.
|
|
99
|
+
|
|
100
|
+
## Compose with
|
|
101
|
+
|
|
102
|
+
- **run-agent-grid** (upstream) — the runs, the attempt ledger and the cell ids the labels use.
|
|
103
|
+
- **build-label-page** — the page a person labels on, and the export into the study's \`data/\`.
|
|
104
|
+
- **build-benchmark** — the statistics computed from the labels.
|
|
105
|
+
`,
|
|
106
|
+
});
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* label-agent-runs — the PAID tier: does this skill's description fire on its own requests, and
|
|
3
|
+
* stay quiet on a sibling's?
|
|
4
|
+
*
|
|
5
|
+
* Declared here, whole: the prompts it must fire on, the prompts it must not, and the
|
|
6
|
+
* measurement — no shared table, no kit. `vigiles eval` discovers this file by its name and runs
|
|
7
|
+
* it; nothing runs at import.
|
|
8
|
+
*
|
|
9
|
+
* The should-not-fire set is `run-agent-grid`'s own requests, imported from its eval file (the
|
|
10
|
+
* sibling it is most easily confused with), plus `build-label-page` (a page a person labels on)
|
|
11
|
+
* and `build-benchmark` (the statistics), written inline.
|
|
12
|
+
*
|
|
13
|
+
* The skills it competes with are every skill this package ships (the `skills/` directory this
|
|
14
|
+
* file sits in), the same set in a checkout and in `node_modules`.
|
|
15
|
+
*
|
|
16
|
+
* ⚠️ NEVER RUN. Written with the skill; its rates are UNKNOWN, not assumed good.
|
|
17
|
+
*
|
|
18
|
+
* Costs money; not CI.
|
|
19
|
+
* npx vigiles eval skills/label-agent-runs/label-agent-runs.eval.ts
|
|
20
|
+
*/
|
|
21
|
+
import { dirname } from "node:path";
|
|
22
|
+
import { fileURLToPath } from "node:url";
|
|
23
|
+
import { defineEval, skillResolved } from "vigiles";
|
|
24
|
+
import { TRIGGER_PROMPTS as GRID } from "../run-agent-grid/run-agent-grid.eval.ts";
|
|
25
|
+
|
|
26
|
+
/** The namespace vigiles installs a loose skills directory under. */
|
|
27
|
+
const LOOSE_NS = "vigiles-loose-skills";
|
|
28
|
+
|
|
29
|
+
export default defineEval({
|
|
30
|
+
measureTriggerRate: {
|
|
31
|
+
name: "label-agent-runs-trigger",
|
|
32
|
+
skillsDir: dirname(dirname(fileURLToPath(import.meta.url))),
|
|
33
|
+
prompts: [
|
|
34
|
+
"прогоны закончились, теперь надо разметить исходы каждого запуска агента",
|
|
35
|
+
"the auto-classifier missed outcomes where the agent handed the decision back without asking a question",
|
|
36
|
+
"нужен второй разметчик на другой модели и каппа Коэна по меткам прогонов",
|
|
37
|
+
"we already ran the analysis, can I still fix a few run labels and how do I record it",
|
|
38
|
+
],
|
|
39
|
+
irrelevantPrompts: [
|
|
40
|
+
...GRID.slice(0, 2),
|
|
41
|
+
"make me a page to label these, and put the answers back into the study folder",
|
|
42
|
+
"нужны парные прогоны, доверительные интервалы и поправка на множественные сравнения",
|
|
43
|
+
],
|
|
44
|
+
fired: (trace) => skillResolved(trace, `${LOOSE_NS}:label-agent-runs`),
|
|
45
|
+
// Four named phrasings per side, deliberately, as for every skill of this package.
|
|
46
|
+
minPrompts: 4,
|
|
47
|
+
// Finished runs and a first label file, so "label the runs" has runs to refer to.
|
|
48
|
+
fixture: {
|
|
49
|
+
"runs/a1/done.json": '{"status":"complete"}\n',
|
|
50
|
+
"runs/a1/final-message.txt":
|
|
51
|
+
"I'll leave the final call on this to you.\n",
|
|
52
|
+
"labels.tsv": "agent\tmodel\tcell\tlabel\n",
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
});
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* label-agent-runs — the agent-facing tier. Does an agent that loads this skill get a label tool
|
|
3
|
+
* it can call, at the path the skill names, with the result the skill promises?
|
|
4
|
+
*
|
|
5
|
+
* Runs the REAL `claude` CLI against a SCRIPTED MOCK MODEL (`runHarnessTest`): $0, no API key.
|
|
6
|
+
* The script: activate the skill, then — through the Bash tool, as §2 and §4 tell an agent to —
|
|
7
|
+
* compare two raters' label files (the report must carry kappa and the disagreement), and have a
|
|
8
|
+
* pair that labels different runs refused. The commands call the COMPILED `labels.js` under the
|
|
9
|
+
* skill's link, which is what runs from `node_modules` (`npm run build` makes it).
|
|
10
|
+
*
|
|
11
|
+
* Also, without the CLI: the shared per-skill checks (`checkSkill`). Whether the skill names only
|
|
12
|
+
* subcommands the script has is not tested here: the spec interpolates them from
|
|
13
|
+
* `scripts/labels-commands.ts`, so a wrong one fails `tsc`.
|
|
14
|
+
*
|
|
15
|
+
* What it does NOT prove: that a real model reads the uncertain runs in full. That is for
|
|
16
|
+
* `label-agent-runs.eval.ts` and a real labelling pass.
|
|
17
|
+
*/
|
|
18
|
+
import { spawnSync } from "node:child_process";
|
|
19
|
+
import { join } from "node:path";
|
|
20
|
+
import { runHarnessTest, skip, type ToolCall } from "vigiles";
|
|
21
|
+
import { createChecker } from "../../lib/check.ts";
|
|
22
|
+
import { checkSkill } from "../../lib/skill-checks.mjs";
|
|
23
|
+
import { observeAgentCli } from "#lib/agent-cli-version";
|
|
24
|
+
import { claudeDir } from "#src/package-dirs";
|
|
25
|
+
import { COMMANDS } from "./scripts/labels-commands.ts";
|
|
26
|
+
|
|
27
|
+
const check = createChecker();
|
|
28
|
+
|
|
29
|
+
await checkSkill("label-agent-runs");
|
|
30
|
+
|
|
31
|
+
// ── Through the real CLI: agreement, as the skill tells an agent to compute it ───────────
|
|
32
|
+
const LINK = join(claudeDir(), "skills", "label-agent-runs");
|
|
33
|
+
const cli = observeAgentCli({ spawnSync });
|
|
34
|
+
if (!cli.present) skip(`label-agent-runs effects: ${cli.note}`);
|
|
35
|
+
|
|
36
|
+
const row = (cell: string, label: string): string =>
|
|
37
|
+
`cli\tm1\t${cell}\t${label}`;
|
|
38
|
+
const file = (...rows: string[]): string =>
|
|
39
|
+
["agent\tmodel\tcell\tlabel", ...rows, ""].join("\n");
|
|
40
|
+
const l = `node ${LINK}/scripts/labels.js`;
|
|
41
|
+
const r = await runHarnessTest({
|
|
42
|
+
pluginDir: claudeDir(),
|
|
43
|
+
// bubblewrap is absent here; the code loaded is this repository's own `.claude`.
|
|
44
|
+
sandbox: false,
|
|
45
|
+
transcript: true,
|
|
46
|
+
timeoutMs: 180_000,
|
|
47
|
+
prompt: "the second rater is done — compute agreement on the run labels",
|
|
48
|
+
allowedTools: ["Skill", "Bash"],
|
|
49
|
+
files: {
|
|
50
|
+
"rater-a.tsv": file(
|
|
51
|
+
row("u1/control/1", "solved"),
|
|
52
|
+
row("u2/control/1", "deferred"),
|
|
53
|
+
row("u3/control/1", "solved"),
|
|
54
|
+
),
|
|
55
|
+
"rater-b.tsv": file(
|
|
56
|
+
row("u1/control/1", "solved"),
|
|
57
|
+
row("u2/control/1", "solved"),
|
|
58
|
+
row("u3/control/1", "solved"),
|
|
59
|
+
),
|
|
60
|
+
"other.tsv": file(row("u9/control/1", "solved")),
|
|
61
|
+
},
|
|
62
|
+
model: [
|
|
63
|
+
{ tool: "Skill", input: { skill: "label-agent-runs" } },
|
|
64
|
+
{
|
|
65
|
+
tool: "Bash",
|
|
66
|
+
input: { command: `${l} ${COMMANDS.agree} rater-a.tsv rater-b.tsv` },
|
|
67
|
+
},
|
|
68
|
+
{
|
|
69
|
+
tool: "Bash",
|
|
70
|
+
input: { command: `${l} ${COMMANDS.agree} rater-a.tsv other.tsv` },
|
|
71
|
+
},
|
|
72
|
+
{ text: "done" },
|
|
73
|
+
],
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const shown = (c: ToolCall | undefined): string =>
|
|
77
|
+
c ? c.resultText.slice(0, 400) : "never called";
|
|
78
|
+
/** Did this call run, with this error flag, and print every one of `parts`? */
|
|
79
|
+
const ran = (
|
|
80
|
+
c: ToolCall | undefined,
|
|
81
|
+
isError: boolean,
|
|
82
|
+
parts: readonly string[],
|
|
83
|
+
): boolean =>
|
|
84
|
+
c !== undefined &&
|
|
85
|
+
c.isError === isError &&
|
|
86
|
+
parts.every((p) => c.resultText.includes(p));
|
|
87
|
+
|
|
88
|
+
try {
|
|
89
|
+
const activation = r.toolCalls.find((c) => c.name === "Skill");
|
|
90
|
+
check(
|
|
91
|
+
"the skill activated",
|
|
92
|
+
activation !== undefined && !activation.isError,
|
|
93
|
+
shown(activation),
|
|
94
|
+
);
|
|
95
|
+
const reached = r.modelRequests
|
|
96
|
+
.map((q) => [q.system, ...q.messages.map((m) => m.text)].join("\n"))
|
|
97
|
+
.join("\n");
|
|
98
|
+
check(
|
|
99
|
+
"the skill body reached the model",
|
|
100
|
+
reached.includes("The automatic classifier is a pre-sort, not a label"),
|
|
101
|
+
`${String(r.modelRequests.length)} requests`,
|
|
102
|
+
);
|
|
103
|
+
const [same, different] = r.toolCalls.filter((c) => c.name === "Bash");
|
|
104
|
+
check(
|
|
105
|
+
"agreement is reported with kappa and the disagreement",
|
|
106
|
+
ran(same, false, [
|
|
107
|
+
"agreement 2/3",
|
|
108
|
+
"cohen kappa 0.000",
|
|
109
|
+
"disagree cli m1 u2/control/1: deferred | solved",
|
|
110
|
+
]),
|
|
111
|
+
shown(same),
|
|
112
|
+
);
|
|
113
|
+
check(
|
|
114
|
+
"raters who labelled different runs are refused",
|
|
115
|
+
ran(different, true, ["label different cells"]),
|
|
116
|
+
shown(different),
|
|
117
|
+
);
|
|
118
|
+
console.log(
|
|
119
|
+
`✓ label-agent-runs [effects]: skill activated; agreement reported, mismatched raters refused.\n CLI: ${cli.note}`,
|
|
120
|
+
);
|
|
121
|
+
} finally {
|
|
122
|
+
r.cleanup();
|
|
123
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# Formats — codebook, label file, notes, rater instruction
|
|
2
|
+
|
|
3
|
+
`labels.js` is `scripts/labels.js` in this skill.
|
|
4
|
+
|
|
5
|
+
## Codebook — `codebook.json`, committed before the first label
|
|
6
|
+
|
|
7
|
+
```json
|
|
8
|
+
{
|
|
9
|
+
"labels": [
|
|
10
|
+
{
|
|
11
|
+
"id": "solved",
|
|
12
|
+
"definition": "The task's own check passes on the committed state, and nothing outside the task changed.",
|
|
13
|
+
"borderline": "The check passes, but the run also edited a fixture the task did not name: not solved until the edit is explained."
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"id": "deferred",
|
|
17
|
+
"definition": "The run stops and hands the decision back to the user instead of acting.",
|
|
18
|
+
"borderline": "\"I'll leave that choice to you.\" — a statement, no question mark: deferred."
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"id": "failed",
|
|
22
|
+
"definition": "The run ends without the task done and without handing the decision back.",
|
|
23
|
+
"borderline": "Times out mid-edit: failed, unless the attempt is a harness error in the ledger."
|
|
24
|
+
}
|
|
25
|
+
]
|
|
26
|
+
}
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
`validate` requires every label to have `id`, `definition` and `borderline`.
|
|
30
|
+
|
|
31
|
+
## Label file — TSV
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
agent model cell label
|
|
35
|
+
cli-a model-a unit-017/control/1 solved
|
|
36
|
+
cli-a model-a unit-017/treatment/1 deferred
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
- Exactly this header, tab-separated, four fields per row.
|
|
40
|
+
- `cell` is `<unit>/<arm>/<seed>`; `agent` and `model` match the `agent` and `model` fields of the
|
|
41
|
+
run-agent-grid ledger, so `validate --ledger` can check that each row is a completed run and each
|
|
42
|
+
completed run has a row.
|
|
43
|
+
- One file per source: `auto.tsv` (the pre-sort), `hand.tsv`, `rater-b.tsv`, and the final
|
|
44
|
+
`labels.tsv` the analysis reads.
|
|
45
|
+
|
|
46
|
+
## Notes — `notes.md`
|
|
47
|
+
|
|
48
|
+
```markdown
|
|
49
|
+
## 2026-01-05
|
|
50
|
+
|
|
51
|
+
unit-017/control/1: solved → deferred. Re-read the final message:
|
|
52
|
+
|
|
53
|
+
> I've prepared both options; I'll leave the final call to you.
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
A cell relabelled, added or removed after the analysis is accepted by `validate --analysed-at` only
|
|
57
|
+
when the cell appears in a section under a `## YYYY-MM-DD` heading that also holds a `>` quote line.
|
|
58
|
+
|
|
59
|
+
## Rater instruction — one committed file, used byte for byte
|
|
60
|
+
|
|
61
|
+
The same file goes to every rater; record `sha256sum rater-instruction.md` in the notes. It holds:
|
|
62
|
+
the codebook (pasted, not summarised), what the rater receives for each run (the final message and
|
|
63
|
+
the ordered action list), the output format (one label per run, no other text), and nothing about
|
|
64
|
+
the arms, the hypothesis or another rater's labels.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The subcommands of `labels`, by name. One table, read twice: the dispatcher in `labels.ts`
|
|
3
|
+
* compares against these, and `SKILL.md.spec.ts` interpolates them into every command it
|
|
4
|
+
* instructs (a name that is not here is a type error there). So the skill cannot instruct a
|
|
5
|
+
* subcommand the script does not have.
|
|
6
|
+
*
|
|
7
|
+
* A module of its own, with no side effects: the spec imports it at compile time, and importing
|
|
8
|
+
* `labels.ts` itself would run the command line.
|
|
9
|
+
*/
|
|
10
|
+
export const COMMANDS = {
|
|
11
|
+
validate: "validate",
|
|
12
|
+
agree: "agree",
|
|
13
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The subcommands of `labels`, by name. One table, read twice: the dispatcher in `labels.ts`
|
|
3
|
+
* compares against these, and `SKILL.md.spec.ts` interpolates them into every command it
|
|
4
|
+
* instructs (a name that is not here is a type error there). So the skill cannot instruct a
|
|
5
|
+
* subcommand the script does not have.
|
|
6
|
+
*
|
|
7
|
+
* A module of its own, with no side effects: the spec imports it at compile time, and importing
|
|
8
|
+
* `labels.ts` itself would run the command line.
|
|
9
|
+
*/
|
|
10
|
+
export const COMMANDS = {
|
|
11
|
+
validate: "validate",
|
|
12
|
+
agree: "agree",
|
|
13
|
+
} as const;
|