paperlint 9.0.1 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -128
- package/dist/build.d.ts +1 -1
- package/dist/build.js +1 -1
- package/dist/cli.d.ts +5 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +52 -6
- package/dist/cli.js.map +1 -1
- package/dist/domain/layout-tree.d.ts +19 -0
- package/dist/domain/layout-tree.d.ts.map +1 -0
- package/dist/domain/layout-tree.js +32 -0
- package/dist/domain/layout-tree.js.map +1 -0
- package/dist/domain/lookup-cache.d.ts +2 -2
- package/dist/domain/lookup-cache.js +2 -2
- package/dist/domain/paper-layout.d.ts +81 -0
- package/dist/domain/paper-layout.d.ts.map +1 -0
- package/dist/domain/paper-layout.js +259 -0
- package/dist/domain/paper-layout.js.map +1 -0
- package/dist/domain/repro-layout.d.ts +80 -0
- package/dist/domain/repro-layout.d.ts.map +1 -0
- package/dist/domain/repro-layout.js +247 -0
- package/dist/domain/repro-layout.js.map +1 -0
- package/dist/domain/scaffold-layout.d.ts +28 -0
- package/dist/domain/scaffold-layout.d.ts.map +1 -0
- package/dist/domain/scaffold-layout.js +81 -0
- package/dist/domain/scaffold-layout.js.map +1 -0
- package/dist/eslint-rules/prereg-coverage.d.ts +98 -0
- package/dist/eslint-rules/prereg-coverage.d.ts.map +1 -0
- package/dist/eslint-rules/prereg-coverage.js +274 -0
- package/dist/eslint-rules/prereg-coverage.js.map +1 -0
- package/dist/folder-layout-hook.d.ts +21 -0
- package/dist/folder-layout-hook.d.ts.map +1 -0
- package/dist/folder-layout-hook.js +117 -0
- package/dist/folder-layout-hook.js.map +1 -0
- package/dist/folder-layout-rule.d.ts +22 -0
- package/dist/folder-layout-rule.d.ts.map +1 -0
- package/dist/folder-layout-rule.js +107 -0
- package/dist/folder-layout-rule.js.map +1 -0
- package/dist/init.js +1 -1
- package/dist/init.js.map +1 -1
- package/dist/new-paper.d.ts +4 -1
- package/dist/new-paper.d.ts.map +1 -1
- package/dist/new-paper.js +40 -6
- package/dist/new-paper.js.map +1 -1
- package/dist/references.d.ts +1 -1
- package/dist/references.d.ts.map +1 -1
- package/dist/references.js +2 -2
- package/dist/references.js.map +1 -1
- package/dist/repro-layout-rule.d.ts +16 -0
- package/dist/repro-layout-rule.d.ts.map +1 -0
- package/dist/repro-layout-rule.js +81 -0
- package/dist/repro-layout-rule.js.map +1 -0
- package/docs/ci.md +60 -0
- package/docs/configuration.md +2 -0
- package/docs/e2e.md +2 -2
- package/docs/install.md +20 -16
- package/docs/layout.md +154 -0
- package/docs/optional-rules.md +5 -4
- package/docs/references.md +5 -2
- package/docs/rules/paper/allowed-files-only.md +192 -0
- package/docs/rules/repro/allowed-files-only.md +128 -0
- package/docs/rules/study/declare-sample.md +146 -0
- package/docs/rules.md +53 -50
- package/docs/skills.md +6 -3
- package/docs/venues.md +94 -0
- package/eslint-rules/prereg-coverage.test.ts +448 -0
- package/eslint-rules/prereg-coverage.ts +419 -0
- package/lib/skill-trigger-cases.harness.mjs +51 -5
- package/lib/skill-trigger-cases.mjs +53 -15
- package/package.json +1 -1
- package/plugin/hooks/hooks.json +9 -0
- package/skills/analyze-sibling-paper/SKILL.md +2 -2
- package/skills/analyze-sibling-paper/SKILL.md.spec.ts +1 -1
- package/skills/build-benchmark/SKILL.md +14 -1
- package/skills/build-benchmark/SKILL.md.spec.ts +13 -0
- package/skills/build-label-page/SKILL.md +118 -0
- package/skills/build-label-page/SKILL.md.spec.ts +129 -0
- package/skills/build-label-page/build-label-page.eval.mjs +15 -0
- package/skills/build-label-page/build-label-page.test.ts +959 -0
- package/skills/build-label-page/references/checklist.md +148 -0
- package/skills/build-label-page/references/example-generator.py +803 -0
- package/skills/find-venue/SKILL.md +5 -3
- package/skills/find-venue/SKILL.md.spec.ts +4 -2
- package/skills/grade-paper-writing/SKILL.md +7 -7
- package/skills/grade-paper-writing/SKILL.md.spec.ts +6 -6
- package/skills/map-prior-work/SKILL.md +2 -2
- package/skills/map-prior-work/SKILL.md.spec.ts +1 -1
- package/skills/paper-pipeline/SKILL.md +8 -6
- package/skills/paper-pipeline/SKILL.md.spec.ts +7 -5
- package/skills/paper-pipeline/grade-paper-writing-ablation.eval.mjs +19 -9
- package/skills/paper-pipeline/references/acceptance-gate.md +19 -19
- package/skills/paper-pipeline/references/artifact-checklist.md +5 -0
- package/skills/paper-pipeline/references/pipeline-status-template.md +30 -1
- package/skills/paper-pipeline/repro/2026-08-07-fork-check.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-fork-check2.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-language-eval-pilot.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-language-eval-raw.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-08-framing-vs-vocabulary-oracle.log +48 -48
- package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-A6-oracle.log +8 -8
- package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-oracle.log +48 -48
- package/skills/paper-pipeline/repro/2026-08-08-parent-replication-gpw.log +2 -2
- package/skills/paper-pipeline/repro/2026-08-08-parent-replication.mjs +3 -3
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-after.log +81 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-before.log +81 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-setupdiff.log +59 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.jsonl +1 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.log +22 -0
- package/skills/paper-pipeline/repro/README.md +4 -0
- package/skills/paper-pipeline/scripts/extract-ref-facts.harness.mjs +2 -2
- package/skills/paper-pipeline/scripts/extract-ref-facts.mjs +2 -2
- package/skills/paper-pipeline/scripts/extract-ref-facts.test.mjs +4 -2
- package/skills/paper-pipeline/scripts/generated-code.harness.mjs +7 -2
- package/skills/paper-pipeline/scripts/generated-code.mjs +8 -4
- package/skills/paper-pipeline/scripts/generated-code.test.ts +15 -1
- package/skills/paper-status/SKILL.md +2 -2
- package/skills/paper-status/SKILL.md.spec.ts +1 -1
- package/skills/study-accepted-papers/SKILL.md +163 -161
- package/skills/study-accepted-papers/SKILL.md.spec.ts +167 -165
- package/skills/study-accepted-papers/references/base-rates.md +69 -0
- package/skills/study-accepted-papers/references/mechanisms.md +181 -0
- package/skills/verify-citations/SKILL.md +2 -2
- package/skills/verify-citations/SKILL.md.spec.ts +1 -1
- package/src/build.ts +1 -1
- package/src/cli.harness.mjs +42 -0
- package/src/cli.test.ts +90 -6
- package/src/cli.ts +78 -5
- package/src/domain/layout-tree.test.ts +32 -0
- package/src/domain/layout-tree.ts +47 -0
- package/src/domain/lookup-cache.ts +2 -2
- package/src/domain/paper-layout.test.ts +264 -0
- package/src/domain/paper-layout.ts +336 -0
- package/src/domain/repro-layout.test.ts +222 -0
- package/src/domain/repro-layout.ts +311 -0
- package/src/domain/scaffold-layout.ts +104 -0
- package/src/folder-layout-hook.test.ts +214 -0
- package/src/folder-layout-hook.ts +141 -0
- package/src/folder-layout-rule.test.ts +235 -0
- package/src/folder-layout-rule.ts +151 -0
- package/src/hooks-settings.harness.ts +9 -8
- package/src/init.ts +1 -1
- package/src/layout-docs.test.ts +52 -0
- package/src/new-paper.harness.mjs +23 -1
- package/src/new-paper.test.ts +299 -6
- package/src/new-paper.ts +51 -5
- package/src/paper-files.test.ts +9 -6
- package/src/references.test.ts +5 -5
- package/src/references.ts +2 -2
- package/src/repro-layout-rule.test.ts +114 -0
- package/src/repro-layout-rule.ts +121 -0
- package/src/rule-cases.test.ts +98 -0
- package/templates/paper/PIPELINE-STATUS.md +41 -7
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `study/declare-sample` — a study's pre-registration says which population the paper's claim is
|
|
3
|
+
* about, what the screens between that population and the sample removed, what share of it the
|
|
4
|
+
* sample can represent, and that the author looked at the drawn units before the run.
|
|
5
|
+
*
|
|
6
|
+
* ── THE FAILURE IT PREVENTS ──────────────────────────────────────────────────────
|
|
7
|
+
* A measurement study of how language models read coding rules drew its rules from three pools,
|
|
8
|
+
* kept only the rules "judgeable from one code excerpt of at most 40 lines", restricted one pool to a
|
|
9
|
+
* classifier's "enforceable" lane, and let a model pick short excerpts. Each screen was reasonable
|
|
10
|
+
* alone; together they left a sample of crisp rules. The labeller found 93 % of the items designed as
|
|
11
|
+
* borderline clear-cut, and the rules looked nothing like the ones met in practice. A spot-check of
|
|
12
|
+
* ten items flagged it (seven of ten clear-cut) and the run went ahead: nothing in the design made
|
|
13
|
+
* anyone state what share of the population the sample covered, or which screens removed which kinds
|
|
14
|
+
* of rules. The four fields below are those statements, and this rule makes them part of the record.
|
|
15
|
+
*
|
|
16
|
+
* ── WHERE IT LOOKS ───────────────────────────────────────────────────────────────
|
|
17
|
+
* A study lives in a folder at the top of the paper's `repro/` (`repro/allowed-files-only` names the
|
|
18
|
+
* layout); `tools/` and `data/` beside them are shared, not studies. Every study folder that holds a
|
|
19
|
+
* `prereg.json` is judged; a study without one owes nothing here. The findings are reported on the paper's `PIPELINE-STATUS.md`, first line, as the other rules
|
|
20
|
+
* about a paper's folders are: `prereg.json` is not a file paperlint lints, and every paper has a
|
|
21
|
+
* status file.
|
|
22
|
+
*
|
|
23
|
+
* ── THE FIELDS ───────────────────────────────────────────────────────────────────
|
|
24
|
+
* population { about, source, size } what the claim is about, where it comes from, how many
|
|
25
|
+
* strata { by, values } the one stratification the screens are counted by
|
|
26
|
+
* screens [{ name, removed: {value: n} }] each screen, and what it removed per stratum value
|
|
27
|
+
* coverage { share, reason?, unreachable } 0–1, or null with a reason; strata it cannot reach
|
|
28
|
+
* author_check { who, when, units_seen, units_drawn, verdict, decision? }
|
|
29
|
+
*
|
|
30
|
+
* A pre-registration holds much more (hypotheses, arms, models, budget); every other key is the
|
|
31
|
+
* author's and is not read. The shape is `PREREG` below — a zod schema whose every field carries the
|
|
32
|
+
* text a finding gives for it — and the relations between fields that a schema of one field cannot
|
|
33
|
+
* see (`removed` keyed by the declared strata, a null share with a reason, a doubtful verdict with a
|
|
34
|
+
* decision) are `crossProblems`.
|
|
35
|
+
*/
|
|
36
|
+
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
|
|
37
|
+
import { basename, dirname, join } from "node:path";
|
|
38
|
+
import { z } from "zod";
|
|
39
|
+
import type { RuleContext } from "./rule-context.ts";
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The folder of a paper that holds its studies, the folders in it that are shared rather than a
|
|
43
|
+
* study's, and the file a study pre-registers in. The first two are `repro/allowed-files-only`'s
|
|
44
|
+
* (`src/domain/repro-layout.ts`, `REPRO_DIR` and `REPRO_DIRS`), copied because a module here is
|
|
45
|
+
* compiled before `src/`; the test beside this file holds the copies equal.
|
|
46
|
+
*/
|
|
47
|
+
export const REPRO_DIR = "repro";
|
|
48
|
+
export const SHARED_DIRS: readonly string[] = ["tools", "data"];
|
|
49
|
+
export const PREREG_FILE = "prereg.json";
|
|
50
|
+
const STATUS_FILE = "PIPELINE-STATUS.md";
|
|
51
|
+
const PAGE =
|
|
52
|
+
"https://github.com/zernie/paperlint/blob/main/docs/rules/study/declare-sample.md";
|
|
53
|
+
|
|
54
|
+
/** The one sentence every finding ends with: why the fields exist. */
|
|
55
|
+
export const WHY =
|
|
56
|
+
"Why: a study once screened its sample down to rules that are easy to judge — its labeller found 93 % of the items designed as borderline clear-cut — and nothing in its design made anyone state what share of the real population the sample covered or which screens removed which kinds of units.";
|
|
57
|
+
|
|
58
|
+
/** A zod error option: `what` is the text a finding gives for the field, missing or malformed. */
|
|
59
|
+
const field = (
|
|
60
|
+
what: string,
|
|
61
|
+
): { readonly error: (iss: { readonly input?: unknown }) => string } => ({
|
|
62
|
+
error: (iss) =>
|
|
63
|
+
iss.input === undefined
|
|
64
|
+
? `is missing — add ${what}`
|
|
65
|
+
: `is not valid — it must be ${what}`,
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
/** Text that says something: a string with more than white space in it. */
|
|
69
|
+
const text = (what: string) =>
|
|
70
|
+
z
|
|
71
|
+
.string(field(what))
|
|
72
|
+
.trim()
|
|
73
|
+
.min(1, { error: `is empty — write ${what}` });
|
|
74
|
+
|
|
75
|
+
/** A whole number above 0. */
|
|
76
|
+
const positive = (what: string) =>
|
|
77
|
+
z.number(field(what)).int(field(what)).positive(field(what));
|
|
78
|
+
|
|
79
|
+
const POPULATION = z.object(
|
|
80
|
+
{
|
|
81
|
+
about: text(
|
|
82
|
+
"what the paper's claim is about, in the paper's own words (e.g. the rules written in agent instruction files of public repositories)",
|
|
83
|
+
),
|
|
84
|
+
source: text(
|
|
85
|
+
"where the population's units come from — the census, crawl or registry, with its date",
|
|
86
|
+
),
|
|
87
|
+
size: positive(
|
|
88
|
+
"the number of units in the population, a whole number above 0 (an estimate is fine; say how it was made in `source`)",
|
|
89
|
+
),
|
|
90
|
+
},
|
|
91
|
+
field(
|
|
92
|
+
"`population`: what the paper's claim is about (`about`), where its units come from (`source`) and how many there are (`size`)",
|
|
93
|
+
),
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
const STRATA = z.object(
|
|
97
|
+
{
|
|
98
|
+
by: text("the property the units are stratified by, e.g. rule type"),
|
|
99
|
+
values: z
|
|
100
|
+
.array(
|
|
101
|
+
text("a value of the stratum"),
|
|
102
|
+
field("the stratum's values, a list of names"),
|
|
103
|
+
)
|
|
104
|
+
.min(1, {
|
|
105
|
+
error: "is empty — list the stratum's values, at least one",
|
|
106
|
+
}),
|
|
107
|
+
},
|
|
108
|
+
field(
|
|
109
|
+
"`strata`: the property the screens are counted by (`by`, e.g. rule type) and its values (`values`)",
|
|
110
|
+
),
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
const REMOVED_WHAT =
|
|
114
|
+
"the number of candidates the screen removed in each declared stratum value, an object {<value>: <count>} with 0 where it removed none";
|
|
115
|
+
|
|
116
|
+
const SCREEN = z.object(
|
|
117
|
+
{
|
|
118
|
+
name: text(
|
|
119
|
+
"what the screen keeps, as it was applied (e.g. judgeable from one code excerpt of at most 40 lines)",
|
|
120
|
+
),
|
|
121
|
+
removed: z.record(
|
|
122
|
+
z.string(),
|
|
123
|
+
z
|
|
124
|
+
.number(field("a count, a whole number from 0"))
|
|
125
|
+
.int(field("a count, a whole number from 0"))
|
|
126
|
+
.nonnegative(field("a count, a whole number from 0")),
|
|
127
|
+
field(REMOVED_WHAT),
|
|
128
|
+
),
|
|
129
|
+
},
|
|
130
|
+
field("a screen: `name` and `removed`"),
|
|
131
|
+
);
|
|
132
|
+
|
|
133
|
+
const SCREENS = z.array(
|
|
134
|
+
SCREEN,
|
|
135
|
+
field(
|
|
136
|
+
"`screens`: one entry per screen applied between the population and the sample, each with `name` and `removed` — [] when none was",
|
|
137
|
+
),
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
const SHARE_WHAT =
|
|
141
|
+
"the share of the population the sample can represent, a number from 0 to 1, or null with a `reason`";
|
|
142
|
+
|
|
143
|
+
const COVERAGE = z.object(
|
|
144
|
+
{
|
|
145
|
+
share: z
|
|
146
|
+
.number(field(SHARE_WHAT))
|
|
147
|
+
.min(0, field(SHARE_WHAT))
|
|
148
|
+
.max(1, field(SHARE_WHAT))
|
|
149
|
+
.nullable(),
|
|
150
|
+
reason: text("why the share cannot be stated").optional(),
|
|
151
|
+
unreachable: z.array(
|
|
152
|
+
text("a stratum or kind of unit"),
|
|
153
|
+
field(
|
|
154
|
+
"the strata (or kinds of units) the sample cannot reach, a list of names — [] only when it reaches them all",
|
|
155
|
+
),
|
|
156
|
+
),
|
|
157
|
+
},
|
|
158
|
+
field(
|
|
159
|
+
"`coverage`: the share of the population the sample can represent (`share`, 0 to 1, or null with a `reason`) and the strata it cannot reach (`unreachable`)",
|
|
160
|
+
),
|
|
161
|
+
);
|
|
162
|
+
|
|
163
|
+
/** The answers to "is this what you meet in practice?"; any but the first needs a decision. */
|
|
164
|
+
export const VERDICTS = ["representative", "partly", "not-representative"];
|
|
165
|
+
|
|
166
|
+
const DECISION = z.object(
|
|
167
|
+
{
|
|
168
|
+
proceed: z.boolean(
|
|
169
|
+
field("whether the study runs on this sample, true or false"),
|
|
170
|
+
),
|
|
171
|
+
by: text("who decided"),
|
|
172
|
+
why: text("why, in a sentence"),
|
|
173
|
+
},
|
|
174
|
+
field(
|
|
175
|
+
"the decision taken after the verdict: `proceed` (true or false), `by` whom, and `why`",
|
|
176
|
+
),
|
|
177
|
+
);
|
|
178
|
+
|
|
179
|
+
const AUTHOR_CHECK = z.object(
|
|
180
|
+
{
|
|
181
|
+
who: text("who looked at the drawn units (a name or a role)"),
|
|
182
|
+
when: z.iso.date(field("the date of the check, YYYY-MM-DD")),
|
|
183
|
+
units_seen: positive(
|
|
184
|
+
"how many of the drawn units — the rules or cases drawn, not the items made from them — the author looked at, a whole number above 0",
|
|
185
|
+
),
|
|
186
|
+
units_drawn: positive("how many units the sample drew"),
|
|
187
|
+
verdict: z.enum(
|
|
188
|
+
VERDICTS,
|
|
189
|
+
field(
|
|
190
|
+
`the author's answer to "is this what you meet in practice?" — one of ${VERDICTS.join(", ")}`,
|
|
191
|
+
),
|
|
192
|
+
),
|
|
193
|
+
decision: DECISION.optional(),
|
|
194
|
+
},
|
|
195
|
+
field(
|
|
196
|
+
"`author_check`: who looked at the drawn units before the run (`who`), when (`when`), how many of how many (`units_seen`, `units_drawn`), and what they said (`verdict`)",
|
|
197
|
+
),
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
/** The declaration a `prereg.json` must carry. Every other key of the file is the author's. */
|
|
201
|
+
export const PREREG = z.object(
|
|
202
|
+
{
|
|
203
|
+
population: POPULATION,
|
|
204
|
+
strata: STRATA,
|
|
205
|
+
screens: SCREENS,
|
|
206
|
+
coverage: COVERAGE,
|
|
207
|
+
author_check: AUTHOR_CHECK,
|
|
208
|
+
},
|
|
209
|
+
field(
|
|
210
|
+
"a JSON object holding `population`, `strata`, `screens`, `coverage` and `author_check`",
|
|
211
|
+
),
|
|
212
|
+
);
|
|
213
|
+
|
|
214
|
+
export type Prereg = z.infer<typeof PREREG>;
|
|
215
|
+
|
|
216
|
+
/** A zod path as it is written in the file: `screens[0].removed.vague`. */
|
|
217
|
+
export const pathText = (path: readonly PropertyKey[]): string =>
|
|
218
|
+
path
|
|
219
|
+
.map((k) => (typeof k === "number" ? `[${String(k)}]` : `.${String(k)}`))
|
|
220
|
+
.join("")
|
|
221
|
+
.replace(/^\./u, "");
|
|
222
|
+
|
|
223
|
+
/** One problem: the field it is about, and what is wrong with it. */
|
|
224
|
+
export interface Problem {
|
|
225
|
+
readonly at: string;
|
|
226
|
+
readonly text: string;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** The values that occur more than once in `values`, each once. */
|
|
230
|
+
const repeated = (values: readonly string[]): readonly string[] => [
|
|
231
|
+
...new Set(values.filter((v, i) => values.indexOf(v) !== i)),
|
|
232
|
+
];
|
|
233
|
+
|
|
234
|
+
/** What one screen's `removed` lacks and holds beyond the declared strata. */
|
|
235
|
+
const removedProblems = (
|
|
236
|
+
removed: Readonly<Record<string, number>>,
|
|
237
|
+
i: number,
|
|
238
|
+
values: readonly string[],
|
|
239
|
+
): readonly Problem[] => [
|
|
240
|
+
...values
|
|
241
|
+
.filter((v) => !Object.hasOwn(removed, v))
|
|
242
|
+
.map((v) => ({
|
|
243
|
+
at: `screens[${String(i)}].removed.${v}`,
|
|
244
|
+
text: `is missing — every screen states how many candidates it removed in each declared stratum value (${values.join(", ")}), 0 where it removed none`,
|
|
245
|
+
})),
|
|
246
|
+
...Object.keys(removed)
|
|
247
|
+
.filter((k) => !values.includes(k))
|
|
248
|
+
.map((k) => ({
|
|
249
|
+
at: `screens[${String(i)}].removed.${k}`,
|
|
250
|
+
text: `is not a declared stratum value — \`strata.values\` declares ${values.join(", ")}; add it there or count it under one of them`,
|
|
251
|
+
})),
|
|
252
|
+
];
|
|
253
|
+
|
|
254
|
+
/** A problem when `when` holds, else none. */
|
|
255
|
+
const problemIf = (
|
|
256
|
+
when: boolean,
|
|
257
|
+
at: string,
|
|
258
|
+
text: string,
|
|
259
|
+
): readonly Problem[] => (when ? [{ at, text }] : []);
|
|
260
|
+
|
|
261
|
+
/** The author's check against itself: no more units seen than drawn, a doubtful verdict decided. */
|
|
262
|
+
const checkProblems = (
|
|
263
|
+
check: Readonly<Prereg["author_check"]>,
|
|
264
|
+
): readonly Problem[] => [
|
|
265
|
+
...problemIf(
|
|
266
|
+
check.units_seen > check.units_drawn,
|
|
267
|
+
"author_check.units_seen",
|
|
268
|
+
`is ${String(check.units_seen)}, more than the ${String(check.units_drawn)} units drawn (\`units_drawn\`)`,
|
|
269
|
+
),
|
|
270
|
+
...problemIf(
|
|
271
|
+
check.verdict !== "representative" && check.decision === undefined,
|
|
272
|
+
"author_check.decision",
|
|
273
|
+
`is missing — the verdict is «${check.verdict}», and a doubtful verdict with no recorded decision is how a sample the author doubted gets run anyway: add \`proceed\` (true or false), \`by\` whom and \`why\``,
|
|
274
|
+
),
|
|
275
|
+
];
|
|
276
|
+
|
|
277
|
+
/** What a schema of one field cannot see: how the fields of a parsed declaration relate. */
|
|
278
|
+
export function crossProblems(p: Readonly<Prereg>): readonly Problem[] {
|
|
279
|
+
const values = p.strata.values;
|
|
280
|
+
return [
|
|
281
|
+
...repeated(values).map((v) => ({
|
|
282
|
+
at: "strata.values",
|
|
283
|
+
text: `names «${v}» more than once — each stratum value once`,
|
|
284
|
+
})),
|
|
285
|
+
...p.screens.flatMap((s, i) => removedProblems(s.removed, i, values)),
|
|
286
|
+
...problemIf(
|
|
287
|
+
p.coverage.share === null && p.coverage.reason === undefined,
|
|
288
|
+
"coverage.reason",
|
|
289
|
+
"is missing — `share` is null, so say why the share cannot be stated",
|
|
290
|
+
),
|
|
291
|
+
...problemIf(
|
|
292
|
+
p.coverage.share === 1 && p.coverage.unreachable.length > 0,
|
|
293
|
+
"coverage.unreachable",
|
|
294
|
+
`contradicts \`share\` 1 — a sample that cannot reach ${p.coverage.unreachable.map((u) => `«${u}»`).join(", ")} does not represent the whole population: lower \`share\`, or empty the list`,
|
|
295
|
+
),
|
|
296
|
+
...checkProblems(p.author_check),
|
|
297
|
+
];
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Everything wrong with the parsed JSON of one `prereg.json`, in the file's field order. */
|
|
301
|
+
export function judgePrereg(data: unknown): readonly Problem[] {
|
|
302
|
+
const parsed = PREREG.safeParse(data);
|
|
303
|
+
return parsed.success
|
|
304
|
+
? crossProblems(parsed.data)
|
|
305
|
+
: parsed.error.issues.map((i) => ({
|
|
306
|
+
at: pathText(i.path),
|
|
307
|
+
text: i.message,
|
|
308
|
+
}));
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** What `JSON.parse` says about the text it refused. */
|
|
312
|
+
const Thrown = z
|
|
313
|
+
.object({ message: z.string() })
|
|
314
|
+
.catch({ message: "unparseable" });
|
|
315
|
+
|
|
316
|
+
/** The problems of one `prereg.json` whose text is `source`: a parse failure, or the judgement. */
|
|
317
|
+
export function preregProblems(source: string): readonly Problem[] {
|
|
318
|
+
const parsed = ((): { readonly data: unknown } | { readonly why: string } => {
|
|
319
|
+
try {
|
|
320
|
+
const data: unknown = JSON.parse(source);
|
|
321
|
+
return { data };
|
|
322
|
+
} catch (e) {
|
|
323
|
+
return { why: Thrown.parse(e).message };
|
|
324
|
+
}
|
|
325
|
+
})();
|
|
326
|
+
return "why" in parsed
|
|
327
|
+
? [
|
|
328
|
+
{
|
|
329
|
+
at: "",
|
|
330
|
+
text: `does not parse as JSON (${parsed.why}) — there is no declaration to read`,
|
|
331
|
+
},
|
|
332
|
+
]
|
|
333
|
+
: judgePrereg(parsed.data);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** The one text a problem in the file `file` (relative to the paper) is reported with. */
|
|
337
|
+
export const problemMessage = (file: string, p: Problem): string =>
|
|
338
|
+
[
|
|
339
|
+
p.at === ""
|
|
340
|
+
? `\`${file}\` ${p.text}.`
|
|
341
|
+
: `\`${file}\`: \`${p.at}\` ${p.text}.`,
|
|
342
|
+
WHY,
|
|
343
|
+
`Rule study/declare-sample: ${PAGE}`,
|
|
344
|
+
].join(" ");
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* The `prereg.json` of every study folder at the top of `<paperDir>/repro/`, relative to the paper:
|
|
348
|
+
* every folder there but the shared ones, whatever its name — a study folder misnamed is
|
|
349
|
+
* `repro/allowed-files-only`'s finding, and its pre-registration is still judged here.
|
|
350
|
+
*/
|
|
351
|
+
export function preregFiles(paperDir: string): readonly string[] {
|
|
352
|
+
const repro = join(paperDir, REPRO_DIR);
|
|
353
|
+
return existsSync(repro) && statSync(repro).isDirectory()
|
|
354
|
+
? readdirSync(repro, { withFileTypes: true })
|
|
355
|
+
// stat, not the entry's own type: a study folder linked into repro/ is a folder to
|
|
356
|
+
// repro/allowed-files-only too, and must not escape the check by being a link
|
|
357
|
+
.filter(
|
|
358
|
+
(e) =>
|
|
359
|
+
!SHARED_DIRS.includes(e.name) &&
|
|
360
|
+
(statSync(join(repro, e.name), {
|
|
361
|
+
throwIfNoEntry: false,
|
|
362
|
+
})?.isDirectory() ??
|
|
363
|
+
false),
|
|
364
|
+
)
|
|
365
|
+
.map((e) => `${REPRO_DIR}/${e.name}/${PREREG_FILE}`)
|
|
366
|
+
.filter((rel) => existsSync(join(paperDir, rel)))
|
|
367
|
+
.sort()
|
|
368
|
+
: [];
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
/** A `prereg.json` that is not a regular file (a folder, a socket): reading it would abort the run. */
|
|
372
|
+
const NOT_A_FILE: readonly Problem[] = [
|
|
373
|
+
{
|
|
374
|
+
at: "",
|
|
375
|
+
text: "is not a file — the declaration is one JSON file; move what the folder holds elsewhere",
|
|
376
|
+
},
|
|
377
|
+
];
|
|
378
|
+
|
|
379
|
+
/** Every finding for the paper folder `paperDir`, one text each. */
|
|
380
|
+
export const paperFindings = (paperDir: string): readonly string[] =>
|
|
381
|
+
preregFiles(paperDir).flatMap((rel) => {
|
|
382
|
+
const path = join(paperDir, rel);
|
|
383
|
+
const problems = statSync(path).isFile()
|
|
384
|
+
? preregProblems(readFileSync(path, "utf8"))
|
|
385
|
+
: NOT_A_FILE;
|
|
386
|
+
return problems.map((p) => problemMessage(rel, p));
|
|
387
|
+
});
|
|
388
|
+
|
|
389
|
+
export default {
|
|
390
|
+
rules: {
|
|
391
|
+
"declare-sample": {
|
|
392
|
+
meta: {
|
|
393
|
+
type: "problem" as const,
|
|
394
|
+
docs: {
|
|
395
|
+
description:
|
|
396
|
+
"every study's prereg.json declares its population, the screens and what each removed per stratum, the sample's coverage, and the author's check of the drawn units",
|
|
397
|
+
url: PAGE,
|
|
398
|
+
},
|
|
399
|
+
schema: [],
|
|
400
|
+
messages: { coverage: "{{text}}" },
|
|
401
|
+
},
|
|
402
|
+
create(context: RuleContext) {
|
|
403
|
+
if (basename(context.filename) !== STATUS_FILE) return {};
|
|
404
|
+
return {
|
|
405
|
+
root() {
|
|
406
|
+
const at = context.sourceCode.getLocFromIndex(0);
|
|
407
|
+
paperFindings(dirname(context.filename)).forEach((t) => {
|
|
408
|
+
context.report({
|
|
409
|
+
loc: { start: at, end: at },
|
|
410
|
+
messageId: "coverage",
|
|
411
|
+
data: { text: t },
|
|
412
|
+
});
|
|
413
|
+
});
|
|
414
|
+
},
|
|
415
|
+
};
|
|
416
|
+
},
|
|
417
|
+
},
|
|
418
|
+
},
|
|
419
|
+
};
|
|
@@ -21,7 +21,7 @@ import { mkdtempSync, writeFileSync, mkdirSync, realpathSync } from "node:fs";
|
|
|
21
21
|
import { tmpdir } from "node:os";
|
|
22
22
|
import { dirname, join } from "node:path";
|
|
23
23
|
import { fileURLToPath } from "node:url";
|
|
24
|
-
import { recordCheck } from "vigiles";
|
|
24
|
+
import { assertPromptDiversity, recordCheck } from "vigiles";
|
|
25
25
|
import { CASES, BY_SKILL, irrelevantFor } from "./skill-trigger-cases.mjs";
|
|
26
26
|
|
|
27
27
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
@@ -86,10 +86,9 @@ const LOCAL_ONE = `export const CASES = [{
|
|
|
86
86
|
soft("I. every case is shaped and every collider resolves", () => {
|
|
87
87
|
for (const c of CASES) {
|
|
88
88
|
assert.equal(typeof c.skill, "string", `a case has no skill name`);
|
|
89
|
-
assert.
|
|
90
|
-
c.prompts.length,
|
|
91
|
-
4
|
|
92
|
-
`${c.skill}: ${c.prompts.length} prompts, expected 4`,
|
|
89
|
+
assert.ok(
|
|
90
|
+
c.prompts.length >= 4,
|
|
91
|
+
`${c.skill}: ${c.prompts.length} prompts, expected at least 4`,
|
|
93
92
|
);
|
|
94
93
|
for (const sib of c.collides)
|
|
95
94
|
assert.ok(
|
|
@@ -112,6 +111,53 @@ soft("I. irrelevantFor still yields a full set for a one-collider case", () => {
|
|
|
112
111
|
);
|
|
113
112
|
});
|
|
114
113
|
|
|
114
|
+
// A collider that contributes nothing to the irrelevant set is named in the table and never
|
|
115
|
+
// measured: the case reads as "must not take X's requests" and no request of X's is ever sent.
|
|
116
|
+
// `irrelevantFor` used to cut the set at four, so a case with more than four colliders silently
|
|
117
|
+
// lost the rest — grade-paper-writing, which names seven, lost three (2026-10-13).
|
|
118
|
+
soft(
|
|
119
|
+
"I. every collider of every case contributes to its irrelevant set",
|
|
120
|
+
() => {
|
|
121
|
+
for (const c of CASES) {
|
|
122
|
+
const got = new Set(irrelevantFor(c.skill));
|
|
123
|
+
const silent = c.collides.filter(
|
|
124
|
+
(sib) => !BY_SKILL.get(sib).prompts.some((p) => got.has(p)),
|
|
125
|
+
);
|
|
126
|
+
assert.deepEqual(silent, [], `${c.skill}: colliders never measured`);
|
|
127
|
+
}
|
|
128
|
+
},
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
// The paid tier refuses a set that is too small or near-duplicate (vigiles' diversity pre-flight,
|
|
132
|
+
// run by `measureTriggerRate` on both sides) — but only when someone pays to run it. The same gate,
|
|
133
|
+
// with the same options `lib/skill-eval-kit.mjs` passes, decided here for free on every push.
|
|
134
|
+
soft(
|
|
135
|
+
"I. every case would pass the paid tier's prompt-diversity pre-flight",
|
|
136
|
+
() => {
|
|
137
|
+
for (const c of CASES) {
|
|
138
|
+
assertPromptDiversity(c.prompts, {
|
|
139
|
+
minPrompts: 4,
|
|
140
|
+
label: `${c.skill} prompts`,
|
|
141
|
+
});
|
|
142
|
+
assertPromptDiversity(irrelevantFor(c.skill), {
|
|
143
|
+
minPrompts: 4,
|
|
144
|
+
label: `${c.skill} irrelevantPrompts`,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
);
|
|
149
|
+
|
|
150
|
+
// A prompt listed under two skills says it must fire both, and each case's precision run then
|
|
151
|
+
// counts the other's correct answer as a false positive.
|
|
152
|
+
soft("I. no prompt is claimed by two cases", () => {
|
|
153
|
+
const owners = new Map();
|
|
154
|
+
for (const c of CASES)
|
|
155
|
+
for (const p of c.prompts)
|
|
156
|
+
owners.set(p, [...(owners.get(p) ?? []), c.skill]);
|
|
157
|
+
const shared = [...owners].filter(([, o]) => o.length > 1);
|
|
158
|
+
assert.deepEqual(shared, []);
|
|
159
|
+
});
|
|
160
|
+
|
|
115
161
|
// ── II. The carrier, all four states ─────────────────────────────────────────
|
|
116
162
|
soft(
|
|
117
163
|
"II. undeclared and absent is SILENT — a consumer with no skills of its own",
|
|
@@ -12,11 +12,11 @@
|
|
|
12
12
|
*
|
|
13
13
|
* skill the skill under test
|
|
14
14
|
* why the territory it owns, in one line — the thing a sibling must NOT take
|
|
15
|
-
* prompts should-fire. FOUR
|
|
15
|
+
* prompts should-fire. At least FOUR, deliberately mixed (below)
|
|
16
16
|
* collides siblings whose own should-fire prompts become this case's
|
|
17
|
-
* irrelevantPrompts
|
|
18
|
-
*
|
|
19
|
-
* from both directions.
|
|
17
|
+
* irrelevantPrompts — at least one from EVERY collider. That is how
|
|
18
|
+
* "the WRONG skill must not fire" is asserted, and it is symmetric:
|
|
19
|
+
* each side of a collision is measured from both directions.
|
|
20
20
|
*
|
|
21
21
|
* 🔴 THE PROMPTS ARE MIXED ON PURPOSE, AND THE MIX DECIDES THE NUMBER.
|
|
22
22
|
* `framing-vs-vocabulary.eval.mjs` measured the same skills two ways: prompts that
|
|
@@ -55,13 +55,36 @@ const PACKAGE_CASES = [
|
|
|
55
55
|
{
|
|
56
56
|
skill: "grade-paper-writing",
|
|
57
57
|
why: "prose craft — the sentence is the unit, NOT the section",
|
|
58
|
+
// The first four are HARVESTED and measured; they stay first and unchanged, so the rate on
|
|
59
|
+
// them stays comparable with pipeline-firing.eval.mjs and the siblings that collide with this
|
|
60
|
+
// case keep receiving the same irrelevant prompts. The last four were added 2026-10-13 with
|
|
61
|
+
// the rewritten description and have never been run — rates UNKNOWN. Eight, not four, because
|
|
62
|
+
// this skill's firing was measured at 12%, then 50–54%, on the same prompts: four prompts
|
|
63
|
+
// cannot tell those apart.
|
|
58
64
|
prompts: [
|
|
59
65
|
"оцени как написано — читается как стена жаргона",
|
|
60
66
|
"is the writing any good or does it read like shit",
|
|
61
67
|
"abstract звучит криво хотя по смыслу всё на месте, дай оценку прозе",
|
|
62
68
|
"grade the craft: title, abstract, sentence clarity, hedge stacking",
|
|
69
|
+
"прочитай черновик и скажи честно, нормально ли он читается",
|
|
70
|
+
"the reviewer says our paper reads like a blog post — is he right about the prose",
|
|
71
|
+
"каждое предложение по отдельности понятно, а абзацы всё равно сливаются в сплошную простыню",
|
|
72
|
+
"grade the prose in section 3 and give me the sentences to rewrite",
|
|
73
|
+
],
|
|
74
|
+
// Every sibling the description sends a request to. Each is a different wrong answer to "is
|
|
75
|
+
// this paper any good?": cut it, re-read one edit, attack the science, predict the committee,
|
|
76
|
+
// gate the submission, study the venue. Of the five added 2026-10-13, only cold-read-diff
|
|
77
|
+
// already named this case; the other four do not name it back, because that would change the
|
|
78
|
+
// irrelevant sets their own rates are measured on.
|
|
79
|
+
collides: [
|
|
80
|
+
"tighten-paper",
|
|
81
|
+
"argument-arc",
|
|
82
|
+
"cold-read-diff",
|
|
83
|
+
"paper-adversarial-review",
|
|
84
|
+
"pc-panel-review",
|
|
85
|
+
"harden-paper",
|
|
86
|
+
"study-accepted-papers",
|
|
63
87
|
],
|
|
64
|
-
collides: ["tighten-paper", "argument-arc"],
|
|
65
88
|
},
|
|
66
89
|
{
|
|
67
90
|
skill: "argument-arc",
|
|
@@ -153,6 +176,17 @@ const PACKAGE_CASES = [
|
|
|
153
176
|
],
|
|
154
177
|
collides: ["research-ideate", "draft-paper"],
|
|
155
178
|
},
|
|
179
|
+
{
|
|
180
|
+
skill: "build-label-page",
|
|
181
|
+
why: "a PERSON labels items on a phone page with a shared store, and the answers are exported into the study",
|
|
182
|
+
prompts: [
|
|
183
|
+
"сделай мне страницу, где я разметил бы эти примеры с телефона",
|
|
184
|
+
"build a gold-labelling page for these 40 items so I can answer on my phone",
|
|
185
|
+
"нужна слепая страница для ревью ответов модели, чтобы я не видел, какой из них чей",
|
|
186
|
+
"make me a page to label these, and put the answers back into the study folder",
|
|
187
|
+
],
|
|
188
|
+
collides: ["build-benchmark", "draft-paper"],
|
|
189
|
+
},
|
|
156
190
|
{
|
|
157
191
|
skill: "camera-ready",
|
|
158
192
|
why: "AFTER accept — de-anonymize, archive, disclose",
|
|
@@ -248,7 +282,7 @@ const PACKAGE_CASES = [
|
|
|
248
282
|
"посмотри что принимали на этой площадке и чем те статьи сильны",
|
|
249
283
|
"хочу дотянуть с accept до strong accept — что для этого нужно именно там",
|
|
250
284
|
"read a dozen accepted papers from this venue and diff our draft against them",
|
|
251
|
-
"
|
|
285
|
+
"before we design the study: what does this venue actually reward? look at its award winners",
|
|
252
286
|
],
|
|
253
287
|
collides: ["find-venue", "harden-paper"],
|
|
254
288
|
},
|
|
@@ -436,14 +470,18 @@ export function irrelevantFor(skill, want = 4) {
|
|
|
436
470
|
// verify-citations — seven of twenty-one, exactly the set with `collides.length
|
|
437
471
|
// === 1`. Two-collider cases are unaffected (ceil(4/2) === 2), so numbers
|
|
438
472
|
// measured before the fix stay comparable.
|
|
473
|
+
//
|
|
474
|
+
// `want` is a FLOOR, not a cap (2026-10-13). The set used to be cut at `want`,
|
|
475
|
+
// so a case with more than four colliders silently dropped the rest: the table
|
|
476
|
+
// said "must not take X's requests" and no request of X's was ever sent. Cases
|
|
477
|
+
// with one or two colliders get exactly what they got before (4); a case with
|
|
478
|
+
// three gets 6, with seven gets 7 — one from each.
|
|
439
479
|
const per = Math.ceil(want / c.collides.length);
|
|
440
|
-
return c.collides
|
|
441
|
-
.
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
})
|
|
448
|
-
.slice(0, want);
|
|
480
|
+
return c.collides.flatMap((sib, i) => {
|
|
481
|
+
const s = BY_SKILL.get(sib);
|
|
482
|
+
if (!s) throw new Error(`${skill} collides with unknown skill "${sib}"`);
|
|
483
|
+
// Rotate by the collider's index so two cases sharing a collider do not
|
|
484
|
+
// receive byte-identical irrelevant sets.
|
|
485
|
+
return [...s.prompts.slice(i), ...s.prompts.slice(0, i)].slice(0, per);
|
|
486
|
+
});
|
|
449
487
|
}
|
package/package.json
CHANGED
package/plugin/hooks/hooks.json
CHANGED
|
@@ -9,6 +9,15 @@
|
|
|
9
9
|
"command": "node \"${CLAUDE_PROJECT_DIR}/node_modules/paperlint/bin/paperlint.mjs\" hook paper-edit-guard"
|
|
10
10
|
}
|
|
11
11
|
]
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"matcher": "Edit|Write|MultiEdit",
|
|
15
|
+
"hooks": [
|
|
16
|
+
{
|
|
17
|
+
"type": "command",
|
|
18
|
+
"command": "node \"${CLAUDE_PROJECT_DIR}/node_modules/paperlint/bin/paperlint.mjs\" hook paper-folder-layout"
|
|
19
|
+
}
|
|
20
|
+
]
|
|
12
21
|
}
|
|
13
22
|
],
|
|
14
23
|
"PostToolUse": [
|
|
@@ -5,7 +5,7 @@ context: fork
|
|
|
5
5
|
allowed-tools: [WebSearch, WebFetch, Read, Write, Grep, Glob, Bash, Agent]
|
|
6
6
|
---
|
|
7
7
|
|
|
8
|
-
<!-- vigiles:sha256:
|
|
8
|
+
<!-- vigiles:sha256:834f128ace92288a compiled from skills/analyze-sibling-paper/SKILL.md.spec.ts -->
|
|
9
9
|
|
|
10
10
|
# analyze-sibling-paper — settle "isn't this just X?" before a reviewer does
|
|
11
11
|
|
|
@@ -115,7 +115,7 @@ again at rebuttal. **A close sibling with no file in `<paper-dir>/siblings/` is
|
|
|
115
115
|
## Compose with
|
|
116
116
|
- `verify-citations` — surfaces the nearest-neighbor works; hand each genuine sibling to this skill.
|
|
117
117
|
- `harden-paper` — its nearest-neighbor-scoop axis (axis 3) calls this for each close competitor.
|
|
118
|
-
- `study-accepted-papers` — venue-corpus levers (different axis: the venue
|
|
118
|
+
- `study-accepted-papers` — venue-corpus levers (different axis: what the venue rewards, not a specific rival).
|
|
119
119
|
- `extend-paper` — a scooped angle, or the un-scooped remainder, can become the next paper's pivot.
|
|
120
120
|
|
|
121
121
|
## Provenance
|
|
@@ -130,7 +130,7 @@ again at rebuttal. **A close sibling with no file in \`<paper-dir>/siblings/\` i
|
|
|
130
130
|
## Compose with
|
|
131
131
|
- \`verify-citations\` — surfaces the nearest-neighbor works; hand each genuine sibling to this skill.
|
|
132
132
|
- \`harden-paper\` — its nearest-neighbor-scoop axis (axis 3) calls this for each close competitor.
|
|
133
|
-
- \`study-accepted-papers\` — venue-corpus levers (different axis: the venue
|
|
133
|
+
- \`study-accepted-papers\` — venue-corpus levers (different axis: what the venue rewards, not a specific rival).
|
|
134
134
|
- \`extend-paper\` — a scooped angle, or the un-scooped remainder, can become the next paper's pivot.
|
|
135
135
|
|
|
136
136
|
## Provenance
|