paperlint 10.1.0 → 10.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +2 -0
  2. package/dist/adapters/study-registry/index.d.ts +6 -0
  3. package/dist/adapters/study-registry/index.d.ts.map +1 -0
  4. package/dist/adapters/study-registry/index.js +58 -0
  5. package/dist/adapters/study-registry/index.js.map +1 -0
  6. package/dist/cli.d.ts.map +1 -1
  7. package/dist/cli.js +40 -2
  8. package/dist/cli.js.map +1 -1
  9. package/dist/domain/repro-layout.d.ts +12 -4
  10. package/dist/domain/repro-layout.d.ts.map +1 -1
  11. package/dist/domain/repro-layout.js +49 -18
  12. package/dist/domain/repro-layout.js.map +1 -1
  13. package/dist/domain/scaffold-layout.d.ts +4 -3
  14. package/dist/domain/scaffold-layout.d.ts.map +1 -1
  15. package/dist/domain/scaffold-layout.js +7 -4
  16. package/dist/domain/scaffold-layout.js.map +1 -1
  17. package/dist/domain/study-registry.d.ts +176 -0
  18. package/dist/domain/study-registry.d.ts.map +1 -0
  19. package/dist/domain/study-registry.js +371 -0
  20. package/dist/domain/study-registry.js.map +1 -0
  21. package/dist/ports/study-registry-codec.d.ts +11 -0
  22. package/dist/ports/study-registry-codec.d.ts.map +1 -0
  23. package/dist/ports/study-registry-codec.js +2 -0
  24. package/dist/ports/study-registry-codec.js.map +1 -0
  25. package/dist/repro-layout-rule.d.ts.map +1 -1
  26. package/dist/repro-layout-rule.js +4 -3
  27. package/dist/repro-layout-rule.js.map +1 -1
  28. package/dist/studies.d.ts +27 -0
  29. package/dist/studies.d.ts.map +1 -0
  30. package/dist/studies.js +160 -0
  31. package/dist/studies.js.map +1 -0
  32. package/dist/study-registry-rule.d.ts +17 -0
  33. package/dist/study-registry-rule.d.ts.map +1 -0
  34. package/dist/study-registry-rule.js +115 -0
  35. package/dist/study-registry-rule.js.map +1 -0
  36. package/docs/design/study-registry.md +130 -0
  37. package/docs/layout.md +47 -44
  38. package/docs/rules/paper/allowed-files-only.md +3 -2
  39. package/docs/rules/repro/allowed-files-only.md +40 -30
  40. package/docs/rules/repro/study-registered.md +162 -0
  41. package/docs/rules/study/declare-sample.md +1 -1
  42. package/docs/rules.md +2 -1
  43. package/package.json +1 -1
  44. package/skills/build-benchmark/SKILL.md +20 -1
  45. package/skills/build-benchmark/SKILL.md.spec.ts +19 -0
  46. package/skills/build-label-page/SKILL.md +7 -4
  47. package/skills/build-label-page/SKILL.md.spec.ts +6 -3
  48. package/skills/paper-pipeline/references/artifact-checklist.md +2 -1
  49. package/skills/paper-pipeline/references/pipeline-status-template.md +5 -3
  50. package/skills/render-paper/check-render.sh +9 -0
  51. package/skills/render-paper/render-paper.harness.mjs +102 -0
  52. package/skills/run-agent-grid/SKILL.md +8 -1
  53. package/skills/run-agent-grid/SKILL.md.spec.ts +7 -0
  54. package/src/adapters/study-registry/index.test.ts +144 -0
  55. package/src/adapters/study-registry/index.ts +70 -0
  56. package/src/cli.ts +51 -2
  57. package/src/domain/paper-layout.test.ts +1 -1
  58. package/src/domain/repro-layout.test.ts +18 -9
  59. package/src/domain/repro-layout.ts +58 -18
  60. package/src/domain/scaffold-layout.ts +7 -4
  61. package/src/domain/study-registry.test.ts +657 -0
  62. package/src/domain/study-registry.ts +598 -0
  63. package/src/folder-layout-hook.test.ts +10 -0
  64. package/src/new-paper.test.ts +28 -2
  65. package/src/ports/study-registry-codec.ts +11 -0
  66. package/src/repro-layout-rule.ts +4 -3
  67. package/src/rule-cases.test.ts +41 -0
  68. package/src/studies-cli.test.ts +113 -0
  69. package/src/studies.test.ts +353 -0
  70. package/src/studies.ts +259 -0
  71. package/src/study-registry-rule.test.ts +317 -0
  72. package/src/study-registry-rule.ts +172 -0
@@ -4,7 +4,7 @@ description: Build a phone labelling page published as a claude.ai Artifact with
4
4
  allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Artifact, ArtifactData]
5
5
  ---
6
6
 
7
- <!-- vigiles:sha256:592d7b4f8f27c4ce compiled from skills/build-label-page/SKILL.md.spec.ts -->
7
+ <!-- vigiles:sha256:7a537e71585bd321 compiled from skills/build-label-page/SKILL.md.spec.ts -->
8
8
 
9
9
  # build-label-page — a page the person labels on a phone, and the labels back in the repo
10
10
 
@@ -55,9 +55,12 @@ The full text, each with its reason and its check, is
55
55
  you meet in practice?". The labelling page is not that check: items made from easy units each look
56
56
  fine on their own.
57
57
  2. **Copy the generator into the study and write the items.** The study folder is
58
- `<paper-dir>/repro/<slug>-YYYY-MM-DD/` (`gold-labels-2026-10-12`; `repro/allowed-files-only` wants the
59
- date, a `README.md` in it saying what the study asks, and a line in `repro/README.md`; where this
60
- sits in stage 3, and what checks each place: `docs/layout.md` in the paperlint package).
58
+ `<paper-dir>/repro/<id>-<slug>/`, made by `npx paperlint studies new <paper-dir> "Gold labels"
59
+ --kind data` when the labelling is a study of its own (`P1-S04-gold-labels`): the command writes
60
+ its `README.md` and its row in `repro/studies.json`, and `repro/study-registered` checks both. When
61
+ the labels are in, update the row (`status`, `result`, `result_file`) and run
62
+ `npx paperlint studies index <paper-dir>`. Where this sits in stage 3, and what checks each place:
63
+ `docs/layout.md` in the paperlint package.
61
64
  `items.json` follows the header of the generator. Source ids in `items.json` are the study's own;
62
65
  the page never shows them.
63
66
 
@@ -65,9 +65,12 @@ The full text, each with its reason and its check, is
65
65
  you meet in practice?". The labelling page is not that check: items made from easy units each look
66
66
  fine on their own.
67
67
  2. **Copy the generator into the study and write the items.** The study folder is
68
- \`<paper-dir>/repro/<slug>-YYYY-MM-DD/\` (\`gold-labels-2026-10-12\`; \`repro/allowed-files-only\` wants the
69
- date, a \`README.md\` in it saying what the study asks, and a line in \`repro/README.md\`; where this
70
- sits in stage 3, and what checks each place: \`docs/layout.md\` in the paperlint package).
68
+ \`<paper-dir>/repro/<id>-<slug>/\`, made by \`npx paperlint studies new <paper-dir> "Gold labels"
69
+ --kind data\` when the labelling is a study of its own (\`P1-S04-gold-labels\`): the command writes
70
+ its \`README.md\` and its row in \`repro/studies.json\`, and \`repro/study-registered\` checks both. When
71
+ the labels are in, update the row (\`status\`, \`result\`, \`result_file\`) and run
72
+ \`npx paperlint studies index <paper-dir>\`. Where this sits in stage 3, and what checks each place:
73
+ \`docs/layout.md\` in the paperlint package.
71
74
  \`items.json\` follows the header of the generator. Source ids in \`items.json\` are the study's own;
72
75
  the page never shows them.
73
76
 
@@ -29,7 +29,8 @@ the 140-run `reproduce.py` (AgenticDev) and the 46-guard `evaluate.py`/`mutate.p
29
29
 
30
30
  In the paper folder the artifact lives in `repro/`, laid out as `repro/allowed-files-only` requires: the
31
31
  scripts the whole paper uses in `repro/tools/`, shared data in `repro/data/`, each study in
32
- `repro/<slug>-YYYY-MM-DD/` with its `README.md`, and `repro/README.md` as the index. The shipped
32
+ `repro/<id>-<slug>/` with its `README.md` and its row in `repro/studies.json`
33
+ (`repro/study-registered`), and `repro/README.md` as the index generated from it. The shipped
33
34
  zip below is built from it; the anonymized copy that was uploaded goes in the paper's `submission/`.
34
35
 
35
36
  - `reproduce.py` / the harness scripts — self-check, print `RESULT: PASS`, exit non-zero on drift.
@@ -39,9 +39,11 @@ one file, so there is one place to read and nothing to keep in step:
39
39
  - **No `PLAN.md` beside it.** A plan in a second file drifts from the status it plans against;
40
40
  `paper/allowed-files-only` reports a `PLAN.md` at the paper's root. A decision's _history_ (why it was
41
41
  taken, what was rejected) is a record, not live state: it goes to `research/` as a dated file.
42
- - **Each study gets a dated folder in `repro/`.** The `study` stage's runs go in
43
- `repro/<slug>-YYYY-MM-DD/` (or `<slug>-YYYY-MM` for one that ran over weeks), with a `README.md`
44
- saying what it asks, how to run it and what it found, and a line in `repro/README.md`. Scripts the
42
+ - **Each study gets a registered folder in `repro/`.** The `study` stage's runs go in
43
+ `repro/<id>-<slug>/`, created by `npx paperlint studies new <paper-dir> "<title>"`, with a `README.md`
44
+ saying what it asks, how to run it and what it found, and a row in `repro/studies.json` that is
45
+ updated when the study ends (then `npx paperlint studies index <paper-dir>`); `repro/study-registered`
46
+ holds it. Scripts the
45
47
  whole paper uses go in `repro/tools/`, shared data in `repro/data/`; `repro/allowed-files-only` reports
46
48
  anything else at the top of `repro/`.
47
49
  - **The scorecard's verdicts live only here.** The dated report `study-accepted-papers` writes under
@@ -184,10 +184,19 @@ if [ -f "$BLG" ]; then
184
184
  grep -E "^(You're|I'm skipping|Repeated entry|I couldn't open)" "$BLG" | sed 's/^/ /' | head -12
185
185
  fi
186
186
  # A cited key that never made it into the .bbl is the same failure seen from the other side.
187
+ # An IEEEtran BST control entry (`@IEEEtranBSTCTL`, activated by `\bstctlcite{key}`) is cited
188
+ # in the .aux like any key, but IEEEtran reads it as configuration and never prints it as a
189
+ # \bibitem — so it is not a missing reference. A key is exempt only when IEEEtran itself reports
190
+ # reading it as a control entry: the .blg line `** IEEEtran BST control entry "key" detected.`,
191
+ # which every IEEEtran*.bst prints. BibTeX wrote that line after parsing the build's own databases,
192
+ # so a record in a .bib the build does not read, inside `@comment{…}`, or misspelled exempts
193
+ # nothing, and every layout BibTeX accepts is recognised without parsing .bib files here.
187
194
  if [ -f "$BASE.aux" ] && [ -f "$BASE.bbl" ]; then
188
195
  missing=0
196
+ ctl=$(sed -n 's/^\*\* IEEEtran BST control entry "\(.*\)" detected\.$/\1/p' "$BASE.blg" 2>/dev/null | sort -u)
189
197
  while read -r key; do
190
198
  [ -z "$key" ] && continue
199
+ printf '%s\n' "$ctl" | grep -qxF -- "$key" && continue
191
200
  grep -qF "{$key}" "$BASE.bbl" || { echo " cited but NOT in the bibliography: $key"; missing=$((missing+1)); }
192
201
  done < <(grep -oE '\\citation\{[^}]*\}' "$BASE.aux" | sed 's/.*{//;s/}//' | tr ',' '\n' | sort -u)
193
202
  [ "$missing" -gt 0 ] && { echo " CITED-BUT-ABSENT references: $missing"; blgerr=$((blgerr+missing)); }
@@ -295,3 +295,105 @@ await checkSkill("render-paper");
295
295
  : ` under ${root}/`),
296
296
  );
297
297
  }
298
+
299
+ // ── AN IEEEtran BST CONTROL ENTRY IS NOT A MISSING REFERENCE ─────────────────
300
+ // `\\bstctlcite{key}` cites an `@IEEEtranBSTCTL{key, …}` entry, so the key lands in the .aux like any
301
+ // citation, but IEEEtran reads it as configuration and never prints a \\bibitem for it. The
302
+ // "cited but NOT in the bibliography" check failed a real IEEE paper on exactly that key (an
303
+ // "et al. after six authors" setting). The exemption comes only from IEEEtran's own .blg line
304
+ // `** IEEEtran BST control entry "key" detected.`: BibTeX wrote it after parsing the build's
305
+ // databases, so a record BibTeX never read (a .bib the build does not use, a record inside
306
+ // `@comment{…}`) or a misspelled key exempts nothing, and a genuinely missing key is still reported.
307
+ {
308
+ const HERE = dirname(fileURLToPath(import.meta.url));
309
+ const CHECK = join(HERE, "check-render.sh");
310
+ // `detected` = the keys IEEEtran reported in the .blg as control entries it read.
311
+ const paper = (citations, detected = ["BSTcontrol"]) => {
312
+ const dir = realpathSync(mkdtempSync(join(tmpdir(), "bstctl-")));
313
+ writeFileSync(
314
+ join(dir, "paper.tex"),
315
+ "\\documentclass{IEEEtran}\n\\begin{document}\n\\bstctlcite{BSTcontrol}\nText \\cite{real}.\n\\bibliographystyle{IEEEtran}\n\\bibliography{refs}\n\\end{document}\n",
316
+ );
317
+ writeFileSync(
318
+ join(dir, "paper.aux"),
319
+ citations.map((k) => `\\citation{${k}}`).join("\n") +
320
+ "\n\\bibstyle{IEEEtran}\n\\bibdata{refs}\n",
321
+ );
322
+ writeFileSync(
323
+ join(dir, "paper.bbl"),
324
+ "\\begin{thebibliography}{1}\n\\bibitem{real}\nA real reference, 2026.\n\\end{thebibliography}\n",
325
+ );
326
+ writeFileSync(
327
+ join(dir, "paper.blg"),
328
+ "This is BibTeX, Version 0.99e\nDatabase file #1: refs.bib\n-- IEEEtran.bst version 1.14 (2015/08/26) by Michael Shell.\n" +
329
+ detected
330
+ .map((k) => `** IEEEtran BST control entry "${k}" detected.\n`)
331
+ .join("") +
332
+ "\nDone.\n",
333
+ );
334
+ writeFileSync(
335
+ join(dir, "paper.log"),
336
+ "This is pdfTeX\nOutput written on paper.pdf (1 page, 1000 bytes).\n",
337
+ );
338
+ const r = spawnSync("bash", [CHECK, dir, "paper"], {
339
+ encoding: "utf8",
340
+ env: { ...process.env, NO_COMPILE: "1" },
341
+ });
342
+ // The script must run to its verdict: a run that stopped early prints no complaint either, and
343
+ // "no complaint" is what the exemption case asserts. stderr and the bash version name the cause.
344
+ assert.match(
345
+ r.stdout,
346
+ /^(PASS|FAIL)\b/m,
347
+ `check-render.sh stopped before its verdict (exit ${r.status}, ${BASH_VERSION}):\n` +
348
+ `--- stdout\n${r.stdout}--- stderr\n${r.stderr}`,
349
+ );
350
+ return r;
351
+ };
352
+ const BASH_VERSION = spawnSync("bash", ["--version"], {
353
+ encoding: "utf8",
354
+ }).stdout.split("\n")[0];
355
+
356
+ const ctlOnly = paper(["BSTcontrol", "real"]);
357
+ assert.doesNotMatch(
358
+ ctlOnly.stdout,
359
+ /CITED-BUT-ABSENT/,
360
+ `a control key IEEEtran reported reading was flagged as a missing reference:\n${ctlOnly.stdout}`,
361
+ );
362
+
363
+ // A key that starts with "-" is still a key, not a grep option.
364
+ const dashKey = paper(["-ctl", "real"], ["-ctl"]);
365
+ assert.doesNotMatch(
366
+ dashKey.stdout,
367
+ /CITED-BUT-ABSENT/,
368
+ `a control key starting with "-" was flagged:\n${dashKey.stdout}`,
369
+ );
370
+
371
+ // A misspelled \\bstctlcite key: IEEEtran detected a different key, so this one is not exempt.
372
+ const typo = paper(["BSTcontrl", "real"]);
373
+ assert.match(
374
+ typo.stdout,
375
+ /cited but NOT in the bibliography: BSTcontrl/,
376
+ `a \\bstctlcite key IEEEtran did not report must be flagged:\n${typo.stdout}`,
377
+ );
378
+
379
+ // A record BibTeX never read — in a .bib the build does not use, or inside @comment{…} — leaves
380
+ // no "detected" line, so the key is not exempt.
381
+ const unread = paper(["BSTcontrol", "real"], []);
382
+ assert.match(
383
+ unread.stdout,
384
+ /cited but NOT in the bibliography: BSTcontrol/,
385
+ `a control key IEEEtran never reported reading must be flagged:\n${unread.stdout}`,
386
+ );
387
+
388
+ const withGap = paper(["BSTcontrol", "real", "gone"]);
389
+ assert.match(
390
+ withGap.stdout,
391
+ /cited but NOT in the bibliography: gone/,
392
+ `a key with no \\bibitem must still be reported:\n${withGap.stdout}`,
393
+ );
394
+ assert.doesNotMatch(
395
+ withGap.stdout,
396
+ /bibliography: BSTcontrol/,
397
+ withGap.stdout,
398
+ );
399
+ }
@@ -4,7 +4,7 @@ description: "Run a paid or counted agent experiment grid end to end — units
4
4
  allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Agent]
5
5
  ---
6
6
 
7
- <!-- vigiles:sha256:aef4152e680a2ccd compiled from skills/run-agent-grid/SKILL.md.spec.ts -->
7
+ <!-- vigiles:sha256:522a747e8eb6ead4 compiled from skills/run-agent-grid/SKILL.md.spec.ts -->
8
8
 
9
9
  # run-agent-grid — running a counted agent grid without spoiling it
10
10
 
@@ -23,6 +23,13 @@ and a driver loop that calls the guard: [`references/registry-and-ledger.md`](re
23
23
  **Exit codes of every guard command:** 0 allowed or clean · 1 refused or a finding · 2 usage ·
24
24
  3 (`gate` only) the cell is already counted, skip it.
25
25
 
26
+ **The grid's study folder.** A grid is one study of the paper. Its folder is made by
27
+ `npx paperlint studies new <paper-dir> "<title>" --kind measurement` — the next id in
28
+ `repro/studies.json`, the folder `repro/<id>-<slug>/` — and `grid.json`, the runs and the ledger live
29
+ in it. When the grid ends, its row is updated (`status`, `result`, `result_file`, `in_paper`) and
30
+ `npx paperlint studies index <paper-dir>` regenerates the index; `repro/study-registered` fails the
31
+ lint until both are done.
32
+
26
33
  ## The order — each gate is passed before the next phase starts
27
34
 
28
35
  ### 1. The registry: what may be measured, written as data (catalog 1, 3, 14)
@@ -24,6 +24,13 @@ and a driver loop that calls the guard: [\`references/registry-and-ledger.md\`](
24
24
  **Exit codes of every guard command:** 0 allowed or clean · 1 refused or a finding · 2 usage ·
25
25
  3 (\`${COMMANDS.gate}\` only) the cell is already counted, skip it.
26
26
 
27
+ **The grid's study folder.** A grid is one study of the paper. Its folder is made by
28
+ \`npx paperlint studies new <paper-dir> "<title>" --kind measurement\` — the next id in
29
+ \`repro/studies.json\`, the folder \`repro/<id>-<slug>/\` — and \`grid.json\`, the runs and the ledger live
30
+ in it. When the grid ends, its row is updated (\`status\`, \`result\`, \`result_file\`, \`in_paper\`) and
31
+ \`npx paperlint studies index <paper-dir>\` regenerates the index; \`repro/study-registered\` fails the
32
+ lint until both are done.
33
+
27
34
  ## The order — each gate is passed before the next phase starts
28
35
 
29
36
  ### 1. The registry: what may be measured, written as data (catalog 1, 3, 14)
@@ -0,0 +1,144 @@
1
+ /**
2
+ * The schema of `repro/studies.json`: the text a registry is parsed from, the extra fields a row keeps,
3
+ * and every way a file can fail to be a registry — refused with the reasons, never read as one.
4
+ */
5
+ import { describe, expect, it } from "vitest";
6
+ import {
7
+ registryText,
8
+ type StudyRegistry,
9
+ } from "../../domain/study-registry.ts";
10
+ import { studyRegistryCodec } from "./index.ts";
11
+
12
+ const parse = studyRegistryCodec.parse;
13
+
14
+ const ROW = {
15
+ id: "P1-S01",
16
+ folder: "P1-S01-baseline",
17
+ title: "Baseline pass rate",
18
+ question: "How often does the agent pass?",
19
+ kind: "measurement",
20
+ status: "done",
21
+ date: "2026-10-15",
22
+ result: "41 of 60 pass",
23
+ result_file: "results.tsv",
24
+ superseded_by: "P1-S02",
25
+ in_paper: "yes",
26
+ } as const;
27
+
28
+ const text = (v: unknown): string => JSON.stringify(v);
29
+ const withRow = (row: Readonly<Record<string, unknown>>): string =>
30
+ text({ prefix: "P1", studies: [row] });
31
+
32
+ describe("parsing a studies.json", () => {
33
+ it("a registry parses to itself, and what registryText writes parses back", () => {
34
+ const registry: StudyRegistry = { prefix: "P1", studies: [ROW] };
35
+ expect(parse(text(registry))).toEqual({ ok: true, registry });
36
+ expect(parse(registryText(registry))).toEqual({ ok: true, registry });
37
+ });
38
+
39
+ it("the optional fields may be absent", () => {
40
+ const { result, result_file, superseded_by, ...row } = ROW;
41
+ expect([result, result_file, superseded_by]).toHaveLength(3);
42
+ expect(parse(withRow(row))).toEqual({
43
+ ok: true,
44
+ registry: { prefix: "P1", studies: [row] },
45
+ });
46
+ });
47
+
48
+ it("🔴 a row's extra fields are kept, whatever they hold", () => {
49
+ const row = {
50
+ ...ROW,
51
+ role: "counted",
52
+ frame: { population: 4210 },
53
+ rules_under_test: ["a", "b"],
54
+ author_check: null,
55
+ };
56
+ expect(parse(withRow(row))).toEqual({
57
+ ok: true,
58
+ registry: { prefix: "P1", studies: [row] },
59
+ });
60
+ });
61
+
62
+ it("an empty registry parses", () => {
63
+ expect(parse(text({ prefix: "AB2", studies: [] }))).toEqual({
64
+ ok: true,
65
+ registry: { prefix: "AB2", studies: [] },
66
+ });
67
+ });
68
+ });
69
+
70
+ describe("refusing what is not a registry", () => {
71
+ it("text that is not JSON", () => {
72
+ expect(parse("{ prefix: P1")).toEqual({ ok: false, why: "not JSON" });
73
+ });
74
+
75
+ it.each([
76
+ [
77
+ "no prefix",
78
+ { studies: [] },
79
+ "prefix: Invalid input: expected string, received undefined",
80
+ ],
81
+ [
82
+ "a prefix of another shape",
83
+ { prefix: "p1", studies: [] },
84
+ "prefix: a capital letter, then capitals or digits (P1)",
85
+ ],
86
+ [
87
+ "an unknown top-level key",
88
+ { prefix: "P1", studies: [], studys: [] },
89
+ '(top): Unrecognized key: "studys"',
90
+ ],
91
+ [
92
+ "studies not a list",
93
+ { prefix: "P1", studies: {} },
94
+ "studies: Invalid input: expected array, received object",
95
+ ],
96
+ ])("%s", (_why, v, why) => {
97
+ expect(parse(text(v))).toEqual({ ok: false, why });
98
+ });
99
+ });
100
+
101
+ describe("refusing a row that is not a study", () => {
102
+ it.each([
103
+ [
104
+ "kind",
105
+ "benchmark",
106
+ 'studies.0.kind: Invalid option: expected one of "measurement"|"spike"|"tooling"|"data"|"artifact"|"salvage"',
107
+ ],
108
+ [
109
+ "status",
110
+ "finished",
111
+ 'studies.0.status: Invalid option: expected one of "designing"|"running"|"done"|"stopped"|"superseded"|"dropped"',
112
+ ],
113
+ [
114
+ "in_paper",
115
+ "maybe",
116
+ 'studies.0.in_paper: Invalid option: expected one of "yes"|"no"|"registry"',
117
+ ],
118
+ ["date", "15.10.2026", "studies.0.date: Invalid ISO date"],
119
+ [
120
+ "title",
121
+ "",
122
+ "studies.0.title: Too small: expected string to have >=1 characters",
123
+ ],
124
+ [
125
+ "result",
126
+ 41,
127
+ "studies.0.result: Invalid input: expected string, received number",
128
+ ],
129
+ ])("a row whose %s is %j", (field, value, why) => {
130
+ expect(parse(withRow({ ...ROW, [field]: value }))).toEqual({
131
+ ok: false,
132
+ why,
133
+ });
134
+ });
135
+
136
+ it("every reason, joined", () => {
137
+ const { question, ...row } = ROW;
138
+ expect(question).toBeDefined();
139
+ expect(parse(text({ prefix: 1, studies: [row] }))).toEqual({
140
+ ok: false,
141
+ why: "prefix: Invalid input: expected string, received number; studies.0.question: Invalid input: expected string, received undefined",
142
+ });
143
+ });
144
+ });
@@ -0,0 +1,70 @@
1
+ /**
2
+ * The schema of `repro/studies.json` — the one place its shape is checked (docs/design/study-registry.md).
3
+ * The text is parsed once, here, into the domain's `StudyRegistry`; whatever reads a registry after it
4
+ * never re-validates.
5
+ *
6
+ * A row is a loose object: a paper merges its own measurement metadata into its rows (`role`, `frame`,
7
+ * `rules_under_test`, `outcome` …), and those fields are kept as they are. The top level is strict, so
8
+ * a misspelt key is an error rather than a key nobody reads.
9
+ */
10
+ import { z } from "zod";
11
+ import {
12
+ IN_PAPER,
13
+ PREFIX_PATTERN,
14
+ STUDY_KINDS,
15
+ STUDY_STATUSES,
16
+ type ParsedRegistry,
17
+ } from "../../domain/study-registry.ts";
18
+ import type { StudyRegistryCodec } from "../../ports/study-registry-codec.ts";
19
+
20
+ const study = z.looseObject({
21
+ id: z.string(),
22
+ folder: z.string(),
23
+ title: z.string().min(1),
24
+ question: z.string(),
25
+ kind: z.enum(STUDY_KINDS),
26
+ status: z.enum(STUDY_STATUSES),
27
+ date: z.iso.date(),
28
+ result: z.string().optional(),
29
+ result_file: z.string().optional(),
30
+ superseded_by: z.string().optional(),
31
+ in_paper: z.enum(IN_PAPER),
32
+ });
33
+
34
+ const registry = z.strictObject({
35
+ prefix: z
36
+ .string()
37
+ .regex(PREFIX_PATTERN, "a capital letter, then capitals or digits (P1)"),
38
+ studies: z.array(study),
39
+ });
40
+
41
+ /** The JSON `text` holds, or undefined when it is not JSON. */
42
+ function jsonOf(text: string): unknown {
43
+ try {
44
+ return JSON.parse(text);
45
+ } catch {
46
+ return undefined;
47
+ }
48
+ }
49
+
50
+ /** A `studies.json`'s text, parsed once at the boundary: the registry, or every reason it is not one. */
51
+ export function parseStudyRegistry(text: string): ParsedRegistry {
52
+ const json = jsonOf(text);
53
+ if (json === undefined) return { ok: false, why: "not JSON" };
54
+ const parsed = registry.safeParse(json);
55
+ return parsed.success
56
+ ? { ok: true, registry: parsed.data }
57
+ : {
58
+ ok: false,
59
+ why: parsed.error.issues
60
+ .map(
61
+ (i) =>
62
+ `${i.path.length === 0 ? "(top)" : i.path.join(".")}: ${i.message}`,
63
+ )
64
+ .join("; "),
65
+ };
66
+ }
67
+
68
+ export const studyRegistryCodec: StudyRegistryCodec = {
69
+ parse: parseStudyRegistry,
70
+ };
package/src/cli.ts CHANGED
@@ -185,6 +185,12 @@ import {
185
185
  REPRO_LAYOUT_RULE_LEVELS,
186
186
  reproLayoutRules,
187
187
  } from "./repro-layout-rule.ts";
188
+ import {
189
+ STUDY_REGISTRY_RULE_LEVELS,
190
+ studyRegistryRules,
191
+ } from "./study-registry-rule.ts";
192
+ import { studyRegistryCodec } from "./adapters/study-registry/index.ts";
193
+ import { runStudies } from "./studies.ts";
188
194
  import typography from "#eslint-rules/paper-typography";
189
195
  import texBuild from "#eslint-rules/tex-build";
190
196
  import reviewFrontmatter from "#eslint-rules/review-frontmatter";
@@ -230,6 +236,12 @@ const USAGE = `paperlint — machine-checkable gates for a paper kept in git
230
236
  send the PDF (default: the paper's own), an abstract, or
231
237
  "submitted". A DRY RUN unless --save: the portal checks the
232
238
  change and keeps nothing
239
+ npx paperlint studies new <paper-dir> "<title>" [--kind <kind>]
240
+ register a study: the next id in repro/studies.json, the folder
241
+ repro/<id>-<slug>/ with its README.md, the row (status designing,
242
+ today's date), and the index in repro/README.md regenerated
243
+ npx paperlint studies index <paper-dir>
244
+ regenerate the index in repro/README.md from repro/studies.json
233
245
  npx paperlint hook <name> run an editor hook (.claude/settings.json calls this)
234
246
  npx paperlint --help
235
247
 
@@ -250,6 +262,11 @@ new:
250
262
  "realm" can be the work's own). Without it such a name is refused: a rejected
251
263
  paper moves to another venue and keeps its folder
252
264
 
265
+ studies:
266
+ --kind <kind> new: measurement (the default), spike, tooling, data, artifact or salvage.
267
+ A study's status, result and in_paper are edited in repro/studies.json;
268
+ the rule repro/study-registered checks them (docs/rules/repro/study-registered.md)
269
+
253
270
  submission:
254
271
  The venue preset declares the portal — "portal": { "kind": "hotcrp", "url": "<site>" } — and the
255
272
  paper's paperlint.json its submission there — "submission": { "id": <number> }. The token comes
@@ -367,6 +384,27 @@ const FRAGMENT_RULES = {
367
384
  /** What the two layout rules read with: the disk and its directory listings. */
368
385
  const LAYOUT_RULE_DEPS = { files: nodeFiles, listDir: nodeListDir };
369
386
 
387
+ /** What `repro/study-registered` and `paperlint studies` read with: the layout's, and the registry's parser. */
388
+ const STUDY_RULE_DEPS = { ...LAYOUT_RULE_DEPS, codec: studyRegistryCodec };
389
+
390
+ /** The `repro` plugin's rules, on every `PIPELINE-STATUS.md`: the layout of `repro/`, and its study registry. */
391
+ const REPRO_RULES = {
392
+ ...reproLayoutRules(LAYOUT_RULE_DEPS),
393
+ ...studyRegistryRules(STUDY_RULE_DEPS),
394
+ };
395
+ const REPRO_RULE_LEVELS = {
396
+ ...REPRO_LAYOUT_RULE_LEVELS,
397
+ ...STUDY_REGISTRY_RULE_LEVELS,
398
+ };
399
+
400
+ /** Today in this machine's time zone, `YYYY-MM-DD`: the date `paperlint studies new` registers a study on. */
401
+ const today = (now = new Date()): string =>
402
+ [
403
+ String(now.getFullYear()),
404
+ String(now.getMonth() + 1).padStart(2, "0"),
405
+ String(now.getDate()).padStart(2, "0"),
406
+ ].join("-");
407
+
370
408
  /** What the talk rules read with: the disk, the shipped presets, and the three media readers. */
371
409
  const TALK_RULE_DEPS = {
372
410
  files: nodeFiles,
@@ -433,7 +471,7 @@ export function buildConfig(
433
471
  ...folderLayoutRules(LAYOUT_RULE_DEPS),
434
472
  },
435
473
  },
436
- repro: { rules: reproLayoutRules(LAYOUT_RULE_DEPS) },
474
+ repro: { rules: REPRO_RULES },
437
475
  // Registered, on for no paper: `"study/declare-sample": "error"` in paperlint.json turns it on.
438
476
  study: { rules: preregCoverage.rules },
439
477
  cycle: { rules: CYCLE_STATUS_RULES },
@@ -444,7 +482,7 @@ export function buildConfig(
444
482
  "paper/source": "error",
445
483
  ...FOLDER_VENUE_RULE_LEVELS,
446
484
  ...FOLDER_LAYOUT_RULE_LEVELS,
447
- ...REPRO_LAYOUT_RULE_LEVELS,
485
+ ...REPRO_RULE_LEVELS,
448
486
  ...CYCLE_RULE_LEVELS,
449
487
  },
450
488
  },
@@ -1626,6 +1664,17 @@ const SIMPLE: Readonly<
1626
1664
  banal: hostBanalInstaller(),
1627
1665
  tex: toolchainTex(cwd),
1628
1666
  }),
1667
+ studies: (a, { log, err, cwd }) =>
1668
+ runStudies(
1669
+ {
1670
+ sub: a.paths[0],
1671
+ paperDir: a.paths[1],
1672
+ title: a.paths[2],
1673
+ extra: a.paths.slice(3),
1674
+ kind: a.kind,
1675
+ },
1676
+ { ...STUDY_RULE_DEPS, cwd, today: today(), log, err },
1677
+ ),
1629
1678
  submission: (a, { log, err, cwd }) =>
1630
1679
  runSubmission(
1631
1680
  {
@@ -257,7 +257,7 @@ describe("strayMessage — stands alone for a reader with no context", () => {
257
257
  ),
258
258
  ).toEqual([]);
259
259
  expect(lines).toContain(
260
- "│ └── <slug>-YYYY-MM-DD/ ← one folder per study or spike (<slug>-YYYY-MM for one that ran over weeks)",
260
+ "│ └── <id>-<slug>/ ← one folder per study or spike, named by its row in studies.json (P1-S03-baseline)",
261
261
  );
262
262
  expect(lines.at(-1)).toMatch(/^└── extra\/ +← anything; never judged$/);
263
263
  });
@@ -38,6 +38,9 @@ describe("isStudyName", () => {
38
38
  "leap-2024-02-29",
39
39
  "leap-2000-02-29",
40
40
  "april-2026-04-30",
41
+ "P1-S01-baseline",
42
+ "P12-S100-a1-b2",
43
+ "AB-S07-x",
41
44
  ])("%s is a study's name", (name) => {
42
45
  expect(isStudyName(name)).toBe(true);
43
46
  });
@@ -56,6 +59,11 @@ describe("isStudyName", () => {
56
59
  ["29 February 1900", "x-1900-02-29"],
57
60
  ["32 October", "x-2026-10-32"],
58
61
  ["a two-digit year", "x-26-10-12"],
62
+ ["an id with one digit", "P1-S1-baseline"],
63
+ ["an id and no slug", "P1-S01"],
64
+ ["a lowercase id", "p1-s01-baseline"],
65
+ ["a capital in the slug", "P1-S01-Baseline"],
66
+ ["an id with no S", "P1-01-baseline"],
59
67
  ])("not with %s: %s", (_why, name) => {
60
68
  expect(isStudyName(name)).toBe(false);
61
69
  });
@@ -158,7 +166,7 @@ describe("judgeRepro — reports a folder, one finding each", () => {
158
166
  {
159
167
  path: "repro/Old_Baseline/",
160
168
  problem: "stray",
161
- cure: "rename it to `old-baseline-YYYY-MM-DD`, one folder per study or spike, named for what it is and the date it began — the date of its first commit is a fine default (`git log --reverse --format=%as -- repro/Old_Baseline | head -1`, run in the paper folder) — and give it a README.md; if it holds scripts or data shared across the paper instead, move them into repro/tools/ or repro/data/",
169
+ cure: "rename it to `<id>-old-baseline` — the next free id in repro/studies.json, then what it is — give it its row there and a README.md (`npx paperlint studies new` makes all three for a new study); in a paper with repro/study-registered off, `old-baseline-YYYY-MM-DD` is accepted instead, dated by the day it began — its first commit is a fine default (`git log --reverse --format=%as -- repro/Old_Baseline | head -1`, run in the paper folder); if it holds scripts or data shared across the paper instead, move them into repro/tools/ or repro/data/",
162
170
  },
163
171
  ]);
164
172
  });
@@ -168,7 +176,7 @@ describe("judgeRepro — reports a folder, one finding each", () => {
168
176
  {
169
177
  path: "repro/ablation-2026-10-12/",
170
178
  problem: "no-readme",
171
- cure: "add repro/ablation-2026-10-12/README.md — what the study asks, how to run it, what it found — and a line for it in repro/README.md",
179
+ cure: "add repro/ablation-2026-10-12/README.md — what the study asks, how to run it, what it found — and give the study its row in repro/studies.json",
172
180
  },
173
181
  ]);
174
182
  });
@@ -194,15 +202,16 @@ describe("reproMessage — stands alone", () => {
194
202
  }),
195
203
  ).toBe(
196
204
  [
197
- `\`repro/run.py\` is loose at the top of the paper's repro/ folder, which holds only an index, the shared tools and data, and one dated folder per study. Where it goes: ${SCRIPT_CURE}.`,
205
+ `\`repro/run.py\` is loose at the top of the paper's repro/ folder, which holds only an index, the study registry, the shared tools and data, and one folder per study. Where it goes: ${SCRIPT_CURE}.`,
198
206
  "What repro/ holds (what interpreters and package managers leave there, like __pycache__/ and node_modules/, is never reported):",
199
207
  "repro/",
200
- "├── README.md ← the index: one line per study",
201
- "├── .gitignore ← keeps the scripts' byproducts out of git",
202
- "├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)",
203
- "├── data/ ← shared data (tables, caches)",
204
- "└── <slug>-YYYY-MM-DD/ ← one folder per study or spike (<slug>-YYYY-MM for one that ran over weeks)",
205
- " └── README.md ← required: what it asks, how to run it, what it found",
208
+ "├── README.md ← the index: one line per study, generated from studies.json",
209
+ "├── studies.json ← the study registry: id, title, status and result of each study",
210
+ "├── .gitignore ← keeps the scripts' byproducts out of git",
211
+ "├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)",
212
+ "├── data/ ← shared data (tables, caches)",
213
+ "└── <id>-<slug>/ ← one folder per study or spike, named by its row in studies.json (P1-S03-baseline)",
214
+ " └── README.md ← required: what it asks, how to run it, what it found",
206
215
  "Rule repro/allowed-files-only: https://github.com/zernie/paperlint/blob/main/docs/rules/repro/allowed-files-only.md",
207
216
  ].join("\n"),
208
217
  );