paperlint 9.0.1 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -128
- package/dist/build.d.ts +1 -1
- package/dist/build.js +1 -1
- package/dist/cli.d.ts +5 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +52 -6
- package/dist/cli.js.map +1 -1
- package/dist/domain/layout-tree.d.ts +19 -0
- package/dist/domain/layout-tree.d.ts.map +1 -0
- package/dist/domain/layout-tree.js +32 -0
- package/dist/domain/layout-tree.js.map +1 -0
- package/dist/domain/lookup-cache.d.ts +2 -2
- package/dist/domain/lookup-cache.js +2 -2
- package/dist/domain/paper-layout.d.ts +81 -0
- package/dist/domain/paper-layout.d.ts.map +1 -0
- package/dist/domain/paper-layout.js +259 -0
- package/dist/domain/paper-layout.js.map +1 -0
- package/dist/domain/repro-layout.d.ts +80 -0
- package/dist/domain/repro-layout.d.ts.map +1 -0
- package/dist/domain/repro-layout.js +247 -0
- package/dist/domain/repro-layout.js.map +1 -0
- package/dist/domain/scaffold-layout.d.ts +28 -0
- package/dist/domain/scaffold-layout.d.ts.map +1 -0
- package/dist/domain/scaffold-layout.js +81 -0
- package/dist/domain/scaffold-layout.js.map +1 -0
- package/dist/eslint-rules/prereg-coverage.d.ts +98 -0
- package/dist/eslint-rules/prereg-coverage.d.ts.map +1 -0
- package/dist/eslint-rules/prereg-coverage.js +274 -0
- package/dist/eslint-rules/prereg-coverage.js.map +1 -0
- package/dist/folder-layout-hook.d.ts +21 -0
- package/dist/folder-layout-hook.d.ts.map +1 -0
- package/dist/folder-layout-hook.js +117 -0
- package/dist/folder-layout-hook.js.map +1 -0
- package/dist/folder-layout-rule.d.ts +22 -0
- package/dist/folder-layout-rule.d.ts.map +1 -0
- package/dist/folder-layout-rule.js +107 -0
- package/dist/folder-layout-rule.js.map +1 -0
- package/dist/init.js +1 -1
- package/dist/init.js.map +1 -1
- package/dist/new-paper.d.ts +4 -1
- package/dist/new-paper.d.ts.map +1 -1
- package/dist/new-paper.js +40 -6
- package/dist/new-paper.js.map +1 -1
- package/dist/references.d.ts +1 -1
- package/dist/references.d.ts.map +1 -1
- package/dist/references.js +2 -2
- package/dist/references.js.map +1 -1
- package/dist/repro-layout-rule.d.ts +16 -0
- package/dist/repro-layout-rule.d.ts.map +1 -0
- package/dist/repro-layout-rule.js +81 -0
- package/dist/repro-layout-rule.js.map +1 -0
- package/docs/ci.md +60 -0
- package/docs/configuration.md +2 -0
- package/docs/e2e.md +2 -2
- package/docs/install.md +20 -16
- package/docs/layout.md +154 -0
- package/docs/optional-rules.md +5 -4
- package/docs/references.md +5 -2
- package/docs/rules/paper/allowed-files-only.md +192 -0
- package/docs/rules/repro/allowed-files-only.md +128 -0
- package/docs/rules/study/declare-sample.md +146 -0
- package/docs/rules.md +53 -50
- package/docs/skills.md +6 -3
- package/docs/venues.md +94 -0
- package/eslint-rules/prereg-coverage.test.ts +448 -0
- package/eslint-rules/prereg-coverage.ts +419 -0
- package/lib/skill-trigger-cases.harness.mjs +51 -5
- package/lib/skill-trigger-cases.mjs +53 -15
- package/package.json +1 -1
- package/plugin/hooks/hooks.json +9 -0
- package/skills/analyze-sibling-paper/SKILL.md +2 -2
- package/skills/analyze-sibling-paper/SKILL.md.spec.ts +1 -1
- package/skills/build-benchmark/SKILL.md +14 -1
- package/skills/build-benchmark/SKILL.md.spec.ts +13 -0
- package/skills/build-label-page/SKILL.md +118 -0
- package/skills/build-label-page/SKILL.md.spec.ts +129 -0
- package/skills/build-label-page/build-label-page.eval.mjs +15 -0
- package/skills/build-label-page/build-label-page.test.ts +959 -0
- package/skills/build-label-page/references/checklist.md +148 -0
- package/skills/build-label-page/references/example-generator.py +803 -0
- package/skills/find-venue/SKILL.md +5 -3
- package/skills/find-venue/SKILL.md.spec.ts +4 -2
- package/skills/grade-paper-writing/SKILL.md +7 -7
- package/skills/grade-paper-writing/SKILL.md.spec.ts +6 -6
- package/skills/map-prior-work/SKILL.md +2 -2
- package/skills/map-prior-work/SKILL.md.spec.ts +1 -1
- package/skills/paper-pipeline/SKILL.md +8 -6
- package/skills/paper-pipeline/SKILL.md.spec.ts +7 -5
- package/skills/paper-pipeline/grade-paper-writing-ablation.eval.mjs +19 -9
- package/skills/paper-pipeline/references/acceptance-gate.md +19 -19
- package/skills/paper-pipeline/references/artifact-checklist.md +5 -0
- package/skills/paper-pipeline/references/pipeline-status-template.md +30 -1
- package/skills/paper-pipeline/repro/2026-08-07-fork-check.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-fork-check2.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-language-eval-pilot.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-language-eval-raw.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-08-framing-vs-vocabulary-oracle.log +48 -48
- package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-A6-oracle.log +8 -8
- package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-oracle.log +48 -48
- package/skills/paper-pipeline/repro/2026-08-08-parent-replication-gpw.log +2 -2
- package/skills/paper-pipeline/repro/2026-08-08-parent-replication.mjs +3 -3
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-after.log +81 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-before.log +81 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-setupdiff.log +59 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.jsonl +1 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.log +22 -0
- package/skills/paper-pipeline/repro/README.md +4 -0
- package/skills/paper-pipeline/scripts/extract-ref-facts.harness.mjs +2 -2
- package/skills/paper-pipeline/scripts/extract-ref-facts.mjs +2 -2
- package/skills/paper-pipeline/scripts/extract-ref-facts.test.mjs +4 -2
- package/skills/paper-pipeline/scripts/generated-code.harness.mjs +7 -2
- package/skills/paper-pipeline/scripts/generated-code.mjs +8 -4
- package/skills/paper-pipeline/scripts/generated-code.test.ts +15 -1
- package/skills/paper-status/SKILL.md +2 -2
- package/skills/paper-status/SKILL.md.spec.ts +1 -1
- package/skills/study-accepted-papers/SKILL.md +163 -161
- package/skills/study-accepted-papers/SKILL.md.spec.ts +167 -165
- package/skills/study-accepted-papers/references/base-rates.md +69 -0
- package/skills/study-accepted-papers/references/mechanisms.md +181 -0
- package/skills/verify-citations/SKILL.md +2 -2
- package/skills/verify-citations/SKILL.md.spec.ts +1 -1
- package/src/build.ts +1 -1
- package/src/cli.harness.mjs +42 -0
- package/src/cli.test.ts +90 -6
- package/src/cli.ts +78 -5
- package/src/domain/layout-tree.test.ts +32 -0
- package/src/domain/layout-tree.ts +47 -0
- package/src/domain/lookup-cache.ts +2 -2
- package/src/domain/paper-layout.test.ts +264 -0
- package/src/domain/paper-layout.ts +336 -0
- package/src/domain/repro-layout.test.ts +222 -0
- package/src/domain/repro-layout.ts +311 -0
- package/src/domain/scaffold-layout.ts +104 -0
- package/src/folder-layout-hook.test.ts +214 -0
- package/src/folder-layout-hook.ts +141 -0
- package/src/folder-layout-rule.test.ts +235 -0
- package/src/folder-layout-rule.ts +151 -0
- package/src/hooks-settings.harness.ts +9 -8
- package/src/init.ts +1 -1
- package/src/layout-docs.test.ts +52 -0
- package/src/new-paper.harness.mjs +23 -1
- package/src/new-paper.test.ts +299 -6
- package/src/new-paper.ts +51 -5
- package/src/paper-files.test.ts +9 -6
- package/src/references.test.ts +5 -5
- package/src/references.ts +2 -2
- package/src/repro-layout-rule.test.ts +114 -0
- package/src/repro-layout-rule.ts +121 -0
- package/src/rule-cases.test.ts +98 -0
- package/templates/paper/PIPELINE-STATUS.md +41 -7
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The layout of a paper's `repro/` (`src/domain/repro-layout.ts`), judged on its top entries alone.
|
|
3
|
+
* Both halves: the layout itself, the byproducts and dated study folders with a README.md are clean;
|
|
4
|
+
* every other entry is reported once, with where it goes by kind, and a study folder without its
|
|
5
|
+
* README.md is reported once.
|
|
6
|
+
*/
|
|
7
|
+
import { describe, expect, it } from "vitest";
|
|
8
|
+
import {
|
|
9
|
+
DATA_CURE,
|
|
10
|
+
FILE_CURE_BY_KIND,
|
|
11
|
+
MOVED_FILES,
|
|
12
|
+
NOTE_CURE,
|
|
13
|
+
SCRIPT_CURE,
|
|
14
|
+
isStudyName,
|
|
15
|
+
judgeRepro,
|
|
16
|
+
reproMessage,
|
|
17
|
+
suggestedStudyName,
|
|
18
|
+
type ReproEntry,
|
|
19
|
+
} from "./repro-layout.ts";
|
|
20
|
+
|
|
21
|
+
const file = (name: string): ReproEntry => ({
|
|
22
|
+
name,
|
|
23
|
+
kind: "file",
|
|
24
|
+
hasReadme: false,
|
|
25
|
+
});
|
|
26
|
+
const dir = (name: string, hasReadme = true): ReproEntry => ({
|
|
27
|
+
name,
|
|
28
|
+
kind: "dir",
|
|
29
|
+
hasReadme,
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
describe("isStudyName", () => {
|
|
33
|
+
it.each([
|
|
34
|
+
"context-reading-2026-10",
|
|
35
|
+
"human-labels-2026-10",
|
|
36
|
+
"ablation-2026-10-12",
|
|
37
|
+
"a1-b2-2026-01-31",
|
|
38
|
+
"leap-2024-02-29",
|
|
39
|
+
"leap-2000-02-29",
|
|
40
|
+
"april-2026-04-30",
|
|
41
|
+
])("%s is a study's name", (name) => {
|
|
42
|
+
expect(isStudyName(name)).toBe(true);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it.each([
|
|
46
|
+
["no date", "ablation"],
|
|
47
|
+
["no slug", "2026-10-12"],
|
|
48
|
+
["a capital", "Ablation-2026-10-12"],
|
|
49
|
+
["an underscore", "human_labels-2026-10"],
|
|
50
|
+
["a doubled hyphen", "a--b-2026-10"],
|
|
51
|
+
["month 13", "x-2026-13"],
|
|
52
|
+
["month 00", "x-2026-00-01"],
|
|
53
|
+
["day 00", "x-2026-10-00"],
|
|
54
|
+
["31 April", "x-2026-04-31"],
|
|
55
|
+
["29 February of a common year", "x-2025-02-29"],
|
|
56
|
+
["29 February 1900", "x-1900-02-29"],
|
|
57
|
+
["32 October", "x-2026-10-32"],
|
|
58
|
+
["a two-digit year", "x-26-10-12"],
|
|
59
|
+
])("not with %s: %s", (_why, name) => {
|
|
60
|
+
expect(isStudyName(name)).toBe(false);
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
describe("suggestedStudyName", () => {
|
|
65
|
+
it("a name that is a study's once lowercased and hyphenated, as it is", () => {
|
|
66
|
+
expect(suggestedStudyName("Human_Labels_2026_10")).toBe(
|
|
67
|
+
"human-labels-2026-10",
|
|
68
|
+
);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("a name with no date: its slug, and the date to fill in", () => {
|
|
72
|
+
expect(suggestedStudyName("Old Baseline")).toBe("old-baseline-YYYY-MM-DD");
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
it("a name with a date that is not one: the slug without it", () => {
|
|
76
|
+
expect(suggestedStudyName("results-2026-13-01")).toBe("results-YYYY-MM-DD");
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it("a name with no slug at all: a placeholder for one", () => {
|
|
80
|
+
expect(suggestedStudyName("2026-10")).toBe("<slug>-YYYY-MM-DD");
|
|
81
|
+
expect(suggestedStudyName("__")).toBe("<slug>-YYYY-MM-DD");
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
describe("judgeRepro — silent", () => {
|
|
86
|
+
it("on the layout: the index, .gitignore, tools/, data/ and dated studies with a README.md", () => {
|
|
87
|
+
expect(
|
|
88
|
+
judgeRepro([
|
|
89
|
+
file("README.md"),
|
|
90
|
+
file(".gitignore"),
|
|
91
|
+
dir("tools", false),
|
|
92
|
+
dir("data", false),
|
|
93
|
+
dir("ablation-2026-10-12"),
|
|
94
|
+
dir("human-labels-2026-10"),
|
|
95
|
+
]),
|
|
96
|
+
).toEqual([]);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it("on what interpreters, package managers and the system leave, file or folder", () => {
|
|
100
|
+
expect(
|
|
101
|
+
judgeRepro([
|
|
102
|
+
dir("__pycache__", false),
|
|
103
|
+
dir(".pytest_cache", false),
|
|
104
|
+
dir(".mypy_cache", false),
|
|
105
|
+
dir(".ruff_cache", false),
|
|
106
|
+
dir(".ipynb_checkpoints", false),
|
|
107
|
+
dir("node_modules", false),
|
|
108
|
+
dir(".venv", false),
|
|
109
|
+
file(".DS_Store"),
|
|
110
|
+
file("Thumbs.db"),
|
|
111
|
+
]),
|
|
112
|
+
).toEqual([]);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe("judgeRepro — reports, one finding per entry", () => {
|
|
117
|
+
it("a loose file, by kind: script, tool manifest, data, note, anything else", () => {
|
|
118
|
+
expect(
|
|
119
|
+
judgeRepro([
|
|
120
|
+
file("paper_numbers.py"),
|
|
121
|
+
file("check-anon.sh"),
|
|
122
|
+
file("harness.MJS"),
|
|
123
|
+
file("requirements.txt"),
|
|
124
|
+
file("Makefile"),
|
|
125
|
+
file("numbers.tsv"),
|
|
126
|
+
file("runs.jsonl"),
|
|
127
|
+
file("notes.md"),
|
|
128
|
+
file("figure.pdf"),
|
|
129
|
+
file("LICENSE"),
|
|
130
|
+
]).map((f) => [f.path, f.problem, f.cure]),
|
|
131
|
+
).toEqual([
|
|
132
|
+
["repro/paper_numbers.py", "stray", SCRIPT_CURE],
|
|
133
|
+
["repro/check-anon.sh", "stray", SCRIPT_CURE],
|
|
134
|
+
["repro/harness.MJS", "stray", SCRIPT_CURE],
|
|
135
|
+
["repro/requirements.txt", "stray", SCRIPT_CURE],
|
|
136
|
+
["repro/Makefile", "stray", SCRIPT_CURE],
|
|
137
|
+
["repro/numbers.tsv", "stray", DATA_CURE],
|
|
138
|
+
["repro/runs.jsonl", "stray", DATA_CURE],
|
|
139
|
+
["repro/notes.md", "stray", NOTE_CURE],
|
|
140
|
+
["repro/figure.pdf", "stray", FILE_CURE_BY_KIND],
|
|
141
|
+
["repro/LICENSE", "stray", FILE_CURE_BY_KIND],
|
|
142
|
+
]);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("a file paperlint once kept at the top: where it is read now, with the command", () => {
|
|
146
|
+
expect(
|
|
147
|
+
judgeRepro(MOVED_FILES.map((m) => file(m.name))).map((f) => f.cure),
|
|
148
|
+
).toEqual(MOVED_FILES.map((m) => m.cure));
|
|
149
|
+
expect(MOVED_FILES[0]?.cure).toContain(
|
|
150
|
+
"git mv repro/references-cache.json repro/data/",
|
|
151
|
+
);
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
describe("judgeRepro — reports a folder, one finding each", () => {
|
|
156
|
+
it("an undated folder: rename it as a study, with its first commit's date as the default", () => {
|
|
157
|
+
expect(judgeRepro([dir("Old_Baseline")])).toEqual([
|
|
158
|
+
{
|
|
159
|
+
path: "repro/Old_Baseline/",
|
|
160
|
+
problem: "stray",
|
|
161
|
+
cure: "rename it to `old-baseline-YYYY-MM-DD`, one folder per study or spike, named for what it is and the date it began — the date of its first commit is a fine default (`git log --reverse --format=%as -- repro/Old_Baseline | head -1`, run in the paper folder) — and give it a README.md; if it holds scripts or data shared across the paper instead, move them into repro/tools/ or repro/data/",
|
|
162
|
+
},
|
|
163
|
+
]);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
it("a study folder without its README.md", () => {
|
|
167
|
+
expect(judgeRepro([dir("ablation-2026-10-12", false)])).toEqual([
|
|
168
|
+
{
|
|
169
|
+
path: "repro/ablation-2026-10-12/",
|
|
170
|
+
problem: "no-readme",
|
|
171
|
+
cure: "add repro/ablation-2026-10-12/README.md — what the study asks, how to run it, what it found — and a line for it in repro/README.md",
|
|
172
|
+
},
|
|
173
|
+
]);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it("a name allowed for a folder, as a file, and the other way round", () => {
|
|
177
|
+
expect(
|
|
178
|
+
judgeRepro([
|
|
179
|
+
file("tools"),
|
|
180
|
+
file("ablation-2026-10-12"),
|
|
181
|
+
dir("README.md"),
|
|
182
|
+
]).map((f) => f.path),
|
|
183
|
+
).toEqual(["repro/tools", "repro/ablation-2026-10-12", "repro/README.md/"]);
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
describe("reproMessage — stands alone", () => {
|
|
188
|
+
it("a stray entry: what it is, where it goes, what repro/ holds, the rule's page", () => {
|
|
189
|
+
expect(
|
|
190
|
+
reproMessage({
|
|
191
|
+
path: "repro/run.py",
|
|
192
|
+
problem: "stray",
|
|
193
|
+
cure: SCRIPT_CURE,
|
|
194
|
+
}),
|
|
195
|
+
).toBe(
|
|
196
|
+
[
|
|
197
|
+
`\`repro/run.py\` is loose at the top of the paper's repro/ folder, which holds only an index, the shared tools and data, and one dated folder per study. Where it goes: ${SCRIPT_CURE}.`,
|
|
198
|
+
"What repro/ holds (what interpreters and package managers leave there, like __pycache__/ and node_modules/, is never reported):",
|
|
199
|
+
"repro/",
|
|
200
|
+
"├── README.md ← the index: one line per study",
|
|
201
|
+
"├── .gitignore ← keeps the scripts' byproducts out of git",
|
|
202
|
+
"├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)",
|
|
203
|
+
"├── data/ ← shared data (tables, caches)",
|
|
204
|
+
"└── <slug>-YYYY-MM-DD/ ← one folder per study or spike (<slug>-YYYY-MM for one that ran over weeks)",
|
|
205
|
+
" └── README.md ← required: what it asks, how to run it, what it found",
|
|
206
|
+
"Rule repro/allowed-files-only: https://github.com/zernie/paperlint/blob/main/docs/rules/repro/allowed-files-only.md",
|
|
207
|
+
].join("\n"),
|
|
208
|
+
);
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
it("a study without its README.md: what to add", () => {
|
|
212
|
+
expect(
|
|
213
|
+
reproMessage({
|
|
214
|
+
path: "repro/x-2026-10/",
|
|
215
|
+
problem: "no-readme",
|
|
216
|
+
cure: "add it",
|
|
217
|
+
}),
|
|
218
|
+
).toMatch(
|
|
219
|
+
/^`repro\/x-2026-10\/` is a study folder without a README\.md, and every study in repro\/ has one\. What to do: add it\.\nWhat repro\/ holds/,
|
|
220
|
+
);
|
|
221
|
+
});
|
|
222
|
+
});
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE LAYOUT OF A PAPER'S `repro/` — what may stand at its top, and nothing else.
|
|
3
|
+
*
|
|
4
|
+
* `repro/` is where a paper's evidence lives, and it fills up the way a desk does: one real paper had
|
|
5
|
+
* 120 entries at its top, about 70 study folders (some with no date) and about 50 loose scripts, data
|
|
6
|
+
* files and notes. Nobody could say which script produces which number, which study is current, or
|
|
7
|
+
* where a new run goes. The layout gives each kind of material one place:
|
|
8
|
+
*
|
|
9
|
+
* repro/
|
|
10
|
+
* ├── README.md the index: one line per study
|
|
11
|
+
* ├── <slug>-YYYY-MM-DD/ one folder per study or spike, with a README.md inside
|
|
12
|
+
* ├── tools/ scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
13
|
+
* └── data/ shared data (tables, caches)
|
|
14
|
+
*
|
|
15
|
+
* plus `.gitignore`. A loose note belongs in its study's folder or in the paper's `research/`.
|
|
16
|
+
*
|
|
17
|
+
* ── THE DATE IN A STUDY'S NAME ────────────────────────────────────────────────────
|
|
18
|
+
* `<slug>-YYYY-MM-DD` is the day the study began. `<slug>-YYYY-MM` is accepted too: a study that ran
|
|
19
|
+
* over weeks (labelling, a reading of a corpus) has a month as its honest date, and both forms sort
|
|
20
|
+
* by time. The date must be a real one (no month 13, no 30 February). The slug is lowercase letters
|
|
21
|
+
* and digits in words joined by single hyphens.
|
|
22
|
+
*
|
|
23
|
+
* ── NEVER REPORTED ───────────────────────────────────────────────────────────────
|
|
24
|
+
* What interpreters, package managers and the operating system leave in any folder they run in
|
|
25
|
+
* (`__pycache__/`, `node_modules/`, `.DS_Store`, …): the scripts' byproducts, not the author's.
|
|
26
|
+
*
|
|
27
|
+
* Pure: the caller lists the folder and says which study folders hold a `README.md`.
|
|
28
|
+
*/
|
|
29
|
+
import { extname } from "node:path";
|
|
30
|
+
import { leaf, renderTree, type TreeNode } from "./layout-tree.ts";
|
|
31
|
+
|
|
32
|
+
/** The folder this layout governs, at the paper folder's root. */
|
|
33
|
+
export const REPRO_DIR = "repro";
|
|
34
|
+
|
|
35
|
+
/** The index at the top of `repro/`, and the file every study folder holds. */
|
|
36
|
+
export const README = "README.md";
|
|
37
|
+
|
|
38
|
+
/** The files `repro/` may hold at its top, by exact name. */
|
|
39
|
+
export const REPRO_FILE_NOTES: readonly {
|
|
40
|
+
readonly name: string;
|
|
41
|
+
readonly note: string;
|
|
42
|
+
}[] = [
|
|
43
|
+
{ name: README, note: "the index: one line per study" },
|
|
44
|
+
{ name: ".gitignore", note: "keeps the scripts' byproducts out of git" },
|
|
45
|
+
];
|
|
46
|
+
export const REPRO_FILES: readonly string[] = REPRO_FILE_NOTES.map(
|
|
47
|
+
(f) => f.name,
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
/** The shared folders `repro/` may hold besides its study folders. */
|
|
51
|
+
export const TOOLS_DIR = "tools";
|
|
52
|
+
export const DATA_DIR = "data";
|
|
53
|
+
export const REPRO_DIR_NOTES: readonly {
|
|
54
|
+
readonly name: string;
|
|
55
|
+
readonly note: string;
|
|
56
|
+
}[] = [
|
|
57
|
+
{
|
|
58
|
+
name: TOOLS_DIR,
|
|
59
|
+
note: "scripts used across the paper (numbers, figures, anonymity check, harnesses)",
|
|
60
|
+
},
|
|
61
|
+
{ name: DATA_DIR, note: "shared data (tables, caches)" },
|
|
62
|
+
];
|
|
63
|
+
export const REPRO_DIRS: readonly string[] = REPRO_DIR_NOTES.map((d) => d.name);
|
|
64
|
+
|
|
65
|
+
/** The study folder as the layout draws it: the name pattern, and the README.md it must hold. */
|
|
66
|
+
const STUDY_NODE: TreeNode = {
|
|
67
|
+
name: "<slug>-YYYY-MM-DD/",
|
|
68
|
+
note: "one folder per study or spike (<slug>-YYYY-MM for one that ran over weeks)",
|
|
69
|
+
children: [
|
|
70
|
+
leaf(README, "required: what it asks, how to run it, what it found"),
|
|
71
|
+
],
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/** The layout of `repro/`, entry by entry. */
|
|
75
|
+
export const REPRO_NODES: readonly TreeNode[] = [
|
|
76
|
+
...REPRO_FILE_NOTES.map((f) => leaf(f.name, f.note)),
|
|
77
|
+
...REPRO_DIR_NOTES.map((d) => leaf(`${d.name}/`, d.note)),
|
|
78
|
+
STUDY_NODE,
|
|
79
|
+
];
|
|
80
|
+
|
|
81
|
+
/** The drawing of `repro/`'s layout: the rule's message, the hook's refusal, the rule's page. */
|
|
82
|
+
export const REPRO_LAYOUT_TREE: string = renderTree(
|
|
83
|
+
`${REPRO_DIR}/`,
|
|
84
|
+
REPRO_NODES,
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* What interpreters, package managers, test runners and the operating system write into the folder
|
|
89
|
+
* they run in. Never reported, whether a file or a folder.
|
|
90
|
+
*/
|
|
91
|
+
export const REPRO_BYPRODUCTS: readonly string[] = [
|
|
92
|
+
"__pycache__",
|
|
93
|
+
".pytest_cache",
|
|
94
|
+
".mypy_cache",
|
|
95
|
+
".ruff_cache",
|
|
96
|
+
".ipynb_checkpoints",
|
|
97
|
+
"node_modules",
|
|
98
|
+
".venv",
|
|
99
|
+
".DS_Store",
|
|
100
|
+
"Thumbs.db",
|
|
101
|
+
];
|
|
102
|
+
|
|
103
|
+
/** A study's name: a slug, then the year, the month, and optionally the day it began. */
|
|
104
|
+
const STUDY_NAME = /^([a-z0-9]+(?:-[a-z0-9]+)*)-(\d{4})-(\d{2})(?:-(\d{2}))?$/u;
|
|
105
|
+
|
|
106
|
+
const isLeapYear = (y: number): boolean =>
|
|
107
|
+
(y % 4 === 0 && y % 100 !== 0) || y % 400 === 0;
|
|
108
|
+
|
|
109
|
+
/** The days in month `m` (1–12) of year `y`. */
|
|
110
|
+
const daysIn = (y: number, m: number): number =>
|
|
111
|
+
m === 2 ? (isLeapYear(y) ? 29 : 28) : [4, 6, 9, 11].includes(m) ? 30 : 31;
|
|
112
|
+
|
|
113
|
+
/** Whether `name` is a study folder's: `<slug>-YYYY-MM-DD` or `<slug>-YYYY-MM`, a real date. */
|
|
114
|
+
export function isStudyName(name: string): boolean {
|
|
115
|
+
const m = STUDY_NAME.exec(name);
|
|
116
|
+
if (m === null) return false;
|
|
117
|
+
const year = Number(m[2]);
|
|
118
|
+
const month = Number(m[3]);
|
|
119
|
+
const day = m[4] === undefined ? 1 : Number(m[4]);
|
|
120
|
+
return month >= 1 && month <= 12 && day >= 1 && day <= daysIn(year, month);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* The study name to suggest for a folder named `name`: its name as a slug, with the date to fill in.
|
|
125
|
+
* A name that becomes a study name once lowercased and hyphenated is suggested as it is.
|
|
126
|
+
*/
|
|
127
|
+
export function suggestedStudyName(name: string): string {
|
|
128
|
+
const slug = name
|
|
129
|
+
.toLowerCase()
|
|
130
|
+
.replace(/[^a-z0-9]+/gu, "-")
|
|
131
|
+
.replace(/^-+|-+$/gu, "");
|
|
132
|
+
if (isStudyName(slug)) return slug;
|
|
133
|
+
const base = slug.replace(/(?:^|-)\d{4}(?:-\d{1,2}){0,2}$/u, "");
|
|
134
|
+
return `${base === "" ? "<slug>" : base}-YYYY-MM-DD`;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** The extensions of a script: what a study or the paper's build runs. */
|
|
138
|
+
export const SCRIPT_EXTENSIONS: readonly string[] = [
|
|
139
|
+
".py",
|
|
140
|
+
".ipynb",
|
|
141
|
+
".sh",
|
|
142
|
+
".bash",
|
|
143
|
+
".zsh",
|
|
144
|
+
".js",
|
|
145
|
+
".mjs",
|
|
146
|
+
".cjs",
|
|
147
|
+
".ts",
|
|
148
|
+
".mts",
|
|
149
|
+
".cts",
|
|
150
|
+
".r",
|
|
151
|
+
".jl",
|
|
152
|
+
".rb",
|
|
153
|
+
".pl",
|
|
154
|
+
".awk",
|
|
155
|
+
".sql",
|
|
156
|
+
];
|
|
157
|
+
|
|
158
|
+
/** Names without a script's extension that still say how the tools run. */
|
|
159
|
+
export const TOOL_FILES: readonly string[] = [
|
|
160
|
+
"Makefile",
|
|
161
|
+
"Dockerfile",
|
|
162
|
+
"requirements.txt",
|
|
163
|
+
"pyproject.toml",
|
|
164
|
+
"uv.lock",
|
|
165
|
+
"package.json",
|
|
166
|
+
"package-lock.json",
|
|
167
|
+
];
|
|
168
|
+
|
|
169
|
+
/** The extensions of data: tables, records, caches, archives, raw output. */
|
|
170
|
+
export const DATA_EXTENSIONS: readonly string[] = [
|
|
171
|
+
".tsv",
|
|
172
|
+
".csv",
|
|
173
|
+
".json",
|
|
174
|
+
".jsonl",
|
|
175
|
+
".ndjson",
|
|
176
|
+
".yaml",
|
|
177
|
+
".yml",
|
|
178
|
+
".xml",
|
|
179
|
+
".txt",
|
|
180
|
+
".log",
|
|
181
|
+
".parquet",
|
|
182
|
+
".sqlite",
|
|
183
|
+
".db",
|
|
184
|
+
".xlsx",
|
|
185
|
+
".npy",
|
|
186
|
+
".npz",
|
|
187
|
+
".pkl",
|
|
188
|
+
".gz",
|
|
189
|
+
".zip",
|
|
190
|
+
".tar",
|
|
191
|
+
];
|
|
192
|
+
|
|
193
|
+
/** The extensions of a note. */
|
|
194
|
+
export const NOTE_EXTENSIONS: readonly string[] = [".md", ".markdown"];
|
|
195
|
+
|
|
196
|
+
/** Where a script goes. */
|
|
197
|
+
export const SCRIPT_CURE =
|
|
198
|
+
"repro/tools/ — the scripts used across the paper (numbers, figures, the anonymity check, harnesses); a script only one study runs goes in that study's folder instead";
|
|
199
|
+
|
|
200
|
+
/** Where data goes. */
|
|
201
|
+
export const DATA_CURE =
|
|
202
|
+
"repro/data/ — the data shared across the paper (tables, caches); what one study's run produced goes in that study's folder instead";
|
|
203
|
+
|
|
204
|
+
/** Where a note goes. */
|
|
205
|
+
export const NOTE_CURE =
|
|
206
|
+
"the dated folder of the study it is about (repro/<slug>-YYYY-MM-DD/), or the paper's research/ when it is research or a decision rather than the record of a run";
|
|
207
|
+
|
|
208
|
+
/** Where a file of any other kind goes. */
|
|
209
|
+
export const FILE_CURE_BY_KIND =
|
|
210
|
+
"decide by what it is — a script → repro/tools/; data → repro/data/; a note → the dated folder of its study, or the paper's research/; what one study produced → that study's folder";
|
|
211
|
+
|
|
212
|
+
/** The files paperlint itself once kept at the top of `repro/`, each with where it is read now. */
|
|
213
|
+
export const MOVED_FILES: readonly {
|
|
214
|
+
readonly name: string;
|
|
215
|
+
readonly cure: string;
|
|
216
|
+
}[] = [
|
|
217
|
+
{
|
|
218
|
+
name: "references-cache.json",
|
|
219
|
+
cure: "repro/data/references-cache.json, where `paperlint build` reads and writes the citation services' answers (`git mv repro/references-cache.json repro/data/`); until it is moved, the next build asks every service again",
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
name: "refs-cache.json",
|
|
223
|
+
cure: "repro/data/refs-cache.json, where the paper-pipeline's extract-ref-facts script keeps its raw answers (`git mv repro/refs-cache.json repro/data/`)",
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
name: "generated-code-grandfathered.txt",
|
|
227
|
+
cure: "repro/tools/generated-code-grandfathered.txt, where the paper-pipeline's generated-code check reads it (`git mv repro/generated-code-grandfathered.txt repro/tools/`)",
|
|
228
|
+
},
|
|
229
|
+
];
|
|
230
|
+
|
|
231
|
+
/** Where a stray file goes: a file paperlint moved, else by the kind its name says. */
|
|
232
|
+
function fileCure(name: string): string {
|
|
233
|
+
const moved = MOVED_FILES.find((f) => f.name === name);
|
|
234
|
+
if (moved !== undefined) return moved.cure;
|
|
235
|
+
const ext = extname(name).toLowerCase();
|
|
236
|
+
if (TOOL_FILES.includes(name) || SCRIPT_EXTENSIONS.includes(ext))
|
|
237
|
+
return SCRIPT_CURE;
|
|
238
|
+
if (DATA_EXTENSIONS.includes(ext)) return DATA_CURE;
|
|
239
|
+
if (NOTE_EXTENSIONS.includes(ext)) return NOTE_CURE;
|
|
240
|
+
return FILE_CURE_BY_KIND;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Where a folder with no study name goes: renamed as a study, or emptied into the shared folders. */
|
|
244
|
+
const folderCure = (name: string): string =>
|
|
245
|
+
`rename it to \`${suggestedStudyName(name)}\`, one folder per study or spike, named for what it is and the date it began — the date of its first commit is a fine default (\`git log --reverse --format=%as -- repro/${name} | head -1\`, run in the paper folder) — and give it a README.md; if it holds scripts or data shared across the paper instead, move them into repro/tools/ or repro/data/`;
|
|
246
|
+
|
|
247
|
+
/** An entry at the top of `repro/`: a file or a folder, and for a folder whether it holds a README.md. */
|
|
248
|
+
export interface ReproEntry {
|
|
249
|
+
readonly name: string;
|
|
250
|
+
readonly kind: "file" | "dir";
|
|
251
|
+
readonly hasReadme: boolean;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** What is wrong with an entry at the top of `repro/`. */
|
|
255
|
+
export interface ReproFinding {
|
|
256
|
+
/** The entry's path from the paper folder; a folder's ends in a slash. */
|
|
257
|
+
readonly path: string;
|
|
258
|
+
/** `stray`: not part of the layout. `no-readme`: a study folder without its README.md. */
|
|
259
|
+
readonly problem: "stray" | "no-readme";
|
|
260
|
+
/** Where it goes, or what to add. */
|
|
261
|
+
readonly cure: string;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
const isAllowed = (e: ReproEntry): boolean =>
|
|
265
|
+
REPRO_BYPRODUCTS.includes(e.name) ||
|
|
266
|
+
(e.kind === "file"
|
|
267
|
+
? REPRO_FILES.includes(e.name)
|
|
268
|
+
: REPRO_DIRS.includes(e.name) || isStudyName(e.name));
|
|
269
|
+
|
|
270
|
+
/** The one finding for `e`, or null when it is in its place. */
|
|
271
|
+
function judgeEntry(e: ReproEntry): ReproFinding | null {
|
|
272
|
+
const path = `${REPRO_DIR}/${e.name}${e.kind === "dir" ? "/" : ""}`;
|
|
273
|
+
if (!isAllowed(e))
|
|
274
|
+
return {
|
|
275
|
+
path,
|
|
276
|
+
problem: "stray",
|
|
277
|
+
cure: e.kind === "dir" ? folderCure(e.name) : fileCure(e.name),
|
|
278
|
+
};
|
|
279
|
+
if (e.kind === "dir" && isStudyName(e.name) && !e.hasReadme)
|
|
280
|
+
return {
|
|
281
|
+
path,
|
|
282
|
+
problem: "no-readme",
|
|
283
|
+
cure: `add ${path}${README} — what the study asks, how to run it, what it found — and a line for it in repro/${README}`,
|
|
284
|
+
};
|
|
285
|
+
return null;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** The entries at the top of `repro/` that are out of place, at most one finding each, in order. Pure. */
|
|
289
|
+
export function judgeRepro(
|
|
290
|
+
entries: readonly ReproEntry[],
|
|
291
|
+
): readonly ReproFinding[] {
|
|
292
|
+
return entries.map(judgeEntry).filter((f): f is ReproFinding => f !== null);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/**
|
|
296
|
+
* The one text a finding is reported with, by the rule `repro/allowed-files-only` and by the hook that fires
|
|
297
|
+
* when such an entry is written. It stands alone: what is wrong, where the entry goes, and what
|
|
298
|
+
* `repro/` holds, for a reader who knows nothing of paperlint.
|
|
299
|
+
*/
|
|
300
|
+
export function reproMessage(f: ReproFinding): string {
|
|
301
|
+
const head =
|
|
302
|
+
f.problem === "stray"
|
|
303
|
+
? `\`${f.path}\` is loose at the top of the paper's repro/ folder, which holds only an index, the shared tools and data, and one dated folder per study. Where it goes: ${f.cure}.`
|
|
304
|
+
: `\`${f.path}\` is a study folder without a ${README}, and every study in repro/ has one. What to do: ${f.cure}.`;
|
|
305
|
+
return [
|
|
306
|
+
head,
|
|
307
|
+
"What repro/ holds (what interpreters and package managers leave there, like __pycache__/ and node_modules/, is never reported):",
|
|
308
|
+
REPRO_LAYOUT_TREE,
|
|
309
|
+
`Rule repro/allowed-files-only: https://github.com/zernie/paperlint/blob/main/docs/rules/repro/allowed-files-only.md`,
|
|
310
|
+
].join("\n");
|
|
311
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WHAT `paperlint new` PUTS IN A NEW PAPER FOLDER BESIDE ITS THREE FILES — the layout, visible from the
|
|
3
|
+
* first commit.
|
|
4
|
+
*
|
|
5
|
+
* The rules `paper/allowed-files-only` and `repro/allowed-files-only` say where every kind of material goes. A
|
|
6
|
+
* paper that starts with three files shows none of it, and the first reviewer's report, the first
|
|
7
|
+
* figure and the first script each land loose at the root. So a new folder already holds the places.
|
|
8
|
+
*
|
|
9
|
+
* ── ONE SOURCE OF TRUTH ──────────────────────────────────────────────────────────
|
|
10
|
+
* Nothing here lists a folder by hand. The folders are `ROOT_DIR_NOTES` (paper-layout.ts) minus the
|
|
11
|
+
* ones named in `NOT_SCAFFOLDED`, each with its reason; `repro/` gets its own shared folders from
|
|
12
|
+
* `REPRO_DIR_NOTES`. A folder added to the layout is scaffolded unless someone writes down why not,
|
|
13
|
+
* and a test fails on an excluded name the layout no longer has.
|
|
14
|
+
*
|
|
15
|
+
* ── `.gitkeep` ───────────────────────────────────────────────────────────────────
|
|
16
|
+
* Git keeps files, not folders: an empty folder is not in the first commit, which is the point of
|
|
17
|
+
* scaffolding. A folder that holds nothing else gets a `.gitkeep`; a folder that holds a real file
|
|
18
|
+
* (`repro/` with its README.md) gets none. `.gitkeep` is the convention, not a paperlint file, and no
|
|
19
|
+
* rule reads it: `paper/stages` looks for `*.pdf` in `versions/`, the review and sibling rules lint
|
|
20
|
+
* `*.md`, `repro/allowed-files-only` judges the top of `repro/` only, and neither `tools/` nor `data/` is
|
|
21
|
+
* listed by any rule.
|
|
22
|
+
*
|
|
23
|
+
* Pure: a list of paths and texts, written by `new-paper.ts`.
|
|
24
|
+
*/
|
|
25
|
+
import { ROOT_DIR_NOTES } from "./paper-layout.ts";
|
|
26
|
+
import {
|
|
27
|
+
DATA_DIR,
|
|
28
|
+
README,
|
|
29
|
+
REPRO_DIR,
|
|
30
|
+
REPRO_DIR_NOTES,
|
|
31
|
+
TOOLS_DIR,
|
|
32
|
+
} from "./repro-layout.ts";
|
|
33
|
+
|
|
34
|
+
/** The placeholder git needs to keep an otherwise empty folder. */
|
|
35
|
+
export const GITKEEP = ".gitkeep";
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The folders of the layout that `paperlint new` does not create, each with why. A folder here is
|
|
39
|
+
* allowed at the root all the same; it appears when the work needs it.
|
|
40
|
+
*/
|
|
41
|
+
export const NOT_SCAFFOLDED: readonly {
|
|
42
|
+
readonly name: string;
|
|
43
|
+
readonly why: string;
|
|
44
|
+
}[] = [
|
|
45
|
+
{
|
|
46
|
+
name: "_build",
|
|
47
|
+
why: "`paperlint build` writes it, and an empty one would claim a build that never ran",
|
|
48
|
+
},
|
|
49
|
+
{
|
|
50
|
+
name: "siblings",
|
|
51
|
+
why: "a card is written when a close competitor turns up (skill analyze-sibling-paper, which also writes the folder's README.md index); a paper may have none, and an empty folder reads as an unfinished analysis",
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
name: "sections",
|
|
55
|
+
why: "the body is split only when it outgrows `paper.tex`; the stub is one file, and an empty `sections/` suggests the split is expected",
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
name: "talk",
|
|
59
|
+
why: "only a paper that declares a talk (`talk` in paperlint.json) has one, and its folder is the declared `talk.dir`, which need not be `talk/`",
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
name: "extra",
|
|
63
|
+
why: "the escape hatch for material nothing else takes; it is never judged, so it has no place in a layout meant to be filled by kind",
|
|
64
|
+
},
|
|
65
|
+
];
|
|
66
|
+
|
|
67
|
+
/** The folders at the paper's root that a new paper holds from the first commit, in layout order. */
|
|
68
|
+
export const SCAFFOLD_DIRS: readonly string[] = ROOT_DIR_NOTES.map(
|
|
69
|
+
(d) => d.name,
|
|
70
|
+
).filter((name) => !NOT_SCAFFOLDED.some((n) => n.name === name));
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The stub index of `repro/`: it makes the folder non-empty (no `.gitkeep`) and says what the rule
|
|
74
|
+
* asks of it. A line per study is added when the first study is.
|
|
75
|
+
*/
|
|
76
|
+
export const REPRO_INDEX_STUB = `# Reproduction index
|
|
77
|
+
|
|
78
|
+
One line per study: \`<slug>-YYYY-MM-DD/\` — what it asks and what it found. Scripts used across the
|
|
79
|
+
paper go in \`${TOOLS_DIR}/\`, shared data in \`${DATA_DIR}/\`.
|
|
80
|
+
`;
|
|
81
|
+
|
|
82
|
+
/** A file `paperlint new` writes into a new paper folder, with its path from that folder. */
|
|
83
|
+
export interface ScaffoldFile {
|
|
84
|
+
readonly path: string;
|
|
85
|
+
readonly text: string;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Every file of the layout scaffold, relative to the paper folder, in a fixed order: a `.gitkeep` in
|
|
90
|
+
* each empty folder, and `repro/README.md` where `repro/` would otherwise hold only folders.
|
|
91
|
+
*/
|
|
92
|
+
export const SCAFFOLD_FILES: readonly ScaffoldFile[] = [
|
|
93
|
+
...SCAFFOLD_DIRS.flatMap((dir): readonly ScaffoldFile[] =>
|
|
94
|
+
dir === REPRO_DIR
|
|
95
|
+
? [
|
|
96
|
+
{ path: `${REPRO_DIR}/${README}`, text: REPRO_INDEX_STUB },
|
|
97
|
+
...REPRO_DIR_NOTES.map((d) => ({
|
|
98
|
+
path: `${REPRO_DIR}/${d.name}/${GITKEEP}`,
|
|
99
|
+
text: "",
|
|
100
|
+
})),
|
|
101
|
+
]
|
|
102
|
+
: [{ path: `${dir}/${GITKEEP}`, text: "" }],
|
|
103
|
+
),
|
|
104
|
+
];
|