paperlint 10.1.1 → 10.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/adapters/study-registry/index.d.ts +357 -0
- package/dist/adapters/study-registry/index.d.ts.map +1 -0
- package/dist/adapters/study-registry/index.js +221 -0
- package/dist/adapters/study-registry/index.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +42 -2
- package/dist/cli.js.map +1 -1
- package/dist/domain/repro-layout.d.ts +12 -4
- package/dist/domain/repro-layout.d.ts.map +1 -1
- package/dist/domain/repro-layout.js +49 -18
- package/dist/domain/repro-layout.js.map +1 -1
- package/dist/domain/scaffold-layout.d.ts +4 -3
- package/dist/domain/scaffold-layout.d.ts.map +1 -1
- package/dist/domain/scaffold-layout.js +7 -4
- package/dist/domain/scaffold-layout.js.map +1 -1
- package/dist/domain/study-registry.d.ts +372 -0
- package/dist/domain/study-registry.d.ts.map +1 -0
- package/dist/domain/study-registry.js +593 -0
- package/dist/domain/study-registry.js.map +1 -0
- package/dist/ports/study-registry-codec.d.ts +15 -0
- package/dist/ports/study-registry-codec.d.ts.map +1 -0
- package/dist/ports/study-registry-codec.js +2 -0
- package/dist/ports/study-registry-codec.js.map +1 -0
- package/dist/repro-layout-rule.d.ts.map +1 -1
- package/dist/repro-layout-rule.js +4 -3
- package/dist/repro-layout-rule.js.map +1 -1
- package/dist/studies.d.ts +27 -0
- package/dist/studies.d.ts.map +1 -0
- package/dist/studies.js +163 -0
- package/dist/studies.js.map +1 -0
- package/dist/study-registry-rule.d.ts +24 -0
- package/dist/study-registry-rule.d.ts.map +1 -0
- package/dist/study-registry-rule.js +145 -0
- package/dist/study-registry-rule.js.map +1 -0
- package/docs/design/study-registry.md +246 -0
- package/docs/layout.md +51 -46
- package/docs/rules/paper/allowed-files-only.md +3 -2
- package/docs/rules/repro/allowed-files-only.md +40 -30
- package/docs/rules/repro/study-plan-name.md +70 -0
- package/docs/rules/repro/study-registered.md +320 -0
- package/docs/rules/study/declare-sample.md +1 -1
- package/docs/rules.md +53 -51
- package/package.json +1 -1
- package/skills/build-benchmark/SKILL.md +39 -8
- package/skills/build-benchmark/SKILL.md.spec.ts +38 -7
- package/skills/build-label-page/SKILL.md +14 -9
- package/skills/build-label-page/SKILL.md.spec.ts +13 -8
- package/skills/paper-pipeline/references/artifact-checklist.md +2 -1
- package/skills/paper-pipeline/references/pipeline-status-template.md +5 -3
- package/skills/run-agent-grid/SKILL.md +13 -1
- package/skills/run-agent-grid/SKILL.md.spec.ts +12 -0
- package/src/adapters/study-registry/index.test.ts +484 -0
- package/src/adapters/study-registry/index.ts +328 -0
- package/src/cli.ts +53 -2
- package/src/domain/paper-layout.test.ts +1 -1
- package/src/domain/repro-layout.test.ts +18 -9
- package/src/domain/repro-layout.ts +58 -18
- package/src/domain/scaffold-layout.ts +7 -4
- package/src/domain/study-measurements.test.ts +629 -0
- package/src/domain/study-registry.test.ts +666 -0
- package/src/domain/study-registry.ts +1044 -0
- package/src/folder-layout-hook.test.ts +10 -0
- package/src/new-paper.test.ts +29 -2
- package/src/ports/study-registry-codec.ts +15 -0
- package/src/repro-layout-rule.ts +4 -3
- package/src/rule-cases.test.ts +68 -0
- package/src/studies-cli.test.ts +113 -0
- package/src/studies.test.ts +359 -0
- package/src/studies.ts +268 -0
- package/src/study-registry-rule.test.ts +456 -0
- package/src/study-registry-rule.ts +239 -0
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The schema of `repro/studies.json` — the one place its shape is checked (docs/design/study-registry.md).
|
|
3
|
+
* The text is parsed once, here, into the domain's `StudyRegistry`; whatever reads a registry after it
|
|
4
|
+
* never re-validates.
|
|
5
|
+
*
|
|
6
|
+
* A row is a loose object: a paper merges its own measurement metadata into its rows (`role`, `frame`,
|
|
7
|
+
* `rules_under_test`, `outcome` …), and those fields are kept as they are. The top level is strict, so
|
|
8
|
+
* a misspelt key is an error rather than a key nobody reads. A measurement keeps its extra fields as a
|
|
9
|
+
* row does; a requirement and an approval are exactly their fields.
|
|
10
|
+
*
|
|
11
|
+
* A study's plan (`plan.json`) is parsed here too, strictly: a key of the paper's own goes in `extra`.
|
|
12
|
+
* Its texts may be empty — the template's are — and the parser says which: a plan comes out ready or
|
|
13
|
+
* a draft naming its empty texts, so nothing downstream looks at a text again. A plan still named
|
|
14
|
+
* `prereg.json` has a loose schema of its own.
|
|
15
|
+
*/
|
|
16
|
+
import { z } from "zod";
|
|
17
|
+
import {
|
|
18
|
+
IN_PAPER,
|
|
19
|
+
MEASUREMENT_ROLES,
|
|
20
|
+
PLAN_FIELDS,
|
|
21
|
+
PLAN_TEXT_FIELDS,
|
|
22
|
+
PREFIX_PATTERN,
|
|
23
|
+
STUDY_KINDS,
|
|
24
|
+
STUDY_STATUSES,
|
|
25
|
+
type Approval,
|
|
26
|
+
type AuthorCheck,
|
|
27
|
+
type Measurement,
|
|
28
|
+
type ParsedPlan,
|
|
29
|
+
type ParsedRegistry,
|
|
30
|
+
type Requirement,
|
|
31
|
+
type Study,
|
|
32
|
+
type StudyPlan,
|
|
33
|
+
type StudyRegistry,
|
|
34
|
+
} from "../../domain/study-registry.ts";
|
|
35
|
+
import type { StudyRegistryCodec } from "../../ports/study-registry-codec.ts";
|
|
36
|
+
|
|
37
|
+
const study = z.looseObject({
|
|
38
|
+
id: z.string(),
|
|
39
|
+
folder: z.string(),
|
|
40
|
+
title: z.string().min(1),
|
|
41
|
+
question: z.string(),
|
|
42
|
+
kind: z.enum(STUDY_KINDS),
|
|
43
|
+
status: z.enum(STUDY_STATUSES),
|
|
44
|
+
date: z.iso.date(),
|
|
45
|
+
result: z.string().optional(),
|
|
46
|
+
result_file: z.string().optional(),
|
|
47
|
+
superseded_by: z.string().optional(),
|
|
48
|
+
in_paper: z.enum(IN_PAPER),
|
|
49
|
+
measurement: z.string().optional(),
|
|
50
|
+
plan: z.string().optional(),
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
const words = z.string().min(1);
|
|
54
|
+
|
|
55
|
+
const requirement = z.strictObject({
|
|
56
|
+
id: words,
|
|
57
|
+
text: words,
|
|
58
|
+
date: z.iso.date(),
|
|
59
|
+
source: words,
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
const approval = z.strictObject({
|
|
63
|
+
by: words,
|
|
64
|
+
date: z.iso.date(),
|
|
65
|
+
quote: words,
|
|
66
|
+
source: words.optional(),
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const measurement = z.looseObject({
|
|
70
|
+
id: words,
|
|
71
|
+
title: words,
|
|
72
|
+
rq: z.string().optional(),
|
|
73
|
+
role: z.enum(MEASUREMENT_ROLES),
|
|
74
|
+
approval: approval.optional(),
|
|
75
|
+
requirements: z.array(z.string()).optional(),
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
const registry = z.strictObject({
|
|
79
|
+
prefix: z
|
|
80
|
+
.string()
|
|
81
|
+
.regex(PREFIX_PATTERN, "a capital letter, then capitals or digits (P1)"),
|
|
82
|
+
author: words.optional(),
|
|
83
|
+
requirements: z.array(requirement).optional(),
|
|
84
|
+
measurements: z.array(measurement).optional(),
|
|
85
|
+
studies: z.array(study),
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
const authorCheck = approval.omit({ source: true });
|
|
89
|
+
|
|
90
|
+
/** A draw: seeded, its seed part of it, or of all the population, with none. */
|
|
91
|
+
const draw = z.discriminatedUnion("kind", [
|
|
92
|
+
z.strictObject({
|
|
93
|
+
kind: z.literal("seeded"),
|
|
94
|
+
method: z.string(),
|
|
95
|
+
seed: z.union([z.string(), z.number()]),
|
|
96
|
+
}),
|
|
97
|
+
z.strictObject({ kind: z.literal("all"), method: z.string() }),
|
|
98
|
+
]);
|
|
99
|
+
|
|
100
|
+
/** The values that occur more than once in `values`, each once. */
|
|
101
|
+
const repeated = (values: readonly string[]): readonly string[] => [
|
|
102
|
+
...new Set(values.filter((v, i) => values.indexOf(v) !== i)),
|
|
103
|
+
];
|
|
104
|
+
|
|
105
|
+
/** Arms whose names are unique: a repeated name is an error naming it. */
|
|
106
|
+
const uniquelyNamed = (
|
|
107
|
+
arms: readonly { readonly name: string }[],
|
|
108
|
+
ctx: Readonly<z.core.$RefinementCtx>,
|
|
109
|
+
): void => {
|
|
110
|
+
repeated(arms.map((a) => a.name)).forEach((name) => {
|
|
111
|
+
ctx.addIssue({
|
|
112
|
+
code: "custom",
|
|
113
|
+
message: `more than one arm is named «${name}» — an arm's name is unique`,
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
/** The paper's own keys: none of them a field of the plan, which would say a thing twice. */
|
|
119
|
+
const extra = z.record(z.string(), z.unknown()).superRefine((e, ctx) => {
|
|
120
|
+
Object.keys(e)
|
|
121
|
+
.filter((k) => PLAN_FIELDS.some((f) => f === k))
|
|
122
|
+
.forEach((k) => {
|
|
123
|
+
ctx.addIssue({
|
|
124
|
+
code: "custom",
|
|
125
|
+
path: [k],
|
|
126
|
+
message: "is a field of the plan itself — set it there, not in extra",
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
/** A plan's shape: closed at every level; the paper's own keys go in `extra`. Null is the one "none". */
|
|
132
|
+
const planShape = z.strictObject({
|
|
133
|
+
question: z.string(),
|
|
134
|
+
population: z.strictObject({ source: z.string(), draw }),
|
|
135
|
+
arms: z.array(z.strictObject({ name: words })).superRefine(uniquelyNamed),
|
|
136
|
+
primary_outcome: z.string(),
|
|
137
|
+
author_check: authorCheck.nullable(),
|
|
138
|
+
extra: extra.optional(),
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* A plan under its old name, `prereg.json`, written by hand before the plan had a schema: open at
|
|
143
|
+
* every level, its own top-level keys kept in `extra`. A seed makes the draw seeded, none (or null) a
|
|
144
|
+
* draw of all; a size, and keys of its own deeper down, are not read; the file is never rewritten.
|
|
145
|
+
*/
|
|
146
|
+
const legacyAsPlan = z
|
|
147
|
+
.looseObject({
|
|
148
|
+
question: z.string(),
|
|
149
|
+
population: z.looseObject({
|
|
150
|
+
source: z.string(),
|
|
151
|
+
draw: z.looseObject({
|
|
152
|
+
method: z.string(),
|
|
153
|
+
seed: z.union([z.string(), z.number()]).nullable().optional(),
|
|
154
|
+
}),
|
|
155
|
+
}),
|
|
156
|
+
arms: z.array(z.looseObject({ name: words })).superRefine(uniquelyNamed),
|
|
157
|
+
primary_outcome: z.string(),
|
|
158
|
+
author_check: authorCheck.nullable().optional(),
|
|
159
|
+
})
|
|
160
|
+
.transform((p): StudyPlan => {
|
|
161
|
+
const {
|
|
162
|
+
question,
|
|
163
|
+
population,
|
|
164
|
+
arms,
|
|
165
|
+
primary_outcome,
|
|
166
|
+
author_check,
|
|
167
|
+
...rest
|
|
168
|
+
} = p;
|
|
169
|
+
const { method, seed } = population.draw;
|
|
170
|
+
return {
|
|
171
|
+
question,
|
|
172
|
+
population: {
|
|
173
|
+
source: population.source,
|
|
174
|
+
draw:
|
|
175
|
+
seed === undefined || seed === null
|
|
176
|
+
? { kind: "all", method }
|
|
177
|
+
: { kind: "seeded", method, seed },
|
|
178
|
+
},
|
|
179
|
+
arms: arms.map((a) => ({ name: a.name })),
|
|
180
|
+
primary_outcome,
|
|
181
|
+
author_check: author_check ?? null,
|
|
182
|
+
extra: rest,
|
|
183
|
+
};
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
/** A text that says something: more than white space. */
|
|
187
|
+
const filled = z.string().regex(/\S/u);
|
|
188
|
+
|
|
189
|
+
/** The texts a plan fills before its study runs; what fails here is what is still empty. */
|
|
190
|
+
const readyTexts = z.object({
|
|
191
|
+
question: filled,
|
|
192
|
+
population: z.object({
|
|
193
|
+
source: filled,
|
|
194
|
+
draw: z.object({ method: filled }),
|
|
195
|
+
}),
|
|
196
|
+
primary_outcome: filled,
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
/** A plan that parsed: ready, or a draft. */
|
|
200
|
+
type ReadyOrDraft = Exclude<ParsedPlan, { readonly kind: "unreadable" }>;
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* A parsed plan, ready when every required text is filled, else a draft naming the empty ones. A
|
|
204
|
+
* draft that carries the author's check is no plan: the author approved one with empty texts.
|
|
205
|
+
*/
|
|
206
|
+
const readyOrDraft = (
|
|
207
|
+
p: StudyPlan,
|
|
208
|
+
ctx: Readonly<z.core.$RefinementCtx>,
|
|
209
|
+
): ReadyOrDraft => {
|
|
210
|
+
const checked = readyTexts.safeParse(p);
|
|
211
|
+
if (checked.success) return { kind: "ready", plan: p };
|
|
212
|
+
const paths = checked.error.issues.map((i) => i.path.join("."));
|
|
213
|
+
const empty = PLAN_TEXT_FIELDS.filter((f) => paths.includes(f));
|
|
214
|
+
if (p.author_check === null)
|
|
215
|
+
return { kind: "draft", plan: { ...p, author_check: null }, empty };
|
|
216
|
+
ctx.addIssue({
|
|
217
|
+
code: "custom",
|
|
218
|
+
path: ["author_check"],
|
|
219
|
+
message: `is set, and the plan leaves ${empty.map((f) => `\`${f}\``).join(", ")} empty — the author approves a complete plan: fill them, or set author_check to null`,
|
|
220
|
+
});
|
|
221
|
+
return z.NEVER;
|
|
222
|
+
};
|
|
223
|
+
|
|
224
|
+
/** A `plan.json`: strict, then ready or draft. */
|
|
225
|
+
const plan = planShape.transform(readyOrDraft);
|
|
226
|
+
|
|
227
|
+
/** A `prereg.json`: loose, mapped onto the plan, then ready or draft. */
|
|
228
|
+
const legacyPlan = legacyAsPlan.transform(readyOrDraft);
|
|
229
|
+
|
|
230
|
+
// ── the schemas ARE the domain's types ─────────────────────────────────────────
|
|
231
|
+
// Each schema's output and its hand-written domain type are the same type, both ways, once both are
|
|
232
|
+
// made readonly all the way down: a field added to one and not the other stops the build.
|
|
233
|
+
|
|
234
|
+
/** `T`, readonly all the way down: zod's outputs are mutable, the domain's are not. */
|
|
235
|
+
type Frozen<T> = T extends readonly (infer U)[]
|
|
236
|
+
? readonly Frozen<U>[]
|
|
237
|
+
: T extends object
|
|
238
|
+
? { readonly [K in keyof T]: Frozen<T[K]> }
|
|
239
|
+
: T;
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* True when `A` and `B`, both made readonly all the way down, are one type, optional modifiers
|
|
243
|
+
* included: two generic signatures whose returns are deferred conditionals are related only when
|
|
244
|
+
* the conditionals are identical. Written out, not through an alias: an alias is compared by the
|
|
245
|
+
* variance of its parameter, which misses a dropped optional field.
|
|
246
|
+
*/
|
|
247
|
+
type Same<A, B> =
|
|
248
|
+
(<T>() => T extends Frozen<A> ? readonly [T] : 2) extends <
|
|
249
|
+
T,
|
|
250
|
+
>() => T extends Frozen<B> ? readonly [T] : 2
|
|
251
|
+
? true
|
|
252
|
+
: false;
|
|
253
|
+
|
|
254
|
+
/** Compiles only when every schema's output is its domain type. */
|
|
255
|
+
export type SchemasAreTheDomain = [
|
|
256
|
+
Same<z.output<typeof registry>, StudyRegistry>,
|
|
257
|
+
Same<z.output<typeof study>, Study>,
|
|
258
|
+
Same<z.output<typeof measurement>, Measurement>,
|
|
259
|
+
Same<z.output<typeof requirement>, Requirement>,
|
|
260
|
+
Same<z.output<typeof approval>, Approval>,
|
|
261
|
+
Same<z.output<typeof authorCheck>, AuthorCheck>,
|
|
262
|
+
Same<z.output<typeof planShape>, StudyPlan>,
|
|
263
|
+
Same<z.output<typeof legacyAsPlan>, StudyPlan>,
|
|
264
|
+
Same<z.output<typeof plan>, ReadyOrDraft>,
|
|
265
|
+
Same<z.output<typeof legacyPlan>, ReadyOrDraft>,
|
|
266
|
+
] extends true[]
|
|
267
|
+
? true
|
|
268
|
+
: never;
|
|
269
|
+
|
|
270
|
+
/** The proof is used, so the build checks it. */
|
|
271
|
+
export const SCHEMAS_ARE_THE_DOMAIN: SchemasAreTheDomain = true;
|
|
272
|
+
|
|
273
|
+
/** The JSON `text` holds, or undefined when it is not JSON. */
|
|
274
|
+
function jsonOf(text: string): unknown {
|
|
275
|
+
try {
|
|
276
|
+
return JSON.parse(text);
|
|
277
|
+
} catch {
|
|
278
|
+
return undefined;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** Every reason zod gave, each with the path it is about. */
|
|
283
|
+
const reasons = (
|
|
284
|
+
issues: readonly {
|
|
285
|
+
readonly path: readonly PropertyKey[];
|
|
286
|
+
readonly message: string;
|
|
287
|
+
}[],
|
|
288
|
+
): string =>
|
|
289
|
+
issues
|
|
290
|
+
.map(
|
|
291
|
+
(i) =>
|
|
292
|
+
`${i.path.length === 0 ? "(top)" : i.path.join(".")}: ${i.message}`,
|
|
293
|
+
)
|
|
294
|
+
.join("; ");
|
|
295
|
+
|
|
296
|
+
/** A `studies.json`'s text, parsed once at the boundary: the registry, or every reason it is not one. */
|
|
297
|
+
export function parseStudyRegistry(text: string): ParsedRegistry {
|
|
298
|
+
const json = jsonOf(text);
|
|
299
|
+
if (json === undefined) return { ok: false, why: "not JSON" };
|
|
300
|
+
const parsed = registry.safeParse(json);
|
|
301
|
+
return parsed.success
|
|
302
|
+
? { ok: true, registry: parsed.data }
|
|
303
|
+
: { ok: false, why: reasons(parsed.error.issues) };
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/** A plan's text, parsed once at the boundary by `schema`: ready, a draft, or every reason it is neither. */
|
|
307
|
+
const planParser =
|
|
308
|
+
(schema: Readonly<z.ZodType<ReadyOrDraft>>) =>
|
|
309
|
+
(text: string): ParsedPlan => {
|
|
310
|
+
const json = jsonOf(text);
|
|
311
|
+
if (json === undefined) return { kind: "unreadable", why: "not JSON" };
|
|
312
|
+
const parsed = schema.safeParse(json);
|
|
313
|
+
return parsed.success
|
|
314
|
+
? parsed.data
|
|
315
|
+
: { kind: "unreadable", why: reasons(parsed.error.issues) };
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
/** A `plan.json`: strict. */
|
|
319
|
+
export const parseStudyPlan = planParser(plan);
|
|
320
|
+
|
|
321
|
+
/** A plan still named `prereg.json`: loose, read only when a row names it. */
|
|
322
|
+
export const parseLegacyPlan = planParser(legacyPlan);
|
|
323
|
+
|
|
324
|
+
export const studyRegistryCodec: StudyRegistryCodec = {
|
|
325
|
+
parse: parseStudyRegistry,
|
|
326
|
+
parsePlan: parseStudyPlan,
|
|
327
|
+
parseLegacyPlan,
|
|
328
|
+
};
|
package/src/cli.ts
CHANGED
|
@@ -185,6 +185,12 @@ import {
|
|
|
185
185
|
REPRO_LAYOUT_RULE_LEVELS,
|
|
186
186
|
reproLayoutRules,
|
|
187
187
|
} from "./repro-layout-rule.ts";
|
|
188
|
+
import {
|
|
189
|
+
STUDY_REGISTRY_RULE_LEVELS,
|
|
190
|
+
studyRegistryRules,
|
|
191
|
+
} from "./study-registry-rule.ts";
|
|
192
|
+
import { studyRegistryCodec } from "./adapters/study-registry/index.ts";
|
|
193
|
+
import { runStudies } from "./studies.ts";
|
|
188
194
|
import typography from "#eslint-rules/paper-typography";
|
|
189
195
|
import texBuild from "#eslint-rules/tex-build";
|
|
190
196
|
import reviewFrontmatter from "#eslint-rules/review-frontmatter";
|
|
@@ -230,6 +236,13 @@ const USAGE = `paperlint — machine-checkable gates for a paper kept in git
|
|
|
230
236
|
send the PDF (default: the paper's own), an abstract, or
|
|
231
237
|
"submitted". A DRY RUN unless --save: the portal checks the
|
|
232
238
|
change and keeps nothing
|
|
239
|
+
npx paperlint studies new <paper-dir> "<title>" [--kind <kind>]
|
|
240
|
+
register a study: the next id in repro/studies.json, the folder
|
|
241
|
+
repro/<id>-<slug>/ with its README.md and plan.json (a template),
|
|
242
|
+
the row (status designing, today's date, its plan), and the
|
|
243
|
+
index in repro/README.md regenerated
|
|
244
|
+
npx paperlint studies index <paper-dir>
|
|
245
|
+
regenerate the index in repro/README.md from repro/studies.json
|
|
233
246
|
npx paperlint hook <name> run an editor hook (.claude/settings.json calls this)
|
|
234
247
|
npx paperlint --help
|
|
235
248
|
|
|
@@ -250,6 +263,12 @@ new:
|
|
|
250
263
|
"realm" can be the work's own). Without it such a name is refused: a rejected
|
|
251
264
|
paper moves to another venue and keeps its folder
|
|
252
265
|
|
|
266
|
+
studies:
|
|
267
|
+
--kind <kind> new: measurement (the default), spike, tooling, data, artifact or salvage.
|
|
268
|
+
A study's status, result and in_paper are edited in repro/studies.json, its
|
|
269
|
+
plan in its plan.json; the rule repro/study-registered checks them
|
|
270
|
+
(docs/rules/repro/study-registered.md)
|
|
271
|
+
|
|
253
272
|
submission:
|
|
254
273
|
The venue preset declares the portal — "portal": { "kind": "hotcrp", "url": "<site>" } — and the
|
|
255
274
|
paper's paperlint.json its submission there — "submission": { "id": <number> }. The token comes
|
|
@@ -367,6 +386,27 @@ const FRAGMENT_RULES = {
|
|
|
367
386
|
/** What the two layout rules read with: the disk and its directory listings. */
|
|
368
387
|
const LAYOUT_RULE_DEPS = { files: nodeFiles, listDir: nodeListDir };
|
|
369
388
|
|
|
389
|
+
/** What `repro/study-registered` and `paperlint studies` read with: the layout's, and the registry's parser. */
|
|
390
|
+
const STUDY_RULE_DEPS = { ...LAYOUT_RULE_DEPS, codec: studyRegistryCodec };
|
|
391
|
+
|
|
392
|
+
/** The `repro` plugin's rules, on every `PIPELINE-STATUS.md`: the layout of `repro/`, and its study registry. */
|
|
393
|
+
const REPRO_RULES = {
|
|
394
|
+
...reproLayoutRules(LAYOUT_RULE_DEPS),
|
|
395
|
+
...studyRegistryRules(STUDY_RULE_DEPS),
|
|
396
|
+
};
|
|
397
|
+
const REPRO_RULE_LEVELS = {
|
|
398
|
+
...REPRO_LAYOUT_RULE_LEVELS,
|
|
399
|
+
...STUDY_REGISTRY_RULE_LEVELS,
|
|
400
|
+
};
|
|
401
|
+
|
|
402
|
+
/** Today in this machine's time zone, `YYYY-MM-DD`: the date `paperlint studies new` registers a study on. */
|
|
403
|
+
const today = (now = new Date()): string =>
|
|
404
|
+
[
|
|
405
|
+
String(now.getFullYear()),
|
|
406
|
+
String(now.getMonth() + 1).padStart(2, "0"),
|
|
407
|
+
String(now.getDate()).padStart(2, "0"),
|
|
408
|
+
].join("-");
|
|
409
|
+
|
|
370
410
|
/** What the talk rules read with: the disk, the shipped presets, and the three media readers. */
|
|
371
411
|
const TALK_RULE_DEPS = {
|
|
372
412
|
files: nodeFiles,
|
|
@@ -433,7 +473,7 @@ export function buildConfig(
|
|
|
433
473
|
...folderLayoutRules(LAYOUT_RULE_DEPS),
|
|
434
474
|
},
|
|
435
475
|
},
|
|
436
|
-
repro: { rules:
|
|
476
|
+
repro: { rules: REPRO_RULES },
|
|
437
477
|
// Registered, on for no paper: `"study/declare-sample": "error"` in paperlint.json turns it on.
|
|
438
478
|
study: { rules: preregCoverage.rules },
|
|
439
479
|
cycle: { rules: CYCLE_STATUS_RULES },
|
|
@@ -444,7 +484,7 @@ export function buildConfig(
|
|
|
444
484
|
"paper/source": "error",
|
|
445
485
|
...FOLDER_VENUE_RULE_LEVELS,
|
|
446
486
|
...FOLDER_LAYOUT_RULE_LEVELS,
|
|
447
|
-
...
|
|
487
|
+
...REPRO_RULE_LEVELS,
|
|
448
488
|
...CYCLE_RULE_LEVELS,
|
|
449
489
|
},
|
|
450
490
|
},
|
|
@@ -1626,6 +1666,17 @@ const SIMPLE: Readonly<
|
|
|
1626
1666
|
banal: hostBanalInstaller(),
|
|
1627
1667
|
tex: toolchainTex(cwd),
|
|
1628
1668
|
}),
|
|
1669
|
+
studies: (a, { log, err, cwd }) =>
|
|
1670
|
+
runStudies(
|
|
1671
|
+
{
|
|
1672
|
+
sub: a.paths[0],
|
|
1673
|
+
paperDir: a.paths[1],
|
|
1674
|
+
title: a.paths[2],
|
|
1675
|
+
extra: a.paths.slice(3),
|
|
1676
|
+
kind: a.kind,
|
|
1677
|
+
},
|
|
1678
|
+
{ ...STUDY_RULE_DEPS, cwd, today: today(), log, err },
|
|
1679
|
+
),
|
|
1629
1680
|
submission: (a, { log, err, cwd }) =>
|
|
1630
1681
|
runSubmission(
|
|
1631
1682
|
{
|
|
@@ -257,7 +257,7 @@ describe("strayMessage — stands alone for a reader with no context", () => {
|
|
|
257
257
|
),
|
|
258
258
|
).toEqual([]);
|
|
259
259
|
expect(lines).toContain(
|
|
260
|
-
"│ └── <slug
|
|
260
|
+
"│ └── <id>-<slug>/ ← one folder per study or spike, named by its row in studies.json (P1-S03-baseline)",
|
|
261
261
|
);
|
|
262
262
|
expect(lines.at(-1)).toMatch(/^└── extra\/ +← anything; never judged$/);
|
|
263
263
|
});
|
|
@@ -38,6 +38,9 @@ describe("isStudyName", () => {
|
|
|
38
38
|
"leap-2024-02-29",
|
|
39
39
|
"leap-2000-02-29",
|
|
40
40
|
"april-2026-04-30",
|
|
41
|
+
"P1-S01-baseline",
|
|
42
|
+
"P12-S100-a1-b2",
|
|
43
|
+
"AB-S07-x",
|
|
41
44
|
])("%s is a study's name", (name) => {
|
|
42
45
|
expect(isStudyName(name)).toBe(true);
|
|
43
46
|
});
|
|
@@ -56,6 +59,11 @@ describe("isStudyName", () => {
|
|
|
56
59
|
["29 February 1900", "x-1900-02-29"],
|
|
57
60
|
["32 October", "x-2026-10-32"],
|
|
58
61
|
["a two-digit year", "x-26-10-12"],
|
|
62
|
+
["an id with one digit", "P1-S1-baseline"],
|
|
63
|
+
["an id and no slug", "P1-S01"],
|
|
64
|
+
["a lowercase id", "p1-s01-baseline"],
|
|
65
|
+
["a capital in the slug", "P1-S01-Baseline"],
|
|
66
|
+
["an id with no S", "P1-01-baseline"],
|
|
59
67
|
])("not with %s: %s", (_why, name) => {
|
|
60
68
|
expect(isStudyName(name)).toBe(false);
|
|
61
69
|
});
|
|
@@ -158,7 +166,7 @@ describe("judgeRepro — reports a folder, one finding each", () => {
|
|
|
158
166
|
{
|
|
159
167
|
path: "repro/Old_Baseline/",
|
|
160
168
|
problem: "stray",
|
|
161
|
-
cure: "rename it to
|
|
169
|
+
cure: "rename it to `<id>-old-baseline` — the next free id in repro/studies.json, then what it is — give it its row there and a README.md (`npx paperlint studies new` makes all three for a new study); in a paper with repro/study-registered off, `old-baseline-YYYY-MM-DD` is accepted instead, dated by the day it began — its first commit is a fine default (`git log --reverse --format=%as -- repro/Old_Baseline | head -1`, run in the paper folder); if it holds scripts or data shared across the paper instead, move them into repro/tools/ or repro/data/",
|
|
162
170
|
},
|
|
163
171
|
]);
|
|
164
172
|
});
|
|
@@ -168,7 +176,7 @@ describe("judgeRepro — reports a folder, one finding each", () => {
|
|
|
168
176
|
{
|
|
169
177
|
path: "repro/ablation-2026-10-12/",
|
|
170
178
|
problem: "no-readme",
|
|
171
|
-
cure: "add repro/ablation-2026-10-12/README.md — what the study asks, how to run it, what it found — and
|
|
179
|
+
cure: "add repro/ablation-2026-10-12/README.md — what the study asks, how to run it, what it found — and give the study its row in repro/studies.json",
|
|
172
180
|
},
|
|
173
181
|
]);
|
|
174
182
|
});
|
|
@@ -194,15 +202,16 @@ describe("reproMessage — stands alone", () => {
|
|
|
194
202
|
}),
|
|
195
203
|
).toBe(
|
|
196
204
|
[
|
|
197
|
-
`\`repro/run.py\` is loose at the top of the paper's repro/ folder, which holds only an index, the shared tools and data, and one
|
|
205
|
+
`\`repro/run.py\` is loose at the top of the paper's repro/ folder, which holds only an index, the study registry, the shared tools and data, and one folder per study. Where it goes: ${SCRIPT_CURE}.`,
|
|
198
206
|
"What repro/ holds (what interpreters and package managers leave there, like __pycache__/ and node_modules/, is never reported):",
|
|
199
207
|
"repro/",
|
|
200
|
-
"├── README.md
|
|
201
|
-
"├── .
|
|
202
|
-
"├──
|
|
203
|
-
"├──
|
|
204
|
-
"
|
|
205
|
-
"
|
|
208
|
+
"├── README.md ← the index: one line per study, generated from studies.json",
|
|
209
|
+
"├── studies.json ← the study registry: id, title, status and result of each study",
|
|
210
|
+
"├── .gitignore ← keeps the scripts' byproducts out of git",
|
|
211
|
+
"├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)",
|
|
212
|
+
"├── data/ ← shared data (tables, caches)",
|
|
213
|
+
"└── <id>-<slug>/ ← one folder per study or spike, named by its row in studies.json (P1-S03-baseline)",
|
|
214
|
+
" └── README.md ← required: what it asks, how to run it, what it found",
|
|
206
215
|
"Rule repro/allowed-files-only: https://github.com/zernie/paperlint/blob/main/docs/rules/repro/allowed-files-only.md",
|
|
207
216
|
].join("\n"),
|
|
208
217
|
);
|
|
@@ -7,15 +7,21 @@
|
|
|
7
7
|
* where a new run goes. The layout gives each kind of material one place:
|
|
8
8
|
*
|
|
9
9
|
* repro/
|
|
10
|
-
* ├── README.md the index: one line per study
|
|
11
|
-
* ├──
|
|
10
|
+
* ├── README.md the index: one line per study, generated from studies.json
|
|
11
|
+
* ├── studies.json the study registry: one row per study (src/domain/study-registry.ts)
|
|
12
|
+
* ├── <id>-<slug>/ one folder per study or spike, named by its row, with a README.md inside
|
|
12
13
|
* ├── tools/ scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
13
14
|
* └── data/ shared data (tables, caches)
|
|
14
15
|
*
|
|
15
16
|
* plus `.gitignore`. A loose note belongs in its study's folder or in the paper's `research/`.
|
|
16
17
|
*
|
|
17
|
-
* ──
|
|
18
|
-
* `<slug
|
|
18
|
+
* ── A STUDY'S NAME ───────────────────────────────────────────────────────────────
|
|
19
|
+
* `<id>-<slug>` (`P1-S03-baseline`): the id of the study's row in `repro/studies.json` — the paper's
|
|
20
|
+
* prefix, `-S`, two or more digits — then the slug. Whether the id has a row, and the row this folder,
|
|
21
|
+
* is `repro/study-registered`'s question; this layout only accepts the shape.
|
|
22
|
+
*
|
|
23
|
+
* The older form, `<slug>-YYYY-MM-DD`, stays accepted, so a paper that turned the registry off is not
|
|
24
|
+
* broken: the date is the day the study began. `<slug>-YYYY-MM` is accepted too: a study that ran
|
|
19
25
|
* over weeks (labelling, a reading of a corpus) has a month as its honest date, and both forms sort
|
|
20
26
|
* by time. The date must be a real one (no month 13, no 30 February). The slug is lowercase letters
|
|
21
27
|
* and digits in words joined by single hyphens.
|
|
@@ -32,6 +38,9 @@ import { leaf, renderTree, type TreeNode } from "./layout-tree.ts";
|
|
|
32
38
|
/** The folder this layout governs, at the paper folder's root. */
|
|
33
39
|
export const REPRO_DIR = "repro";
|
|
34
40
|
|
|
41
|
+
/** The study registry, at the top of `repro/` (src/domain/study-registry.ts). */
|
|
42
|
+
export const STUDIES_FILE = "studies.json";
|
|
43
|
+
|
|
35
44
|
/** The index at the top of `repro/`, and the file every study folder holds. */
|
|
36
45
|
export const README = "README.md";
|
|
37
46
|
|
|
@@ -40,7 +49,14 @@ export const REPRO_FILE_NOTES: readonly {
|
|
|
40
49
|
readonly name: string;
|
|
41
50
|
readonly note: string;
|
|
42
51
|
}[] = [
|
|
43
|
-
{
|
|
52
|
+
{
|
|
53
|
+
name: README,
|
|
54
|
+
note: "the index: one line per study, generated from studies.json",
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
name: STUDIES_FILE,
|
|
58
|
+
note: "the study registry: id, title, status and result of each study",
|
|
59
|
+
},
|
|
44
60
|
{ name: ".gitignore", note: "keeps the scripts' byproducts out of git" },
|
|
45
61
|
];
|
|
46
62
|
export const REPRO_FILES: readonly string[] = REPRO_FILE_NOTES.map(
|
|
@@ -64,8 +80,8 @@ export const REPRO_DIRS: readonly string[] = REPRO_DIR_NOTES.map((d) => d.name);
|
|
|
64
80
|
|
|
65
81
|
/** The study folder as the layout draws it: the name pattern, and the README.md it must hold. */
|
|
66
82
|
const STUDY_NODE: TreeNode = {
|
|
67
|
-
name: "<slug
|
|
68
|
-
note: "one folder per study or spike
|
|
83
|
+
name: "<id>-<slug>/",
|
|
84
|
+
note: "one folder per study or spike, named by its row in studies.json (P1-S03-baseline)",
|
|
69
85
|
children: [
|
|
70
86
|
leaf(README, "required: what it asks, how to run it, what it found"),
|
|
71
87
|
],
|
|
@@ -100,9 +116,20 @@ export const REPRO_BYPRODUCTS: readonly string[] = [
|
|
|
100
116
|
"Thumbs.db",
|
|
101
117
|
];
|
|
102
118
|
|
|
103
|
-
/** A
|
|
119
|
+
/** A slug: lowercase letters and digits, in words joined by single hyphens. */
|
|
120
|
+
export const STUDY_SLUG = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u;
|
|
121
|
+
|
|
122
|
+
/** A dated study's name: a slug, then the year, the month, and optionally the day it began. */
|
|
104
123
|
const STUDY_NAME = /^([a-z0-9]+(?:-[a-z0-9]+)*)-(\d{4})-(\d{2})(?:-(\d{2}))?$/u;
|
|
105
124
|
|
|
125
|
+
/** A registered study's name: the id (a prefix, `-S`, two or more digits), then the slug. */
|
|
126
|
+
const REGISTERED_STUDY_NAME =
|
|
127
|
+
/^[A-Z][A-Z0-9]*-S\d{2,}-[a-z0-9]+(?:-[a-z0-9]+)*$/u;
|
|
128
|
+
|
|
129
|
+
/** Whether `name` is a registered study folder's: `<id>-<slug>`, as `P1-S03-baseline`. */
|
|
130
|
+
export const isRegisteredStudyName = (name: string): boolean =>
|
|
131
|
+
REGISTERED_STUDY_NAME.test(name);
|
|
132
|
+
|
|
106
133
|
const isLeapYear = (y: number): boolean =>
|
|
107
134
|
(y % 4 === 0 && y % 100 !== 0) || y % 400 === 0;
|
|
108
135
|
|
|
@@ -110,8 +137,8 @@ const isLeapYear = (y: number): boolean =>
|
|
|
110
137
|
const daysIn = (y: number, m: number): number =>
|
|
111
138
|
m === 2 ? (isLeapYear(y) ? 29 : 28) : [4, 6, 9, 11].includes(m) ? 30 : 31;
|
|
112
139
|
|
|
113
|
-
/** Whether `name` is a study folder's: `<slug>-YYYY-MM-DD` or `<slug>-YYYY-MM`, a real date. */
|
|
114
|
-
|
|
140
|
+
/** Whether `name` is a dated study folder's: `<slug>-YYYY-MM-DD` or `<slug>-YYYY-MM`, a real date. */
|
|
141
|
+
function isDatedStudyName(name: string): boolean {
|
|
115
142
|
const m = STUDY_NAME.exec(name);
|
|
116
143
|
if (m === null) return false;
|
|
117
144
|
const year = Number(m[2]);
|
|
@@ -120,6 +147,10 @@ export function isStudyName(name: string): boolean {
|
|
|
120
147
|
return month >= 1 && month <= 12 && day >= 1 && day <= daysIn(year, month);
|
|
121
148
|
}
|
|
122
149
|
|
|
150
|
+
/** Whether `name` is a study folder's: `<id>-<slug>`, or the dated `<slug>-YYYY-MM-DD` / `<slug>-YYYY-MM`. */
|
|
151
|
+
export const isStudyName = (name: string): boolean =>
|
|
152
|
+
isRegisteredStudyName(name) || isDatedStudyName(name);
|
|
153
|
+
|
|
123
154
|
/**
|
|
124
155
|
* The study name to suggest for a folder named `name`: its name as a slug, with the date to fill in.
|
|
125
156
|
* A name that becomes a study name once lowercased and hyphenated is suggested as it is.
|
|
@@ -129,9 +160,18 @@ export function suggestedStudyName(name: string): string {
|
|
|
129
160
|
.toLowerCase()
|
|
130
161
|
.replace(/[^a-z0-9]+/gu, "-")
|
|
131
162
|
.replace(/^-+|-+$/gu, "");
|
|
132
|
-
if (
|
|
133
|
-
|
|
134
|
-
|
|
163
|
+
if (isDatedStudyName(slug)) return slug;
|
|
164
|
+
return `${suggestedSlug(name)}-YYYY-MM-DD`;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** The slug to suggest for a folder named `name`: its name as a slug, any trailing date dropped. */
|
|
168
|
+
export function suggestedSlug(name: string): string {
|
|
169
|
+
const base = name
|
|
170
|
+
.toLowerCase()
|
|
171
|
+
.replace(/[^a-z0-9]+/gu, "-")
|
|
172
|
+
.replace(/^-+|-+$/gu, "")
|
|
173
|
+
.replace(/(?:^|-)\d{4}(?:-\d{1,2}){0,2}$/u, "");
|
|
174
|
+
return base === "" ? "<slug>" : base;
|
|
135
175
|
}
|
|
136
176
|
|
|
137
177
|
/** The extensions of a script: what a study or the paper's build runs. */
|
|
@@ -203,11 +243,11 @@ export const DATA_CURE =
|
|
|
203
243
|
|
|
204
244
|
/** Where a note goes. */
|
|
205
245
|
export const NOTE_CURE =
|
|
206
|
-
"the
|
|
246
|
+
"the folder of the study it is about (repro/<id>-<slug>/), or the paper's research/ when it is research or a decision rather than the record of a run";
|
|
207
247
|
|
|
208
248
|
/** Where a file of any other kind goes. */
|
|
209
249
|
export const FILE_CURE_BY_KIND =
|
|
210
|
-
"decide by what it is — a script → repro/tools/; data → repro/data/; a note → the
|
|
250
|
+
"decide by what it is — a script → repro/tools/; data → repro/data/; a note → the folder of its study, or the paper's research/; what one study produced → that study's folder";
|
|
211
251
|
|
|
212
252
|
/** The files paperlint itself once kept at the top of `repro/`, each with where it is read now. */
|
|
213
253
|
export const MOVED_FILES: readonly {
|
|
@@ -242,7 +282,7 @@ function fileCure(name: string): string {
|
|
|
242
282
|
|
|
243
283
|
/** Where a folder with no study name goes: renamed as a study, or emptied into the shared folders. */
|
|
244
284
|
const folderCure = (name: string): string =>
|
|
245
|
-
`rename it to
|
|
285
|
+
`rename it to \`<id>-${suggestedSlug(name)}\` — the next free id in repro/${STUDIES_FILE}, then what it is — give it its row there and a README.md (\`npx paperlint studies new\` makes all three for a new study); in a paper with repro/study-registered off, \`${suggestedStudyName(name)}\` is accepted instead, dated by the day it began — its first commit is a fine default (\`git log --reverse --format=%as -- repro/${name} | head -1\`, run in the paper folder); if it holds scripts or data shared across the paper instead, move them into repro/tools/ or repro/data/`;
|
|
246
286
|
|
|
247
287
|
/** An entry at the top of `repro/`: a file or a folder, and for a folder whether it holds a README.md. */
|
|
248
288
|
export interface ReproEntry {
|
|
@@ -280,7 +320,7 @@ function judgeEntry(e: ReproEntry): ReproFinding | null {
|
|
|
280
320
|
return {
|
|
281
321
|
path,
|
|
282
322
|
problem: "no-readme",
|
|
283
|
-
cure: `add ${path}${README} — what the study asks, how to run it, what it found — and
|
|
323
|
+
cure: `add ${path}${README} — what the study asks, how to run it, what it found — and give the study its row in repro/${STUDIES_FILE}`,
|
|
284
324
|
};
|
|
285
325
|
return null;
|
|
286
326
|
}
|
|
@@ -300,7 +340,7 @@ export function judgeRepro(
|
|
|
300
340
|
export function reproMessage(f: ReproFinding): string {
|
|
301
341
|
const head =
|
|
302
342
|
f.problem === "stray"
|
|
303
|
-
? `\`${f.path}\` is loose at the top of the paper's repro/ folder, which holds only an index, the shared tools and data, and one
|
|
343
|
+
? `\`${f.path}\` is loose at the top of the paper's repro/ folder, which holds only an index, the study registry, the shared tools and data, and one folder per study. Where it goes: ${f.cure}.`
|
|
304
344
|
: `\`${f.path}\` is a study folder without a ${README}, and every study in repro/ has one. What to do: ${f.cure}.`;
|
|
305
345
|
return [
|
|
306
346
|
head,
|