paperlint 10.2.0 → 10.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/study-registry/index.d.ts +352 -1
- package/dist/adapters/study-registry/index.d.ts.map +1 -1
- package/dist/adapters/study-registry/index.js +171 -8
- package/dist/adapters/study-registry/index.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +6 -4
- package/dist/cli.js.map +1 -1
- package/dist/domain/study-registry.d.ts +201 -5
- package/dist/domain/study-registry.d.ts.map +1 -1
- package/dist/domain/study-registry.js +224 -2
- package/dist/domain/study-registry.js.map +1 -1
- package/dist/ports/study-registry-codec.d.ts +6 -2
- package/dist/ports/study-registry-codec.d.ts.map +1 -1
- package/dist/studies.d.ts.map +1 -1
- package/dist/studies.js +7 -4
- package/dist/studies.js.map +1 -1
- package/dist/study-registry-rule.d.ts +12 -5
- package/dist/study-registry-rule.d.ts.map +1 -1
- package/dist/study-registry-rule.js +67 -37
- package/dist/study-registry-rule.js.map +1 -1
- package/docs/design/study-registry.md +131 -15
- package/docs/layout.md +23 -21
- package/docs/rules/repro/study-plan-name.md +70 -0
- package/docs/rules/repro/study-registered.md +164 -6
- package/docs/rules.md +53 -52
- package/package.json +1 -1
- package/skills/build-benchmark/SKILL.md +24 -12
- package/skills/build-benchmark/SKILL.md.spec.ts +23 -11
- package/skills/build-label-page/SKILL.md +9 -7
- package/skills/build-label-page/SKILL.md.spec.ts +8 -6
- package/skills/run-agent-grid/SKILL.md +10 -5
- package/skills/run-agent-grid/SKILL.md.spec.ts +9 -4
- package/src/adapters/study-registry/index.test.ts +340 -0
- package/src/adapters/study-registry/index.ts +268 -10
- package/src/cli.ts +6 -4
- package/src/domain/study-measurements.test.ts +629 -0
- package/src/domain/study-registry.test.ts +10 -1
- package/src/domain/study-registry.ts +456 -10
- package/src/new-paper.test.ts +1 -0
- package/src/ports/study-registry-codec.ts +6 -2
- package/src/rule-cases.test.ts +27 -0
- package/src/studies-cli.test.ts +2 -2
- package/src/studies.test.ts +6 -0
- package/src/studies.ts +12 -3
- package/src/study-registry-rule.test.ts +140 -1
- package/src/study-registry-rule.ts +106 -39
|
@@ -61,6 +61,55 @@ export const INDEX_END = "<!-- studies:end -->";
|
|
|
61
61
|
/** The README written when `repro/` has none. */
|
|
62
62
|
const INDEX_HEADING = "# Reproduction index";
|
|
63
63
|
|
|
64
|
+
/** What a measurement is to the paper: the claim it carries, or what that claim is compared with. */
|
|
65
|
+
export const MEASUREMENT_ROLES = ["headline", "baseline"] as const;
|
|
66
|
+
export type MeasurementRole = (typeof MEASUREMENT_ROLES)[number];
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The statuses of a study that has run: its plan was due before the first run. A `designing` study
|
|
70
|
+
* has not run yet, and a `dropped` one never will.
|
|
71
|
+
*/
|
|
72
|
+
export const RAN_STATUSES: readonly StudyStatus[] = [
|
|
73
|
+
"running",
|
|
74
|
+
"done",
|
|
75
|
+
"stopped",
|
|
76
|
+
"superseded",
|
|
77
|
+
];
|
|
78
|
+
|
|
79
|
+
/** A requirement of the author's on the paper's measurements, in their words. */
|
|
80
|
+
export interface Requirement {
|
|
81
|
+
readonly id: string;
|
|
82
|
+
/** The author's words, verbatim. */
|
|
83
|
+
readonly text: string;
|
|
84
|
+
readonly date: string;
|
|
85
|
+
/** Where they said it: a chat, a message, a review comment. */
|
|
86
|
+
readonly source: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Someone's approval, in their words: of a measurement, or of a study's drawn units. */
|
|
90
|
+
export interface Approval {
|
|
91
|
+
readonly by: string;
|
|
92
|
+
readonly date: string;
|
|
93
|
+
readonly quote: string;
|
|
94
|
+
readonly source?: string | undefined;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* One measurement: what the paper measures, made of one or more studies (a pilot, the main run, a
|
|
99
|
+
* replication) whose rows name it in `measurement`. A measurement may carry more fields.
|
|
100
|
+
*/
|
|
101
|
+
export interface Measurement {
|
|
102
|
+
readonly id: string;
|
|
103
|
+
readonly title: string;
|
|
104
|
+
/** The research question it answers. */
|
|
105
|
+
readonly rq?: string | undefined;
|
|
106
|
+
readonly role: MeasurementRole;
|
|
107
|
+
readonly approval?: Approval | undefined;
|
|
108
|
+
/** The ids of the requirements it must meet. */
|
|
109
|
+
readonly requirements?: readonly string[] | undefined;
|
|
110
|
+
readonly [extra: string]: unknown;
|
|
111
|
+
}
|
|
112
|
+
|
|
64
113
|
/**
|
|
65
114
|
* One study. The named fields are the registry's; a row may carry more (a paper merges its own
|
|
66
115
|
* measurement metadata into it), and they are kept as they are.
|
|
@@ -77,14 +126,126 @@ export interface Study {
|
|
|
77
126
|
readonly result_file?: string | undefined;
|
|
78
127
|
readonly superseded_by?: string | undefined;
|
|
79
128
|
readonly in_paper: InPaper;
|
|
129
|
+
/** The id of the measurement the study belongs to. */
|
|
130
|
+
readonly measurement?: string | undefined;
|
|
131
|
+
/** The study's plan: a path inside its folder, `plan.json`. */
|
|
132
|
+
readonly plan?: string | undefined;
|
|
80
133
|
readonly [extra: string]: unknown;
|
|
81
134
|
}
|
|
82
135
|
|
|
83
136
|
export interface StudyRegistry {
|
|
84
137
|
readonly prefix: string;
|
|
138
|
+
/** The paper's author: the one person whose approval makes a measurement the headline. */
|
|
139
|
+
readonly author?: string | undefined;
|
|
140
|
+
readonly requirements?: readonly Requirement[] | undefined;
|
|
141
|
+
/** When present, every row's `measurement` names one of them. */
|
|
142
|
+
readonly measurements?: readonly Measurement[] | undefined;
|
|
85
143
|
readonly studies: readonly Study[];
|
|
86
144
|
}
|
|
87
145
|
|
|
146
|
+
/** The file `paperlint studies new` writes a study's plan to, beside its README.md. */
|
|
147
|
+
export const PLAN_FILE = "plan.json";
|
|
148
|
+
|
|
149
|
+
/** The plan's name before it was a plan: still read as one, and reported by `repro/study-plan-name`. */
|
|
150
|
+
export const LEGACY_PLAN_FILE = "prereg.json";
|
|
151
|
+
|
|
152
|
+
/** The author's check of a study's drawn units, before the runs. */
|
|
153
|
+
export type AuthorCheck = Omit<Approval, "source">;
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* A study's plan, fixed before its runs: what it measures, on what population and how it is drawn,
|
|
157
|
+
* its arms, its primary outcome, and the author's check of the drawn units. Closed: a key the paper
|
|
158
|
+
* needs of its own goes in `extra`, so a misspelt key is an error, not a key nobody reads. "Not
|
|
159
|
+
* known" and "not yet" have one spelling each: null.
|
|
160
|
+
*/
|
|
161
|
+
export interface StudyPlan {
|
|
162
|
+
readonly question: string;
|
|
163
|
+
readonly population: {
|
|
164
|
+
/** Where the units come from: the census, crawl or registry, with its date. */
|
|
165
|
+
readonly source: string;
|
|
166
|
+
readonly draw: Draw;
|
|
167
|
+
};
|
|
168
|
+
/** The conditions compared, each name once; none for a census. */
|
|
169
|
+
readonly arms: readonly { readonly name: string }[];
|
|
170
|
+
readonly primary_outcome: string;
|
|
171
|
+
/** null: the author has not checked the drawn units yet. Only a ready plan carries one. */
|
|
172
|
+
readonly author_check: AuthorCheck | null;
|
|
173
|
+
/** The paper's own keys: hypotheses, models, budget … Kept, never read; none is a plan field. */
|
|
174
|
+
readonly extra?: Readonly<Record<string, unknown>> | undefined;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* How the sample is drawn from the population: a seeded draw, whose seed is part of it, or all of
|
|
179
|
+
* the population (a census), which has none. The size of the population is not stored: it is
|
|
180
|
+
* counted from the source, and a stored copy would only contradict it.
|
|
181
|
+
*/
|
|
182
|
+
export type Draw =
|
|
183
|
+
| {
|
|
184
|
+
readonly kind: "seeded";
|
|
185
|
+
/** `random`, `stratified` … */
|
|
186
|
+
readonly method: string;
|
|
187
|
+
readonly seed: string | number;
|
|
188
|
+
}
|
|
189
|
+
| {
|
|
190
|
+
readonly kind: "all";
|
|
191
|
+
/** `census` … */
|
|
192
|
+
readonly method: string;
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
/** The top-level fields of a plan: none of them may be a key of `extra`. */
|
|
196
|
+
export const PLAN_FIELDS = [
|
|
197
|
+
"question",
|
|
198
|
+
"population",
|
|
199
|
+
"arms",
|
|
200
|
+
"primary_outcome",
|
|
201
|
+
"author_check",
|
|
202
|
+
"extra",
|
|
203
|
+
] as const;
|
|
204
|
+
|
|
205
|
+
/** A plan with empty required texts: the author has not approved it. */
|
|
206
|
+
export type DraftPlan = StudyPlan & { readonly author_check: null };
|
|
207
|
+
|
|
208
|
+
/** The texts a plan must fill before the study runs, in the order they are named. */
|
|
209
|
+
export const PLAN_TEXT_FIELDS = [
|
|
210
|
+
"question",
|
|
211
|
+
"population.source",
|
|
212
|
+
"population.draw.method",
|
|
213
|
+
"primary_outcome",
|
|
214
|
+
] as const;
|
|
215
|
+
export type PlanTextField = (typeof PLAN_TEXT_FIELDS)[number];
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* A plan's text, parsed: ready (every required text filled), a draft (its empty texts named, and
|
|
219
|
+
* no check of the author's: only a complete plan is approved), or the reasons it is not a plan. Whether a plan is complete is decided here, once, by the parser.
|
|
220
|
+
*/
|
|
221
|
+
export type ParsedPlan =
|
|
222
|
+
| { readonly kind: "ready"; readonly plan: StudyPlan }
|
|
223
|
+
| {
|
|
224
|
+
readonly kind: "draft";
|
|
225
|
+
readonly plan: DraftPlan;
|
|
226
|
+
readonly empty: readonly PlanTextField[];
|
|
227
|
+
}
|
|
228
|
+
| { readonly kind: "unreadable"; readonly why: string };
|
|
229
|
+
|
|
230
|
+
/** What a study's folder holds of the plan its row names: absent, or what its text parsed to. */
|
|
231
|
+
export type PlanState = { readonly kind: "absent" } | ParsedPlan;
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* The plan `paperlint studies new` writes: every field present, its value to fill. Its draw is of
|
|
235
|
+
* all the population (a census); a sampled study changes it to `{ "kind": "seeded", "method", "seed" }`.
|
|
236
|
+
*/
|
|
237
|
+
export const planTemplate = (): DraftPlan => ({
|
|
238
|
+
question: "",
|
|
239
|
+
population: { source: "", draw: { kind: "all", method: "" } },
|
|
240
|
+
arms: [],
|
|
241
|
+
primary_outcome: "",
|
|
242
|
+
author_check: null,
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
/** A plan's text as `paperlint studies new` writes it: JSON, two-space indented, a final newline. */
|
|
246
|
+
export const planText = (p: StudyPlan): string =>
|
|
247
|
+
`${JSON.stringify(p, null, 2)}\n`;
|
|
248
|
+
|
|
88
249
|
/** A registry's text, parsed: the registry, or the one reason it is not one. */
|
|
89
250
|
export type ParsedRegistry =
|
|
90
251
|
| { readonly ok: true; readonly registry: StudyRegistry }
|
|
@@ -105,6 +266,8 @@ export interface StudyInput {
|
|
|
105
266
|
readonly readme: string | null;
|
|
106
267
|
/** Each `<folder>/<result_file>` of the registry that is a file. */
|
|
107
268
|
readonly existingResults: ReadonlySet<string>;
|
|
269
|
+
/** The plan each row names, by `<folder>/<plan>`; one not listed is absent. */
|
|
270
|
+
readonly plans: ReadonlyMap<string, PlanState>;
|
|
108
271
|
}
|
|
109
272
|
|
|
110
273
|
/** One defect of the registry, of a row, or of the index. */
|
|
@@ -157,8 +320,79 @@ export type StudyFinding =
|
|
|
157
320
|
| {
|
|
158
321
|
readonly problem: "stale-index";
|
|
159
322
|
readonly why: "no-readme" | "no-markers" | "differs";
|
|
323
|
+
}
|
|
324
|
+
| {
|
|
325
|
+
readonly problem: "unknown-measurement";
|
|
326
|
+
readonly id: string;
|
|
327
|
+
readonly measurement: string;
|
|
328
|
+
}
|
|
329
|
+
| { readonly problem: "empty-measurement"; readonly measurement: string }
|
|
330
|
+
| {
|
|
331
|
+
readonly problem: "unknown-requirement";
|
|
332
|
+
readonly measurement: string;
|
|
333
|
+
readonly requirement: string;
|
|
334
|
+
}
|
|
335
|
+
| {
|
|
336
|
+
readonly problem: "headline-approval";
|
|
337
|
+
readonly measurement: string;
|
|
338
|
+
readonly why: "missing";
|
|
339
|
+
readonly author: string | null;
|
|
340
|
+
}
|
|
341
|
+
| {
|
|
342
|
+
readonly problem: "headline-approval";
|
|
343
|
+
readonly measurement: string;
|
|
344
|
+
readonly why: "not-author";
|
|
345
|
+
readonly by: string;
|
|
346
|
+
readonly author: string;
|
|
347
|
+
}
|
|
348
|
+
| {
|
|
349
|
+
readonly problem: "bad-plan";
|
|
350
|
+
readonly id: string;
|
|
351
|
+
readonly path: string;
|
|
352
|
+
readonly why: "missing" | "outside";
|
|
353
|
+
}
|
|
354
|
+
| {
|
|
355
|
+
readonly problem: "bad-plan";
|
|
356
|
+
readonly id: string;
|
|
357
|
+
readonly path: string;
|
|
358
|
+
readonly why: "unreadable";
|
|
359
|
+
readonly reasons: string;
|
|
360
|
+
}
|
|
361
|
+
| {
|
|
362
|
+
readonly problem: "bad-plan";
|
|
363
|
+
readonly id: string;
|
|
364
|
+
readonly path: string;
|
|
365
|
+
readonly why: "unfilled";
|
|
366
|
+
readonly fields: readonly PlanTextField[];
|
|
367
|
+
readonly status: StudyStatus;
|
|
368
|
+
}
|
|
369
|
+
| {
|
|
370
|
+
readonly problem: "author-check";
|
|
371
|
+
readonly id: string;
|
|
372
|
+
readonly measurement: string;
|
|
373
|
+
readonly status: StudyStatus;
|
|
374
|
+
readonly why: "no-plan" | "missing";
|
|
375
|
+
}
|
|
376
|
+
| {
|
|
377
|
+
readonly problem: "author-check";
|
|
378
|
+
readonly id: string;
|
|
379
|
+
readonly measurement: string;
|
|
380
|
+
readonly status: StudyStatus;
|
|
381
|
+
readonly why: "not-author";
|
|
382
|
+
readonly by: string;
|
|
383
|
+
readonly author: string;
|
|
384
|
+
}
|
|
385
|
+
| {
|
|
386
|
+
readonly problem: "legacy-plan";
|
|
387
|
+
readonly id: string;
|
|
388
|
+
readonly path: string;
|
|
160
389
|
};
|
|
161
390
|
|
|
391
|
+
/** The findings `repro/study-plan-name` reports, a warning; every other is `repro/study-registered`'s. */
|
|
392
|
+
export const PLAN_NAME_PROBLEMS: readonly StudyFinding["problem"][] = [
|
|
393
|
+
"legacy-plan",
|
|
394
|
+
];
|
|
395
|
+
|
|
162
396
|
/** The number of an id of `prefix` (`P1-S07` → 7); null for an id of another shape. */
|
|
163
397
|
function idNumber(prefix: string, id: string): number | null {
|
|
164
398
|
const m = new RegExp(`^${prefix}-S(\\d{2,})$`, "u").exec(id);
|
|
@@ -198,6 +432,15 @@ const isInside = (path: string): boolean => {
|
|
|
198
432
|
return n !== ".." && !n.startsWith("../");
|
|
199
433
|
};
|
|
200
434
|
|
|
435
|
+
/**
|
|
436
|
+
* The `<folder>/<plan>` of each row whose plan lies inside its folder: the files the judge needs read
|
|
437
|
+
* (`StudyInput.plans`). A plan outside its folder is reported without being read.
|
|
438
|
+
*/
|
|
439
|
+
export const planKeys = (r: StudyRegistry): readonly string[] =>
|
|
440
|
+
r.studies.flatMap((s) =>
|
|
441
|
+
s.plan !== undefined && isInside(s.plan) ? [`${s.folder}/${s.plan}`] : [],
|
|
442
|
+
);
|
|
443
|
+
|
|
201
444
|
/** One row of a registry, with what the judge read beside it. */
|
|
202
445
|
interface Row {
|
|
203
446
|
readonly r: StudyRegistry;
|
|
@@ -283,6 +526,102 @@ const noSuccessor: RowCheck = ({ r, s }) => {
|
|
|
283
526
|
: null;
|
|
284
527
|
};
|
|
285
528
|
|
|
529
|
+
/** A row's `measurement`, when the registry declares its measurements, names one of them. */
|
|
530
|
+
const unknownMeasurement: RowCheck = ({ r, s }) =>
|
|
531
|
+
r.measurements === undefined ||
|
|
532
|
+
s.measurement === undefined ||
|
|
533
|
+
r.measurements.some((m) => m.id === s.measurement)
|
|
534
|
+
? null
|
|
535
|
+
: { problem: "unknown-measurement", id: s.id, measurement: s.measurement };
|
|
536
|
+
|
|
537
|
+
/** The state of the plan the row names (absent when outside its folder); null when it names none. */
|
|
538
|
+
const planOf = ({ s, input }: Row): PlanState | null => {
|
|
539
|
+
if (s.plan === undefined) return null;
|
|
540
|
+
return isInside(s.plan)
|
|
541
|
+
? (input.plans.get(`${s.folder}/${s.plan}`) ?? { kind: "absent" })
|
|
542
|
+
: { kind: "absent" };
|
|
543
|
+
};
|
|
544
|
+
|
|
545
|
+
const hasRun = (s: Study): boolean => RAN_STATUSES.includes(s.status);
|
|
546
|
+
|
|
547
|
+
/** The finding for the plan `path` of `s`, in the state its folder holds it; null when it stands. */
|
|
548
|
+
const planFinding = (
|
|
549
|
+
s: Study,
|
|
550
|
+
path: string,
|
|
551
|
+
state: PlanState,
|
|
552
|
+
): StudyFinding | null => {
|
|
553
|
+
const at = { problem: "bad-plan", id: s.id, path } as const;
|
|
554
|
+
switch (state.kind) {
|
|
555
|
+
case "absent":
|
|
556
|
+
return { ...at, why: "missing" };
|
|
557
|
+
case "unreadable":
|
|
558
|
+
return { ...at, why: "unreadable", reasons: state.why };
|
|
559
|
+
case "draft":
|
|
560
|
+
return hasRun(s)
|
|
561
|
+
? { ...at, why: "unfilled", fields: state.empty, status: s.status }
|
|
562
|
+
: null;
|
|
563
|
+
case "ready":
|
|
564
|
+
return null;
|
|
565
|
+
}
|
|
566
|
+
};
|
|
567
|
+
|
|
568
|
+
/** The plan a row names is a file inside its folder, of the schema, and ready once the study has run. */
|
|
569
|
+
const badPlan: RowCheck = ({ s, input }) => {
|
|
570
|
+
const path = s.plan;
|
|
571
|
+
if (path === undefined) return null;
|
|
572
|
+
if (!isInside(path))
|
|
573
|
+
return { problem: "bad-plan", id: s.id, path, why: "outside" };
|
|
574
|
+
return planFinding(
|
|
575
|
+
s,
|
|
576
|
+
path,
|
|
577
|
+
input.plans.get(`${s.folder}/${path}`) ?? { kind: "absent" },
|
|
578
|
+
);
|
|
579
|
+
};
|
|
580
|
+
|
|
581
|
+
/** A plan under its old name: read, and named for renaming. */
|
|
582
|
+
const legacyPlan: RowCheck = ({ s }) =>
|
|
583
|
+
s.plan !== undefined && posix.basename(s.plan) === LEGACY_PLAN_FILE
|
|
584
|
+
? { problem: "legacy-plan", id: s.id, path: s.plan }
|
|
585
|
+
: null;
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* A study of a headline measurement that has run carries the author's check of its drawn units in
|
|
589
|
+
* its plan, by the registry's author when it names one. A plan that cannot be read is `badPlan`'s.
|
|
590
|
+
*/
|
|
591
|
+
const authorCheck: RowCheck = (row) => {
|
|
592
|
+
const { r, s } = row;
|
|
593
|
+
const m = r.measurements?.find((o) => o.id === s.measurement);
|
|
594
|
+
if (m?.role !== "headline" || !hasRun(s)) return null;
|
|
595
|
+
const at = {
|
|
596
|
+
problem: "author-check",
|
|
597
|
+
id: s.id,
|
|
598
|
+
measurement: m.id,
|
|
599
|
+
status: s.status,
|
|
600
|
+
} as const;
|
|
601
|
+
const state = planOf(row);
|
|
602
|
+
if (state === null) return { ...at, why: "no-plan" };
|
|
603
|
+
const wrong =
|
|
604
|
+
state.kind === "ready" || state.kind === "draft"
|
|
605
|
+
? checkProblem(r, state.plan)
|
|
606
|
+
: null;
|
|
607
|
+
return wrong === null ? null : { ...at, ...wrong };
|
|
608
|
+
};
|
|
609
|
+
|
|
610
|
+
/** What a read plan's author check lacks: it is not there, or it is not the registry's author's. */
|
|
611
|
+
const checkProblem = (
|
|
612
|
+
r: StudyRegistry,
|
|
613
|
+
plan: StudyPlan,
|
|
614
|
+
):
|
|
615
|
+
| { readonly why: "missing" }
|
|
616
|
+
| { readonly why: "not-author"; readonly by: string; readonly author: string }
|
|
617
|
+
| null => {
|
|
618
|
+
const check = plan.author_check;
|
|
619
|
+
if (check === null) return { why: "missing" };
|
|
620
|
+
return r.author === undefined || check.by === r.author
|
|
621
|
+
? null
|
|
622
|
+
: { why: "not-author", by: check.by, author: r.author };
|
|
623
|
+
};
|
|
624
|
+
|
|
286
625
|
/** The checks of a row, in the order their findings are reported. */
|
|
287
626
|
const ROW_CHECKS: readonly RowCheck[] = [
|
|
288
627
|
badId,
|
|
@@ -294,8 +633,58 @@ const ROW_CHECKS: readonly RowCheck[] = [
|
|
|
294
633
|
noResult,
|
|
295
634
|
resultFile,
|
|
296
635
|
noSuccessor,
|
|
636
|
+
unknownMeasurement,
|
|
637
|
+
badPlan,
|
|
638
|
+
legacyPlan,
|
|
639
|
+
authorCheck,
|
|
297
640
|
];
|
|
298
641
|
|
|
642
|
+
/** A headline measurement's approval: missing, or not the registry's author's; null when it stands. */
|
|
643
|
+
const headlineApproval = (
|
|
644
|
+
r: StudyRegistry,
|
|
645
|
+
m: Measurement,
|
|
646
|
+
): StudyFinding | null => {
|
|
647
|
+
if (m.role !== "headline") return null;
|
|
648
|
+
const author = r.author ?? null;
|
|
649
|
+
if (m.approval === undefined)
|
|
650
|
+
return {
|
|
651
|
+
problem: "headline-approval",
|
|
652
|
+
measurement: m.id,
|
|
653
|
+
why: "missing",
|
|
654
|
+
author,
|
|
655
|
+
};
|
|
656
|
+
return author === null || m.approval.by === author
|
|
657
|
+
? null
|
|
658
|
+
: {
|
|
659
|
+
problem: "headline-approval",
|
|
660
|
+
measurement: m.id,
|
|
661
|
+
why: "not-author",
|
|
662
|
+
by: m.approval.by,
|
|
663
|
+
author,
|
|
664
|
+
};
|
|
665
|
+
};
|
|
666
|
+
|
|
667
|
+
/** What is wrong with one measurement: no study, a requirement not held, a headline not approved. */
|
|
668
|
+
const judgeMeasurement = (
|
|
669
|
+
r: StudyRegistry,
|
|
670
|
+
m: Measurement,
|
|
671
|
+
): readonly StudyFinding[] => {
|
|
672
|
+
const approval = headlineApproval(r, m);
|
|
673
|
+
return [
|
|
674
|
+
...(r.studies.some((s) => s.measurement === m.id)
|
|
675
|
+
? []
|
|
676
|
+
: [{ problem: "empty-measurement", measurement: m.id } as const]),
|
|
677
|
+
...(m.requirements ?? [])
|
|
678
|
+
.filter((id) => !(r.requirements ?? []).some((q) => q.id === id))
|
|
679
|
+
.map((requirement): StudyFinding => ({
|
|
680
|
+
problem: "unknown-requirement",
|
|
681
|
+
measurement: m.id,
|
|
682
|
+
requirement,
|
|
683
|
+
})),
|
|
684
|
+
...(approval === null ? [] : [approval]),
|
|
685
|
+
];
|
|
686
|
+
};
|
|
687
|
+
|
|
299
688
|
/** What is wrong with a row, in a fixed order. */
|
|
300
689
|
const judgeRow = (row: Row): readonly StudyFinding[] =>
|
|
301
690
|
ROW_CHECKS.map((check) => check(row)).filter(
|
|
@@ -339,6 +728,7 @@ export function judgeStudies(input: StudyInput): readonly StudyFinding[] {
|
|
|
339
728
|
next,
|
|
340
729
|
})),
|
|
341
730
|
...r.studies.flatMap((s, i) => judgeRow({ r, s, i, input })),
|
|
731
|
+
...(r.measurements ?? []).flatMap((m) => judgeMeasurement(r, m)),
|
|
342
732
|
...(index === null ? [] : [index]),
|
|
343
733
|
];
|
|
344
734
|
}
|
|
@@ -466,6 +856,14 @@ const RENAME =
|
|
|
466
856
|
|
|
467
857
|
const tick = (s: string): string => `\`${s}\``;
|
|
468
858
|
|
|
859
|
+
/** `a`, `b` and `c`: names in backticks, the last joined by "and". */
|
|
860
|
+
const listed = (names: readonly string[]): string => {
|
|
861
|
+
const t = names.map(tick);
|
|
862
|
+
return [t.slice(0, -1).join(", "), ...t.slice(-1)]
|
|
863
|
+
.filter((part) => part !== "")
|
|
864
|
+
.join(" and ");
|
|
865
|
+
};
|
|
866
|
+
|
|
469
867
|
/** Each finding by its problem: the map the messages are looked up in. */
|
|
470
868
|
type FindingOf = {
|
|
471
869
|
readonly [F in StudyFinding as F["problem"]]: F;
|
|
@@ -506,15 +904,58 @@ const HEADS: { readonly [P in Problem]: (f: FindingOf[P]) => string } = {
|
|
|
506
904
|
"no-successor": (f) =>
|
|
507
905
|
`${REGISTRY}: ${tick(f.id)} is \`superseded\` ${f.superseded_by === null ? "without `superseded_by`" : `by ${tick(f.superseded_by)}, which is no study in the registry`}. What to do: name the id of the study that replaced it.`,
|
|
508
906
|
"stale-index": (f) => `${STALE_HEAD[f.why]} ${RUN_INDEX}`,
|
|
907
|
+
"unknown-measurement": (f) =>
|
|
908
|
+
`${REGISTRY}: the row ${tick(f.id)} names the measurement ${tick(f.measurement)}, which is not in \`measurements\`. What to do: add the measurement to \`measurements\`, or correct the row's \`measurement\`.`,
|
|
909
|
+
"empty-measurement": (f) =>
|
|
910
|
+
`${REGISTRY}: no row names the measurement ${tick(f.measurement)}, and a measurement is made of studies. What to do: set \`"measurement": "${f.measurement}"\` on the rows of its studies, or remove the measurement.`,
|
|
911
|
+
"unknown-requirement": (f) =>
|
|
912
|
+
`${REGISTRY}: the measurement ${tick(f.measurement)} must meet the requirement ${tick(f.requirement)}, which is not in \`requirements\`. What to do: add the requirement — the author's words verbatim in \`text\`, with its \`date\` and \`source\` — or correct the measurement's \`requirements\`.`,
|
|
913
|
+
"headline-approval": (f) =>
|
|
914
|
+
f.why === "missing"
|
|
915
|
+
? `${REGISTRY}: the measurement ${tick(f.measurement)} is a \`headline\` one and has no \`approval\`. What to do: record ${f.author === null ? "the author's approval" : `the approval of ${f.author}, the registry's \`author\``} — \`by\`, \`date\`, and their words in \`quote\` — or make it a \`baseline\`.`
|
|
916
|
+
: `${REGISTRY}: the measurement ${tick(f.measurement)} is a \`headline\` one, and its \`approval\` is by ${f.by}, not by the registry's \`author\`, ${f.author}. What to do: only the author approves a headline measurement; record their own approval.`,
|
|
917
|
+
"bad-plan": (f) => {
|
|
918
|
+
const of = `the \`plan\` of ${tick(f.id)}, ${tick(f.path)},`;
|
|
919
|
+
const plan = `The plan of ${tick(f.id)}, ${tick(f.path)} in its folder,`;
|
|
920
|
+
switch (f.why) {
|
|
921
|
+
case "missing":
|
|
922
|
+
return `${REGISTRY}: ${of} is not a file in its folder. What to do: write the plan there (its fields are on the rule's page), or correct the row's \`plan\`.`;
|
|
923
|
+
case "outside":
|
|
924
|
+
return `${REGISTRY}: ${of} is outside its folder. What to do: keep the plan in the study's folder, as \`${PLAN_FILE}\`, and name it relative to the folder.`;
|
|
925
|
+
case "unreadable":
|
|
926
|
+
return `${plan} is not a study plan: ${f.reasons}. What to do: give it the fields on the rule's page — \`question\`, \`population\`, \`arms\`, \`primary_outcome\`.`;
|
|
927
|
+
case "unfilled":
|
|
928
|
+
return `${plan} leaves ${listed(f.fields)} empty, and the study is ${tick(f.status)}: a plan is written before the runs. What to do: fill them in, or set the study back to \`designing\`.`;
|
|
929
|
+
}
|
|
930
|
+
},
|
|
931
|
+
"author-check": (f) => {
|
|
932
|
+
const head = `${tick(f.id)} is ${tick(f.status)} in the headline measurement ${tick(f.measurement)}, and`;
|
|
933
|
+
switch (f.why) {
|
|
934
|
+
case "no-plan":
|
|
935
|
+
return `${head} its row names no plan to hold the author's check of the drawn units. What to do: write its plan (\`${PLAN_FILE}\` in its folder, named in the row's \`plan\`) with \`author_check\` — \`by\`, \`date\`, and the author's words in \`quote\`.`;
|
|
936
|
+
case "missing":
|
|
937
|
+
return `${head} its plan has no \`author_check\`. What to do: show the author the drawn units, then record their answer in the plan's \`author_check\` — \`by\`, \`date\`, and their words in \`quote\`.`;
|
|
938
|
+
case "not-author":
|
|
939
|
+
return `${head} the \`author_check\` in its plan is by ${f.by}, not by the registry's \`author\`, ${f.author}. What to do: only the author approves the drawn units of a headline measurement; record their own answer.`;
|
|
940
|
+
}
|
|
941
|
+
},
|
|
942
|
+
"legacy-plan": (f) =>
|
|
943
|
+
`The plan of ${tick(f.id)} is ${tick(f.path)}, its old name; it is read as the plan. What to do: rename it to \`${PLAN_FILE}\` in the study's folder (\`git mv\`) and set the row's \`plan\` to \`${PLAN_FILE}\`.`,
|
|
509
944
|
};
|
|
510
945
|
|
|
511
946
|
/** The head of finding `f` of problem `p`: the lookup, typed by the problem. */
|
|
512
947
|
const headOf = <P extends Problem>(p: P, f: FindingOf[P]): string =>
|
|
513
948
|
HEADS[p](f);
|
|
514
949
|
|
|
950
|
+
/** The rule that reports finding `f`. */
|
|
951
|
+
const ruleOf = (f: StudyFinding): string =>
|
|
952
|
+
PLAN_NAME_PROBLEMS.includes(f.problem)
|
|
953
|
+
? "repro/study-plan-name"
|
|
954
|
+
: "repro/study-registered";
|
|
955
|
+
|
|
515
956
|
/** The one text a finding is reported with: what is wrong, what to do, and the rule's page. */
|
|
516
957
|
export const studyMessage = (f: StudyFinding): string =>
|
|
517
|
-
`${headOf(f.problem, f)}\nRule
|
|
958
|
+
`${headOf(f.problem, f)}\nRule ${ruleOf(f)}: https://github.com/zernie/paperlint/blob/main/docs/rules/${ruleOf(f)}.md`;
|
|
518
959
|
|
|
519
960
|
// ── a new study ───────────────────────────────────────────────────────────────────
|
|
520
961
|
|
|
@@ -541,20 +982,20 @@ function titleProblem(title: string): string | null {
|
|
|
541
982
|
: null;
|
|
542
983
|
}
|
|
543
984
|
|
|
544
|
-
/**
|
|
545
|
-
|
|
546
|
-
* `in_paper: no` and an empty question; or why the title cannot make one.
|
|
547
|
-
*/
|
|
548
|
-
export function addStudy(
|
|
549
|
-
r: StudyRegistry,
|
|
550
|
-
n: NewStudy,
|
|
551
|
-
):
|
|
985
|
+
/** A registry with a study added, and the study; or why it could not be. */
|
|
986
|
+
export type Added =
|
|
552
987
|
| {
|
|
553
988
|
readonly ok: true;
|
|
554
989
|
readonly registry: StudyRegistry;
|
|
555
990
|
readonly study: Study;
|
|
556
991
|
}
|
|
557
|
-
| { readonly ok: false; readonly why: string }
|
|
992
|
+
| { readonly ok: false; readonly why: string };
|
|
993
|
+
|
|
994
|
+
/**
|
|
995
|
+
* `r` with a new study appended: the next id, the folder `<id>-<slug>`, `status: designing`,
|
|
996
|
+
* `in_paper: no`, an empty question and its plan, `plan.json`; or why the title cannot make one.
|
|
997
|
+
*/
|
|
998
|
+
export function addStudy(r: StudyRegistry, n: NewStudy): Added {
|
|
558
999
|
const title = n.title.trim().replace(/\s+/gu, " ");
|
|
559
1000
|
const why = titleProblem(title);
|
|
560
1001
|
if (why !== null) return { ok: false, why };
|
|
@@ -568,6 +1009,7 @@ export function addStudy(
|
|
|
568
1009
|
status: "designing",
|
|
569
1010
|
date: n.date,
|
|
570
1011
|
in_paper: "no",
|
|
1012
|
+
plan: PLAN_FILE,
|
|
571
1013
|
};
|
|
572
1014
|
return {
|
|
573
1015
|
ok: true,
|
|
@@ -589,6 +1031,10 @@ export const studyReadme = (s: Study): string =>
|
|
|
589
1031
|
`Its row in \`../${STUDIES_FILE}\` holds its status. When the study ends, set \`status\`, \`result\`,`,
|
|
590
1032
|
"`result_file` and `in_paper` there, then run `npx paperlint studies index <paper-dir>`.",
|
|
591
1033
|
"",
|
|
1034
|
+
`Its plan is \`${PLAN_FILE}\`: what it measures, the population and the draw, the arms and the primary`,
|
|
1035
|
+
"outcome, filled in before the first run; the author's check of the drawn units goes in its",
|
|
1036
|
+
"`author_check`.",
|
|
1037
|
+
"",
|
|
592
1038
|
"## Question",
|
|
593
1039
|
"",
|
|
594
1040
|
"## How to run",
|
package/src/new-paper.test.ts
CHANGED
|
@@ -793,6 +793,7 @@ describe("paperlint lint — a scaffolded paper has no layout finding", () => {
|
|
|
793
793
|
"paper/allowed-files-only",
|
|
794
794
|
"repro/allowed-files-only",
|
|
795
795
|
"repro/study-registered",
|
|
796
|
+
"repro/study-plan-name",
|
|
796
797
|
"review/frontmatter",
|
|
797
798
|
"sibling/frontmatter",
|
|
798
799
|
"paper/stages",
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* How a `repro/studies.json
|
|
2
|
+
* How a `repro/studies.json`, and a study's `plan.json`, become values: its text parsed once, where it enters, against the schema
|
|
3
3
|
* of `StudyRegistry` (`src/domain/study-registry.ts`). The one implementation is
|
|
4
4
|
* `src/adapters/study-registry/` (zod); `src/cli.ts` wires it.
|
|
5
5
|
*/
|
|
6
|
-
import type { ParsedRegistry } from "../domain/study-registry.ts";
|
|
6
|
+
import type { ParsedPlan, ParsedRegistry } from "../domain/study-registry.ts";
|
|
7
7
|
|
|
8
8
|
export interface StudyRegistryCodec {
|
|
9
9
|
/** A `studies.json`'s text: the registry, or the one reason it is not one of this schema. */
|
|
10
10
|
readonly parse: (text: string) => ParsedRegistry;
|
|
11
|
+
/** A study's `plan.json` text, strict: ready, a draft naming its empty texts, or why it is no plan. */
|
|
12
|
+
readonly parsePlan: (text: string) => ParsedPlan;
|
|
13
|
+
/** A plan still named `prereg.json`, loose: the same three outcomes. */
|
|
14
|
+
readonly parseLegacyPlan: (text: string) => ParsedPlan;
|
|
11
15
|
}
|
package/src/rule-cases.test.ts
CHANGED
|
@@ -18,6 +18,8 @@ import { z } from "zod";
|
|
|
18
18
|
import { run, rulePlugins } from "./cli.ts";
|
|
19
19
|
import { sha256Hex } from "./domain/sha256.ts";
|
|
20
20
|
import {
|
|
21
|
+
planTemplate,
|
|
22
|
+
planText,
|
|
21
23
|
registryText,
|
|
22
24
|
withIndex,
|
|
23
25
|
type StudyRegistry,
|
|
@@ -66,6 +68,20 @@ const REGISTERED_STUDY: Readonly<Record<string, string>> = {
|
|
|
66
68
|
[`${P}/repro/P1-S01-ablation/README.md`]: "# ablation\n",
|
|
67
69
|
};
|
|
68
70
|
|
|
71
|
+
/** The registered study with its plan named `name`, the plan's text beside its README. */
|
|
72
|
+
const STUDY_WITH_PLAN = (name: string): Readonly<Record<string, string>> => {
|
|
73
|
+
const r: StudyRegistry = {
|
|
74
|
+
...REGISTRY,
|
|
75
|
+
studies: REGISTRY.studies.map((s) => ({ ...s, plan: name })),
|
|
76
|
+
};
|
|
77
|
+
return {
|
|
78
|
+
[`${P}/repro/studies.json`]: registryText(r),
|
|
79
|
+
[`${P}/repro/README.md`]: withIndex("# repro\n", r),
|
|
80
|
+
[`${P}/repro/P1-S01-ablation/README.md`]: "# ablation\n",
|
|
81
|
+
[`${P}/repro/P1-S01-ablation/${name}`]: planText(planTemplate()),
|
|
82
|
+
};
|
|
83
|
+
};
|
|
84
|
+
|
|
69
85
|
/** `study/declare-sample` is off by default; a paper turns it on in its `paperlint.json`. */
|
|
70
86
|
const DECLARE_SAMPLE_ON: Readonly<Record<string, string>> = {
|
|
71
87
|
[`${P}/paperlint.json`]: JSON.stringify({
|
|
@@ -771,6 +787,17 @@ const CASES: Readonly<Record<string, RuleCases>> = {
|
|
|
771
787
|
// Every folder registered, and the index the registry generates.
|
|
772
788
|
silent: paper(tex("x"), REGISTERED_STUDY),
|
|
773
789
|
},
|
|
790
|
+
// A study's plan still named prereg.json: read as its plan, and a warning to rename it.
|
|
791
|
+
"repro/study-plan-name": {
|
|
792
|
+
reports: {
|
|
793
|
+
tree: paper(tex("x"), STUDY_WITH_PLAN("prereg.json")),
|
|
794
|
+
file: `${P}/PIPELINE-STATUS.md`,
|
|
795
|
+
severity: 1,
|
|
796
|
+
line: 1,
|
|
797
|
+
},
|
|
798
|
+
// The same plan as plan.json.
|
|
799
|
+
silent: paper(tex("x"), STUDY_WITH_PLAN("plan.json")),
|
|
800
|
+
},
|
|
774
801
|
// A study's prereg.json that declares none of population, strata, screens, coverage, author_check.
|
|
775
802
|
// Off by default: the paper turns it on, as both trees do.
|
|
776
803
|
"study/declare-sample": {
|
package/src/studies-cli.test.ts
CHANGED
|
@@ -83,7 +83,7 @@ describe("paperlint studies index — through the CLI", () => {
|
|
|
83
83
|
writeTree(repro, {
|
|
84
84
|
"studies.json": readFileSync(file, "utf8").replace(
|
|
85
85
|
'"designing"',
|
|
86
|
-
'"
|
|
86
|
+
'"dropped"',
|
|
87
87
|
),
|
|
88
88
|
});
|
|
89
89
|
expect(await registryFindings(dir)).toEqual([
|
|
@@ -96,7 +96,7 @@ describe("paperlint studies index — through the CLI", () => {
|
|
|
96
96
|
[],
|
|
97
97
|
]);
|
|
98
98
|
expect(readFileSync(join(repro, "README.md"), "utf8")).toContain(
|
|
99
|
-
"| P1-S01 | Baseline pass rate |
|
|
99
|
+
"| P1-S01 | Baseline pass rate | dropped |",
|
|
100
100
|
);
|
|
101
101
|
expect(await registryFindings(dir)).toEqual([]);
|
|
102
102
|
});
|