paperlint 10.2.0 → 10.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +15 -4
- package/README.md +1 -0
- package/dist/adapters/git/index.d.ts +10 -0
- package/dist/adapters/git/index.d.ts.map +1 -0
- package/dist/adapters/git/index.js +151 -0
- package/dist/adapters/git/index.js.map +1 -0
- package/dist/adapters/memory/index.d.ts +6 -0
- package/dist/adapters/memory/index.d.ts.map +1 -1
- package/dist/adapters/memory/index.js +8 -0
- package/dist/adapters/memory/index.js.map +1 -1
- package/dist/adapters/node/index.d.ts +1 -0
- package/dist/adapters/node/index.d.ts.map +1 -1
- package/dist/adapters/node/index.js +1 -0
- package/dist/adapters/node/index.js.map +1 -1
- package/dist/adapters/node/list-entries.io.d.ts +3 -0
- package/dist/adapters/node/list-entries.io.d.ts.map +1 -0
- package/dist/adapters/node/list-entries.io.js +24 -0
- package/dist/adapters/node/list-entries.io.js.map +1 -0
- package/dist/adapters/study-registry/index.d.ts +412 -1
- package/dist/adapters/study-registry/index.d.ts.map +1 -1
- package/dist/adapters/study-registry/index.js +245 -8
- package/dist/adapters/study-registry/index.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +38 -6
- package/dist/cli.js.map +1 -1
- package/dist/domain/decide-first.d.ts +100 -0
- package/dist/domain/decide-first.d.ts.map +1 -0
- package/dist/domain/decide-first.js +102 -0
- package/dist/domain/decide-first.js.map +1 -0
- package/dist/domain/index-columns.d.ts +45 -0
- package/dist/domain/index-columns.d.ts.map +1 -0
- package/dist/domain/index-columns.js +62 -0
- package/dist/domain/index-columns.js.map +1 -0
- package/dist/domain/study-mode.d.ts +37 -0
- package/dist/domain/study-mode.d.ts.map +1 -0
- package/dist/domain/study-mode.js +45 -0
- package/dist/domain/study-mode.js.map +1 -0
- package/dist/domain/study-registry.d.ts +253 -10
- package/dist/domain/study-registry.d.ts.map +1 -1
- package/dist/domain/study-registry.js +325 -27
- package/dist/domain/study-registry.js.map +1 -1
- package/dist/outputs-before-decision-rule.d.ts +17 -0
- package/dist/outputs-before-decision-rule.d.ts.map +1 -0
- package/dist/outputs-before-decision-rule.js +90 -0
- package/dist/outputs-before-decision-rule.js.map +1 -0
- package/dist/ports/git-history.d.ts +44 -0
- package/dist/ports/git-history.d.ts.map +1 -0
- package/dist/ports/git-history.js +2 -0
- package/dist/ports/git-history.js.map +1 -0
- package/dist/ports/list-entries.d.ts +17 -0
- package/dist/ports/list-entries.d.ts.map +1 -0
- package/dist/ports/list-entries.js +2 -0
- package/dist/ports/list-entries.js.map +1 -0
- package/dist/ports/study-registry-codec.d.ts +9 -2
- package/dist/ports/study-registry-codec.d.ts.map +1 -1
- package/dist/studies.d.ts +4 -0
- package/dist/studies.d.ts.map +1 -1
- package/dist/studies.js +55 -24
- package/dist/studies.js.map +1 -1
- package/dist/study-gate.d.ts +14 -0
- package/dist/study-gate.d.ts.map +1 -0
- package/dist/study-gate.js +95 -0
- package/dist/study-gate.js.map +1 -0
- package/dist/study-index-input.d.ts +5 -0
- package/dist/study-index-input.d.ts.map +1 -0
- package/dist/study-index-input.js +13 -0
- package/dist/study-index-input.js.map +1 -0
- package/dist/study-modes.d.ts +45 -0
- package/dist/study-modes.d.ts.map +1 -0
- package/dist/study-modes.js +70 -0
- package/dist/study-modes.js.map +1 -0
- package/dist/study-registry-rule.d.ts +29 -6
- package/dist/study-registry-rule.d.ts.map +1 -1
- package/dist/study-registry-rule.js +91 -40
- package/dist/study-registry-rule.js.map +1 -1
- package/dist/study-run.d.ts +19 -0
- package/dist/study-run.d.ts.map +1 -0
- package/dist/study-run.js +9 -0
- package/dist/study-run.js.map +1 -0
- package/docs/design/study-modes.md +247 -0
- package/docs/design/study-registry.md +137 -15
- package/docs/e2e.md +8 -4
- package/docs/layout.md +26 -21
- package/docs/rules/repro/outputs-before-decision.md +119 -0
- package/docs/rules/repro/study-plan-name.md +70 -0
- package/docs/rules/repro/study-registered.md +206 -15
- package/docs/rules.md +54 -52
- package/docs/testing.md +1 -0
- package/package.json +2 -1
- package/scripts/check.ts +6 -0
- package/skills/build-benchmark/SKILL.md +24 -12
- package/skills/build-benchmark/SKILL.md.spec.ts +23 -11
- package/skills/build-label-page/SKILL.md +9 -7
- package/skills/build-label-page/SKILL.md.spec.ts +8 -6
- package/skills/camera-ready/SKILL.md +6 -1
- package/skills/camera-ready/SKILL.md.spec.ts +5 -0
- package/skills/draft-paper/SKILL.md +11 -3
- package/skills/draft-paper/SKILL.md.spec.ts +10 -2
- package/skills/harden-paper/SKILL.md +12 -2
- package/skills/harden-paper/SKILL.md.spec.ts +11 -1
- package/skills/osf-artifact-upload/SKILL.md +7 -1
- package/skills/osf-artifact-upload/SKILL.md.spec.ts +6 -0
- package/skills/pc-panel-review/SKILL.md +10 -3
- package/skills/pc-panel-review/SKILL.md.spec.ts +9 -2
- package/skills/run-agent-grid/SKILL.md +10 -5
- package/skills/run-agent-grid/SKILL.md.spec.ts +9 -4
- package/skills/tighten-paper/SKILL.md +5 -1
- package/skills/tighten-paper/SKILL.md.spec.ts +4 -0
- package/src/adapters/git/history.test.ts +121 -0
- package/src/adapters/git/index.test.ts +268 -0
- package/src/adapters/git/index.ts +227 -0
- package/src/adapters/memory/index.test.ts +15 -1
- package/src/adapters/memory/index.ts +13 -0
- package/src/adapters/node/index.ts +1 -0
- package/src/adapters/node/list-entries.io.ts +26 -0
- package/src/adapters/node/list-entries.test.ts +43 -0
- package/src/adapters/study-registry/index.test.ts +343 -0
- package/src/adapters/study-registry/index.ts +374 -10
- package/src/adapters/study-registry/mode.test.ts +162 -0
- package/src/cli.ts +52 -6
- package/src/domain/decide-first.test.ts +258 -0
- package/src/domain/decide-first.ts +243 -0
- package/src/domain/index-columns.test.ts +216 -0
- package/src/domain/index-columns.ts +105 -0
- package/src/domain/study-measurements.test.ts +633 -0
- package/src/domain/study-mode.test.ts +55 -0
- package/src/domain/study-mode.ts +78 -0
- package/src/domain/study-registry.test.ts +48 -32
- package/src/domain/study-registry.ts +621 -36
- package/src/new-paper.test.ts +6 -1
- package/src/outputs-before-decision-rule.test.ts +206 -0
- package/src/outputs-before-decision-rule.ts +124 -0
- package/src/ports/git-history.ts +44 -0
- package/src/ports/list-entries.ts +18 -0
- package/src/ports/study-registry-codec.ts +13 -2
- package/src/rule-cases.test.ts +103 -1
- package/src/studies-cli.test.ts +2 -2
- package/src/studies-index-modes.test.ts +129 -0
- package/src/studies.test.ts +39 -9
- package/src/studies.ts +99 -29
- package/src/study-gate.test.ts +316 -0
- package/src/study-gate.ts +161 -0
- package/src/study-index-input.ts +23 -0
- package/src/study-modes.ts +120 -0
- package/src/study-registry-rule.test.ts +333 -7
- package/src/study-registry-rule.ts +145 -41
- package/src/study-run.ts +21 -0
|
@@ -6,14 +6,22 @@
|
|
|
6
6
|
* on the first line of `PIPELINE-STATUS.md`, with the domain's text.
|
|
7
7
|
*/
|
|
8
8
|
import { describe, expect, it } from "vitest";
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
memoryFiles,
|
|
11
|
+
memoryListDir,
|
|
12
|
+
memoryListEntries,
|
|
13
|
+
} from "./adapters/memory/index.ts";
|
|
10
14
|
import { studyRegistryCodec } from "./adapters/study-registry/index.ts";
|
|
11
15
|
import {
|
|
16
|
+
planTemplate,
|
|
17
|
+
planText,
|
|
12
18
|
registryText,
|
|
13
19
|
studyMessage,
|
|
14
20
|
withIndex,
|
|
15
21
|
type Study,
|
|
22
|
+
type StudyPlan,
|
|
16
23
|
type StudyRegistry,
|
|
24
|
+
type IndexInput,
|
|
17
25
|
} from "./domain/study-registry.ts";
|
|
18
26
|
import {
|
|
19
27
|
registryOf,
|
|
@@ -22,6 +30,9 @@ import {
|
|
|
22
30
|
studyRegistryRules,
|
|
23
31
|
} from "./study-registry-rule.ts";
|
|
24
32
|
|
|
33
|
+
/** No plan and no history: the index with every mode column `—`. */
|
|
34
|
+
const NO_FACTS: IndexInput = { plans: new Map(), history: new Map() };
|
|
35
|
+
|
|
25
36
|
const DIR = "/work/papers/example";
|
|
26
37
|
const SCAFFOLD = {
|
|
27
38
|
[`${DIR}/PIPELINE-STATUS.md`]: "# status\n",
|
|
@@ -43,7 +54,7 @@ const study = (over: Partial<Study> = {}): Study => ({
|
|
|
43
54
|
/** A registered `repro/`: the registry, its index, and a README.md in each row's folder. */
|
|
44
55
|
const registered = (
|
|
45
56
|
r: StudyRegistry,
|
|
46
|
-
readme: string | null = withIndex("# Reproduction index\n", r),
|
|
57
|
+
readme: string | null = withIndex("# Reproduction index\n", r, NO_FACTS),
|
|
47
58
|
): Record<string, string> => ({
|
|
48
59
|
...SCAFFOLD,
|
|
49
60
|
[`${DIR}/repro/studies.json`]: registryText(r),
|
|
@@ -65,13 +76,15 @@ interface Reported {
|
|
|
65
76
|
function lint(
|
|
66
77
|
tree: Readonly<Record<string, string>>,
|
|
67
78
|
file = `${DIR}/PIPELINE-STATUS.md`,
|
|
79
|
+
name: "study-registered" | "study-plan-name" = "study-registered",
|
|
68
80
|
): Reported[] {
|
|
69
81
|
const files = memoryFiles(tree);
|
|
70
82
|
const rule = studyRegistryRules({
|
|
71
83
|
files,
|
|
72
84
|
listDir: memoryListDir(files),
|
|
85
|
+
entries: memoryListEntries(files),
|
|
73
86
|
codec: studyRegistryCodec,
|
|
74
|
-
})[
|
|
87
|
+
})[name];
|
|
75
88
|
const out: Reported[] = [];
|
|
76
89
|
const visitor = rule.create({
|
|
77
90
|
filename: file,
|
|
@@ -219,9 +232,9 @@ describe("repro/study-registered — reports a row's status and the index", () =
|
|
|
219
232
|
|
|
220
233
|
it("a stale index", () => {
|
|
221
234
|
const r = P1(study());
|
|
222
|
-
expect(messages(registered(r, withIndex("# x\n", P1())))).toEqual(
|
|
223
|
-
studyMessage({ problem: "stale-index", why: "differs" }),
|
|
224
|
-
|
|
235
|
+
expect(messages(registered(r, withIndex("# x\n", P1(), NO_FACTS)))).toEqual(
|
|
236
|
+
[studyMessage({ problem: "stale-index", why: "differs" })],
|
|
237
|
+
);
|
|
225
238
|
});
|
|
226
239
|
});
|
|
227
240
|
|
|
@@ -271,6 +284,7 @@ describe("repro/study-registered — level", () => {
|
|
|
271
284
|
it("is an error from the start", () => {
|
|
272
285
|
expect(STUDY_REGISTRY_RULE_LEVELS).toEqual({
|
|
273
286
|
"repro/study-registered": "error",
|
|
287
|
+
"repro/study-plan-name": "warn",
|
|
274
288
|
});
|
|
275
289
|
});
|
|
276
290
|
});
|
|
@@ -296,7 +310,7 @@ describe("repro/study-registered — a result_file the disk cannot hold", () =>
|
|
|
296
310
|
const files = memoryFiles({
|
|
297
311
|
...SCAFFOLD,
|
|
298
312
|
[`${DIR}/repro/studies.json`]: registryText(r),
|
|
299
|
-
[`${DIR}/repro/README.md`]: withIndex("# index\n", r),
|
|
313
|
+
[`${DIR}/repro/README.md`]: withIndex("# index\n", r, NO_FACTS),
|
|
300
314
|
[`${DIR}/repro/P1-S01-baseline/README.md`]: "# s\n",
|
|
301
315
|
});
|
|
302
316
|
const probe = {
|
|
@@ -310,8 +324,320 @@ describe("repro/study-registered — a result_file the disk cannot hold", () =>
|
|
|
310
324
|
studyFindings(DIR, {
|
|
311
325
|
files: probe,
|
|
312
326
|
listDir: memoryListDir(files),
|
|
327
|
+
entries: memoryListEntries(files),
|
|
313
328
|
codec: studyRegistryCodec,
|
|
314
329
|
}).map((f) => f.problem),
|
|
315
330
|
).toEqual(["result-file"]);
|
|
316
331
|
});
|
|
317
332
|
});
|
|
333
|
+
|
|
334
|
+
describe("repro/study-registered — a study folder that is a symbolic link", () => {
|
|
335
|
+
it("never reads a plan or a result through it", () => {
|
|
336
|
+
const r = P1(
|
|
337
|
+
study({
|
|
338
|
+
status: "done",
|
|
339
|
+
result: "41 of 60",
|
|
340
|
+
result_file: "r.txt",
|
|
341
|
+
plan: "plan.json",
|
|
342
|
+
}),
|
|
343
|
+
);
|
|
344
|
+
const files = memoryFiles({
|
|
345
|
+
...registered(r),
|
|
346
|
+
[`${DIR}/repro/P1-S01-baseline/plan.json`]: "not JSON",
|
|
347
|
+
[`${DIR}/repro/P1-S01-baseline/r.txt`]: "41 of 60\n",
|
|
348
|
+
});
|
|
349
|
+
const found = studyFindings(DIR, {
|
|
350
|
+
files,
|
|
351
|
+
listDir: memoryListDir(files),
|
|
352
|
+
entries: (dir) =>
|
|
353
|
+
memoryListEntries(files)(dir).map((e) =>
|
|
354
|
+
e.name === "P1-S01-baseline" ? { ...e, kind: "symlink" as const } : e,
|
|
355
|
+
),
|
|
356
|
+
codec: studyRegistryCodec,
|
|
357
|
+
});
|
|
358
|
+
// Guards: the files behind the link are not read — the plan and the result read as missing.
|
|
359
|
+
expect(
|
|
360
|
+
found.map((f) =>
|
|
361
|
+
f.problem === "bad-plan" || f.problem === "result-file"
|
|
362
|
+
? `${f.problem}:${f.why}`
|
|
363
|
+
: f.problem,
|
|
364
|
+
),
|
|
365
|
+
).toEqual(["result-file:missing", "bad-plan:missing"]);
|
|
366
|
+
});
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
describe("repro/study-registered — a row whose folder leads outside repro/", () => {
|
|
370
|
+
it("is reported by name, and nothing is read through it: its plan and result read as missing", () => {
|
|
371
|
+
const r = P1(
|
|
372
|
+
study({
|
|
373
|
+
folder: "../outside",
|
|
374
|
+
status: "done",
|
|
375
|
+
result: "41 of 60",
|
|
376
|
+
result_file: "r.txt",
|
|
377
|
+
plan: "plan.json",
|
|
378
|
+
}),
|
|
379
|
+
);
|
|
380
|
+
const files = memoryFiles({
|
|
381
|
+
...SCAFFOLD,
|
|
382
|
+
[`${DIR}/repro/studies.json`]: registryText(r),
|
|
383
|
+
[`${DIR}/repro/README.md`]: withIndex("# index\n", r, NO_FACTS),
|
|
384
|
+
[`${DIR}/outside/plan.json`]: "not JSON",
|
|
385
|
+
[`${DIR}/outside/r.txt`]: "41 of 60\n",
|
|
386
|
+
});
|
|
387
|
+
const read: string[] = [];
|
|
388
|
+
const spy = {
|
|
389
|
+
...files,
|
|
390
|
+
readBytes: (p: Parameters<typeof files.readBytes>[0]) => {
|
|
391
|
+
read.push(p);
|
|
392
|
+
return files.readBytes(p);
|
|
393
|
+
},
|
|
394
|
+
isFile: (p: Parameters<typeof files.isFile>[0]): boolean => {
|
|
395
|
+
read.push(p);
|
|
396
|
+
return files.isFile(p);
|
|
397
|
+
},
|
|
398
|
+
};
|
|
399
|
+
const found = studyFindings(DIR, {
|
|
400
|
+
files: spy,
|
|
401
|
+
listDir: memoryListDir(files),
|
|
402
|
+
entries: memoryListEntries(files),
|
|
403
|
+
codec: studyRegistryCodec,
|
|
404
|
+
});
|
|
405
|
+
// Guards: a row naming `../outside` never has a file outside `repro/` read for it.
|
|
406
|
+
expect([
|
|
407
|
+
found.map((f) =>
|
|
408
|
+
f.problem === "bad-plan" || f.problem === "result-file"
|
|
409
|
+
? `${f.problem}:${f.why}`
|
|
410
|
+
: f.problem,
|
|
411
|
+
),
|
|
412
|
+
read.filter((p) => p.includes("/outside/")),
|
|
413
|
+
]).toEqual([
|
|
414
|
+
["folder-name", "no-folder", "result-file:missing", "bad-plan:missing"],
|
|
415
|
+
[],
|
|
416
|
+
]);
|
|
417
|
+
});
|
|
418
|
+
});
|
|
419
|
+
|
|
420
|
+
/** A plan as a study would hold it once the author has checked its drawn units. */
|
|
421
|
+
const FILLED_PLAN: StudyPlan = {
|
|
422
|
+
question: "How many rules can a linter check?",
|
|
423
|
+
population: {
|
|
424
|
+
source: "the crawl of 2026-10-01",
|
|
425
|
+
draw: { kind: "all", method: "census" },
|
|
426
|
+
},
|
|
427
|
+
arms: [],
|
|
428
|
+
primary_outcome: "the share of rules a linter can check",
|
|
429
|
+
author_check: { by: "A. Author", date: "2026-10-14", quote: "Go." },
|
|
430
|
+
mode: {
|
|
431
|
+
kind: "decide_before_run",
|
|
432
|
+
outputs: ["results.tsv"],
|
|
433
|
+
decision: {
|
|
434
|
+
file: "repro/P1-S01-baseline/decision.md",
|
|
435
|
+
note: "the sample approved",
|
|
436
|
+
},
|
|
437
|
+
},
|
|
438
|
+
};
|
|
439
|
+
const FILLED = planText(FILLED_PLAN);
|
|
440
|
+
|
|
441
|
+
/** The index of `r` as `studies index` writes it, FILLED_PLAN as its study's plan and `history` read. */
|
|
442
|
+
const FILLED_INDEX = (
|
|
443
|
+
r: StudyRegistry,
|
|
444
|
+
history: IndexInput["history"] = new Map(),
|
|
445
|
+
): string =>
|
|
446
|
+
withIndex("# Reproduction index\n", r, {
|
|
447
|
+
plans: new Map([
|
|
448
|
+
["P1-S01-baseline/plan.json", { kind: "ready", plan: FILLED_PLAN }],
|
|
449
|
+
]),
|
|
450
|
+
history,
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
/** A registry with a headline measurement of one running study whose row names `plan`. */
|
|
454
|
+
const HEADLINE = (plan: string): StudyRegistry => ({
|
|
455
|
+
prefix: "P1",
|
|
456
|
+
author: "A. Author",
|
|
457
|
+
measurements: [
|
|
458
|
+
{
|
|
459
|
+
id: "M1",
|
|
460
|
+
title: "Rule census",
|
|
461
|
+
role: "headline",
|
|
462
|
+
approval: { by: "A. Author", date: "2026-10-12", quote: "Yes." },
|
|
463
|
+
},
|
|
464
|
+
],
|
|
465
|
+
studies: [study({ status: "running", measurement: "M1", plan })],
|
|
466
|
+
});
|
|
467
|
+
|
|
468
|
+
describe("repro/study-registered — the plan a row names, read from its folder", () => {
|
|
469
|
+
it("is silent on a plan.json the author checked, its mode in the index", () => {
|
|
470
|
+
const r = HEADLINE("plan.json");
|
|
471
|
+
expect(
|
|
472
|
+
lint({
|
|
473
|
+
...registered(r, FILLED_INDEX(r)),
|
|
474
|
+
[`${DIR}/repro/P1-S01-baseline/plan.json`]: FILLED,
|
|
475
|
+
}),
|
|
476
|
+
).toEqual([]);
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
it("reports a plan that is not there", () => {
|
|
480
|
+
expect(messages(registered(HEADLINE("plan.json")))).toEqual([
|
|
481
|
+
studyMessage({
|
|
482
|
+
problem: "bad-plan",
|
|
483
|
+
id: "P1-S01",
|
|
484
|
+
path: "plan.json",
|
|
485
|
+
why: "missing",
|
|
486
|
+
}),
|
|
487
|
+
]);
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
it("reports a plan that is not JSON, with the reason", () => {
|
|
491
|
+
expect(
|
|
492
|
+
messages({
|
|
493
|
+
...registered(HEADLINE("plan.json")),
|
|
494
|
+
[`${DIR}/repro/P1-S01-baseline/plan.json`]: "{",
|
|
495
|
+
}),
|
|
496
|
+
).toEqual([
|
|
497
|
+
studyMessage({
|
|
498
|
+
problem: "bad-plan",
|
|
499
|
+
id: "P1-S01",
|
|
500
|
+
path: "plan.json",
|
|
501
|
+
why: "unreadable",
|
|
502
|
+
reasons: "not JSON",
|
|
503
|
+
}),
|
|
504
|
+
]);
|
|
505
|
+
});
|
|
506
|
+
|
|
507
|
+
it("reports a running study of the headline whose plan is still the template", () => {
|
|
508
|
+
expect(
|
|
509
|
+
lint({
|
|
510
|
+
...registered(HEADLINE("plan.json")),
|
|
511
|
+
[`${DIR}/repro/P1-S01-baseline/plan.json`]: planText(
|
|
512
|
+
planTemplate("spike", "x"),
|
|
513
|
+
),
|
|
514
|
+
}).map((r) => [r.messageId, r.line]),
|
|
515
|
+
).toEqual([
|
|
516
|
+
["bad-plan", 1],
|
|
517
|
+
["author-check", 1],
|
|
518
|
+
]);
|
|
519
|
+
});
|
|
520
|
+
});
|
|
521
|
+
|
|
522
|
+
describe("repro/study-registered — a plan the disk cannot hold", () => {
|
|
523
|
+
it("reads as missing, and the disk is never asked", () => {
|
|
524
|
+
const r = HEADLINE("a\u0000.json");
|
|
525
|
+
const files = memoryFiles(registered(r));
|
|
526
|
+
const probe = {
|
|
527
|
+
...files,
|
|
528
|
+
isFile: (p: Parameters<typeof files.isFile>[0]): boolean => {
|
|
529
|
+
if (p.includes("\0")) throw new Error("ERR_INVALID_ARG_VALUE");
|
|
530
|
+
return files.isFile(p);
|
|
531
|
+
},
|
|
532
|
+
};
|
|
533
|
+
expect(
|
|
534
|
+
studyFindings(DIR, {
|
|
535
|
+
files: probe,
|
|
536
|
+
listDir: memoryListDir(files),
|
|
537
|
+
entries: memoryListEntries(files),
|
|
538
|
+
codec: studyRegistryCodec,
|
|
539
|
+
}).map((f) => f.problem),
|
|
540
|
+
).toEqual(["bad-plan"]);
|
|
541
|
+
});
|
|
542
|
+
});
|
|
543
|
+
|
|
544
|
+
describe("repro/study-plan-name — a plan under its old name", () => {
|
|
545
|
+
const OLD = {
|
|
546
|
+
...registered(HEADLINE("prereg.json")),
|
|
547
|
+
[`${DIR}/repro/P1-S01-baseline/prereg.json`]: FILLED,
|
|
548
|
+
};
|
|
549
|
+
|
|
550
|
+
it("is read as the plan: repro/study-registered is silent", () => {
|
|
551
|
+
expect(lint(OLD)).toEqual([]);
|
|
552
|
+
});
|
|
553
|
+
|
|
554
|
+
it("is reported by repro/study-plan-name, naming plan.json", () => {
|
|
555
|
+
expect(lint(OLD, `${DIR}/PIPELINE-STATUS.md`, "study-plan-name")).toEqual([
|
|
556
|
+
{
|
|
557
|
+
messageId: "legacy-plan",
|
|
558
|
+
message: studyMessage({
|
|
559
|
+
problem: "legacy-plan",
|
|
560
|
+
id: "P1-S01",
|
|
561
|
+
path: "prereg.json",
|
|
562
|
+
}),
|
|
563
|
+
line: 1,
|
|
564
|
+
},
|
|
565
|
+
]);
|
|
566
|
+
});
|
|
567
|
+
|
|
568
|
+
it("is silent on plan.json, and on any file but PIPELINE-STATUS.md", () => {
|
|
569
|
+
const tree = {
|
|
570
|
+
...registered(HEADLINE("plan.json")),
|
|
571
|
+
[`${DIR}/repro/P1-S01-baseline/plan.json`]: FILLED,
|
|
572
|
+
};
|
|
573
|
+
expect([
|
|
574
|
+
lint(tree, `${DIR}/PIPELINE-STATUS.md`, "study-plan-name"),
|
|
575
|
+
lint(OLD, `${DIR}/paper.tex`, "study-plan-name"),
|
|
576
|
+
]).toEqual([[], []]);
|
|
577
|
+
});
|
|
578
|
+
});
|
|
579
|
+
|
|
580
|
+
/** A plan whose texts are filled and which declares no mode; checked by the author when `check`. */
|
|
581
|
+
const noMode = (check: boolean): string =>
|
|
582
|
+
planText({
|
|
583
|
+
question: "q",
|
|
584
|
+
population: { source: "s", draw: { kind: "all", method: "census" } },
|
|
585
|
+
arms: [],
|
|
586
|
+
primary_outcome: "o",
|
|
587
|
+
author_check: check
|
|
588
|
+
? { by: "A. Author", date: "2026-10-14", quote: "Go." }
|
|
589
|
+
: null,
|
|
590
|
+
});
|
|
591
|
+
|
|
592
|
+
/** The first line of each finding over the headline study with `plan` as its plan.json. */
|
|
593
|
+
const firstLines = (plan: string): readonly string[] =>
|
|
594
|
+
lint({
|
|
595
|
+
...registered(HEADLINE("plan.json")),
|
|
596
|
+
[`${DIR}/repro/P1-S01-baseline/plan.json`]: plan,
|
|
597
|
+
}).map((r) => r.message.split("\n")[0] ?? "");
|
|
598
|
+
|
|
599
|
+
it("repro/study-registered — a measurement's plan that has run without a mode is reported; approved without one, it is no plan", () => {
|
|
600
|
+
// Guards: decide-first is the default — a measurement that ran with no mode is reported, never silent.
|
|
601
|
+
expect([firstLines(noMode(false)), firstLines(noMode(true))]).toEqual([
|
|
602
|
+
[
|
|
603
|
+
"The plan of `P1-S01`, `plan.json` in its folder, leaves `mode` empty, and the study is `running`: a plan is written before the runs. What to do: fill them in, or set the study back to `designing`.",
|
|
604
|
+
"`P1-S01` is `running` in the headline measurement `M1`, and its plan has no `author_check`. What to do: show the author the drawn units, then record their answer in the plan's `author_check` — `by`, `date`, and their words in `quote`.",
|
|
605
|
+
],
|
|
606
|
+
[
|
|
607
|
+
"The plan of `P1-S01`, `plan.json` in its folder, is not a study plan: author_check: is set, and the plan leaves `mode` empty — the author approves a complete plan: fill them, or set author_check to null. What to do: give it the fields on the rule's page — `question`, `population`, `arms`, `primary_outcome`, and for a measurement its `mode`.",
|
|
608
|
+
],
|
|
609
|
+
]);
|
|
610
|
+
});
|
|
611
|
+
|
|
612
|
+
describe("repro/study-registered — the index's mode columns", () => {
|
|
613
|
+
it("reports an index whose mode column is not the plan's, and never compares the two read from git", () => {
|
|
614
|
+
const r = HEADLINE("plan.json");
|
|
615
|
+
const at = { commit: "a".repeat(40), date: "2026-10-14T09:00:00+05:00" };
|
|
616
|
+
const lintWith = (readme: string): string[] =>
|
|
617
|
+
lint({
|
|
618
|
+
...registered(r, readme),
|
|
619
|
+
[`${DIR}/repro/P1-S01-baseline/plan.json`]: FILLED,
|
|
620
|
+
}).map((x) => x.messageId);
|
|
621
|
+
// Guards: the mode is in the tree, so a stale one is reported; the commits are what git said when
|
|
622
|
+
// `studies index` ran — a squash merge renames them — so whatever they say is in sync.
|
|
623
|
+
expect([
|
|
624
|
+
lintWith(withIndex("# Reproduction index\n", r, NO_FACTS)),
|
|
625
|
+
lintWith(
|
|
626
|
+
FILLED_INDEX(
|
|
627
|
+
r,
|
|
628
|
+
new Map([
|
|
629
|
+
[
|
|
630
|
+
"P1-S01",
|
|
631
|
+
{
|
|
632
|
+
kind: "read",
|
|
633
|
+
decision: at,
|
|
634
|
+
decisionDeleted: false,
|
|
635
|
+
firstOutput: at,
|
|
636
|
+
},
|
|
637
|
+
],
|
|
638
|
+
]),
|
|
639
|
+
),
|
|
640
|
+
),
|
|
641
|
+
]).toEqual([["stale-index"], []]);
|
|
642
|
+
});
|
|
643
|
+
});
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* `repro/study-registered` — every study folder at the top of a paper's `repro/` has exactly one row in
|
|
3
|
-
* `repro/studies.json`, every row names a folder that exists,
|
|
4
|
-
* one the registry generates.
|
|
3
|
+
* `repro/studies.json`, every row names a folder that exists, a row's measurement and plan hold
|
|
4
|
+
* together, and the index in `repro/README.md` is the one the registry generates. Beside it,
|
|
5
|
+
* `repro/study-plan-name`, a warning: a plan still named `prereg.json`.
|
|
5
6
|
*
|
|
6
7
|
* What is judged, and why, is `src/domain/study-registry.ts` and docs/rules/repro/study-registered.md.
|
|
7
|
-
* This file lists `repro/`, reads the registry, the README
|
|
8
|
+
* This file lists `repro/`, reads the registry, the README, the result files and the plans, and reports.
|
|
8
9
|
*
|
|
9
10
|
* ── WHY A RULE OF ITS OWN, NOT MORE OF `repro/allowed-files-only` ─────────────────────
|
|
10
11
|
* `repro/allowed-files-only` judges what may stand at the top of `repro/`, by name, and accepts the
|
|
@@ -16,7 +17,7 @@
|
|
|
16
17
|
* `PIPELINE-STATUS.md`, first line, like the layout rules: every paper has it, and ESLint lints no
|
|
17
18
|
* file in `repro/`.
|
|
18
19
|
*/
|
|
19
|
-
import { basename, dirname, join } from "node:path";
|
|
20
|
+
import { basename, dirname, join, posix } from "node:path";
|
|
20
21
|
import { callerPath } from "./caller-path.ts";
|
|
21
22
|
import {
|
|
22
23
|
README,
|
|
@@ -25,9 +26,14 @@ import {
|
|
|
25
26
|
REPRO_DIRS,
|
|
26
27
|
} from "./domain/repro-layout.ts";
|
|
27
28
|
import {
|
|
29
|
+
LEGACY_PLAN_FILE,
|
|
30
|
+
PLAN_NAME_PROBLEMS,
|
|
28
31
|
STUDIES_FILE,
|
|
32
|
+
isFolderName,
|
|
29
33
|
judgeStudies,
|
|
34
|
+
planRefs,
|
|
30
35
|
studyMessage,
|
|
36
|
+
type PlanState,
|
|
31
37
|
type RegistryState,
|
|
32
38
|
type StudyFinding,
|
|
33
39
|
} from "./domain/study-registry.ts";
|
|
@@ -37,6 +43,7 @@ import type {
|
|
|
37
43
|
FolderRuleModule,
|
|
38
44
|
} from "./folder-venue-rule.ts";
|
|
39
45
|
import type { Files } from "./ports/files.ts";
|
|
46
|
+
import type { ListEntries } from "./ports/list-entries.ts";
|
|
40
47
|
import type { StudyRegistryCodec } from "./ports/study-registry-codec.ts";
|
|
41
48
|
import { reproEntriesOf } from "./repro-layout-rule.ts";
|
|
42
49
|
import { rulePageUrl } from "./venue-rules.ts";
|
|
@@ -46,10 +53,36 @@ const STATUS_FILE = "PIPELINE-STATUS.md";
|
|
|
46
53
|
/** What the rule reads with: the disk, its listings, and the registry's parser. */
|
|
47
54
|
export interface StudyRegistryDeps extends FolderLayoutDeps {
|
|
48
55
|
readonly codec: StudyRegistryCodec;
|
|
56
|
+
/** `repro/`'s entries, links not followed: a study folder that is a link is never read through. */
|
|
57
|
+
readonly entries: ListEntries;
|
|
49
58
|
}
|
|
50
59
|
|
|
60
|
+
/** The names at the top of `<dir>/repro/` that are symbolic links. */
|
|
61
|
+
const linksIn = (
|
|
62
|
+
dir: string,
|
|
63
|
+
deps: Pick<StudyRegistryDeps, "entries">,
|
|
64
|
+
): ReadonlySet<string> =>
|
|
65
|
+
new Set(
|
|
66
|
+
deps
|
|
67
|
+
.entries(callerPath(join(dir, REPRO_DIR)))
|
|
68
|
+
.filter((e) => e.kind === "symlink")
|
|
69
|
+
.map((e) => e.name),
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The test every reader of a study's folder passes a row through: its folder is one folder name
|
|
74
|
+
* inside `repro/` (`isFolderName`) and not a symbolic link, which would lead the reads anywhere.
|
|
75
|
+
*/
|
|
76
|
+
export const readableFolders = (
|
|
77
|
+
dir: string,
|
|
78
|
+
deps: Pick<StudyRegistryDeps, "entries">,
|
|
79
|
+
): ((folder: string) => boolean) => {
|
|
80
|
+
const links = linksIn(dir, deps);
|
|
81
|
+
return (folder) => isFolderName(folder) && !links.has(folder);
|
|
82
|
+
};
|
|
83
|
+
|
|
51
84
|
/** The text of the file at `p`; null when there is none, or a directory sits there. */
|
|
52
|
-
const textAt = (files: Files, p: string): string | null => {
|
|
85
|
+
export const textAt = (files: Files, p: string): string | null => {
|
|
53
86
|
const b = files.isFile(callerPath(p)) ? files.readBytes(callerPath(p)) : null;
|
|
54
87
|
return b === null ? null : new TextDecoder().decode(b);
|
|
55
88
|
};
|
|
@@ -83,9 +116,12 @@ const existingResults = (
|
|
|
83
116
|
dir: string,
|
|
84
117
|
r: RegistryState,
|
|
85
118
|
files: Files,
|
|
119
|
+
readable: (folder: string) => boolean,
|
|
86
120
|
): ReadonlySet<string> =>
|
|
87
121
|
new Set(
|
|
88
122
|
(r.kind === "read" ? r.registry.studies : [])
|
|
123
|
+
// A row whose folder is not one inside `repro/`, or is a link, is never read through.
|
|
124
|
+
.filter((s) => readable(s.folder))
|
|
89
125
|
.flatMap((s) =>
|
|
90
126
|
s.result_file === undefined ? [] : [`${s.folder}/${s.result_file}`],
|
|
91
127
|
)
|
|
@@ -97,21 +133,52 @@ const existingResults = (
|
|
|
97
133
|
),
|
|
98
134
|
);
|
|
99
135
|
|
|
100
|
-
/**
|
|
136
|
+
/**
|
|
137
|
+
* The plan each row of `r` names, read and parsed, by `<folder>/<plan>`; a plan not there is left out.
|
|
138
|
+
* A plan still named `prereg.json` is parsed by the loose schema of its old name.
|
|
139
|
+
*/
|
|
140
|
+
export const plansOf = (
|
|
141
|
+
dir: string,
|
|
142
|
+
r: RegistryState,
|
|
143
|
+
deps: Pick<StudyRegistryDeps, "files" | "codec">,
|
|
144
|
+
readable: (folder: string) => boolean,
|
|
145
|
+
): ReadonlyMap<string, PlanState> =>
|
|
146
|
+
new Map(
|
|
147
|
+
(r.kind === "read" ? planRefs(r.registry) : [])
|
|
148
|
+
// A NUL is no path the disk can hold, and probing one throws: such a plan reads as missing. A
|
|
149
|
+
// folder that is a link is never read through.
|
|
150
|
+
.filter(({ key, folder }) => !key.includes("\0") && readable(folder))
|
|
151
|
+
.flatMap(({ key, kind }): readonly (readonly [string, PlanState])[] => {
|
|
152
|
+
const text = textAt(deps.files, join(dir, REPRO_DIR, key));
|
|
153
|
+
if (text === null) return [];
|
|
154
|
+
return [
|
|
155
|
+
[
|
|
156
|
+
key,
|
|
157
|
+
posix.basename(key) === LEGACY_PLAN_FILE
|
|
158
|
+
? deps.codec.parseLegacyPlan(text)
|
|
159
|
+
: deps.codec.parsePlan(text, kind),
|
|
160
|
+
],
|
|
161
|
+
];
|
|
162
|
+
}),
|
|
163
|
+
);
|
|
164
|
+
|
|
165
|
+
/** Every finding of the paper folder `dir`'s study registry — THE function behind both rules. */
|
|
101
166
|
export function studyFindings(
|
|
102
167
|
dir: string,
|
|
103
168
|
deps: StudyRegistryDeps,
|
|
104
169
|
): readonly StudyFinding[] {
|
|
170
|
+
const readable = readableFolders(dir, deps);
|
|
105
171
|
const registry = registryOf(dir, deps);
|
|
106
172
|
return judgeStudies({
|
|
107
173
|
folders: studyFolders(dir, deps),
|
|
108
174
|
registry,
|
|
109
175
|
readme: textAt(deps.files, join(dir, REPRO_DIR, README)),
|
|
110
|
-
existingResults: existingResults(dir, registry, deps.files),
|
|
176
|
+
existingResults: existingResults(dir, registry, deps.files, readable),
|
|
177
|
+
plans: plansOf(dir, registry, deps, readable),
|
|
111
178
|
});
|
|
112
179
|
}
|
|
113
180
|
|
|
114
|
-
/** The problems
|
|
181
|
+
/** The problems `repro/study-registered` reports, each its own message id; the text is `studyMessage`. */
|
|
115
182
|
const PROBLEMS: readonly StudyFinding["problem"][] = [
|
|
116
183
|
"no-registry",
|
|
117
184
|
"unreadable",
|
|
@@ -126,47 +193,84 @@ const PROBLEMS: readonly StudyFinding["problem"][] = [
|
|
|
126
193
|
"result-file",
|
|
127
194
|
"no-successor",
|
|
128
195
|
"stale-index",
|
|
196
|
+
"unknown-measurement",
|
|
197
|
+
"empty-measurement",
|
|
198
|
+
"unknown-requirement",
|
|
199
|
+
"headline-approval",
|
|
200
|
+
"bad-plan",
|
|
201
|
+
"author-check",
|
|
129
202
|
];
|
|
130
203
|
|
|
131
|
-
|
|
204
|
+
/** A rule's meta: its page, and a message id per problem, the whole text being `studyMessage`. */
|
|
205
|
+
const metaOf = (
|
|
206
|
+
name: string,
|
|
207
|
+
description: string,
|
|
208
|
+
problems: readonly StudyFinding["problem"][],
|
|
209
|
+
): FolderRuleModule["meta"] => ({
|
|
132
210
|
type: "suggestion",
|
|
133
|
-
docs: {
|
|
134
|
-
description:
|
|
135
|
-
"every study folder at the top of a paper's repro/ has exactly one row in repro/studies.json, every row names a folder that exists, and repro/README.md's index is the one the registry generates",
|
|
136
|
-
url: rulePageUrl("repro/study-registered"),
|
|
137
|
-
},
|
|
211
|
+
docs: { description, url: rulePageUrl(name) },
|
|
138
212
|
schema: [],
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
};
|
|
213
|
+
messages: Object.fromEntries(problems.map((p) => [p, "{{text}}"])),
|
|
214
|
+
});
|
|
142
215
|
|
|
143
|
-
/**
|
|
144
|
-
|
|
216
|
+
/** A rule that reports the findings of `problems`, reading the folder through `deps`. */
|
|
217
|
+
const ruleOf = (
|
|
218
|
+
meta: FolderRuleModule["meta"],
|
|
219
|
+
problems: readonly StudyFinding["problem"][],
|
|
145
220
|
deps: StudyRegistryDeps,
|
|
146
|
-
):
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
});
|
|
221
|
+
): FolderRuleModule => ({
|
|
222
|
+
meta,
|
|
223
|
+
create(context: FolderRuleContext) {
|
|
224
|
+
if (basename(context.filename) !== STATUS_FILE) return {};
|
|
225
|
+
return {
|
|
226
|
+
root() {
|
|
227
|
+
const at = context.sourceCode.getLocFromIndex(0);
|
|
228
|
+
studyFindings(dirname(context.filename), deps)
|
|
229
|
+
.filter((f) => problems.includes(f.problem))
|
|
230
|
+
.forEach((f) => {
|
|
231
|
+
context.report({
|
|
232
|
+
loc: { start: at, end: at },
|
|
233
|
+
messageId: f.problem,
|
|
234
|
+
data: { text: studyMessage(f) },
|
|
161
235
|
});
|
|
162
|
-
}
|
|
163
|
-
};
|
|
236
|
+
});
|
|
164
237
|
},
|
|
165
|
-
}
|
|
238
|
+
};
|
|
239
|
+
},
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* The two rules, reading the folder through `deps`: `study-registered`, the registry's errors, and
|
|
244
|
+
* `study-plan-name`, a warning for a plan still named `prereg.json` — read as the plan, so a paper
|
|
245
|
+
* that wrote one before `plan.json` existed is not broken, and told the name to move to.
|
|
246
|
+
*/
|
|
247
|
+
export function studyRegistryRules(
|
|
248
|
+
deps: StudyRegistryDeps,
|
|
249
|
+
): Readonly<Record<"study-registered" | "study-plan-name", FolderRuleModule>> {
|
|
250
|
+
return {
|
|
251
|
+
"study-registered": ruleOf(
|
|
252
|
+
metaOf(
|
|
253
|
+
"repro/study-registered",
|
|
254
|
+
"every study folder at the top of a paper's repro/ has exactly one row in repro/studies.json, every row names a folder that exists, its measurement and its plan hold together, and repro/README.md's index is the one the registry generates",
|
|
255
|
+
PROBLEMS,
|
|
256
|
+
),
|
|
257
|
+
PROBLEMS,
|
|
258
|
+
deps,
|
|
259
|
+
),
|
|
260
|
+
"study-plan-name": ruleOf(
|
|
261
|
+
metaOf(
|
|
262
|
+
"repro/study-plan-name",
|
|
263
|
+
"a study's plan is named plan.json, not prereg.json",
|
|
264
|
+
PLAN_NAME_PROBLEMS,
|
|
265
|
+
),
|
|
266
|
+
PLAN_NAME_PROBLEMS,
|
|
267
|
+
deps,
|
|
268
|
+
),
|
|
166
269
|
};
|
|
167
270
|
}
|
|
168
271
|
|
|
169
|
-
/** The
|
|
170
|
-
export const STUDY_REGISTRY_RULE_LEVELS: Readonly<
|
|
171
|
-
|
|
172
|
-
|
|
272
|
+
/** The levels they are on at in paperlint's own config, for every paper's `PIPELINE-STATUS.md`. */
|
|
273
|
+
export const STUDY_REGISTRY_RULE_LEVELS: Readonly<{
|
|
274
|
+
"repro/study-registered": "error";
|
|
275
|
+
"repro/study-plan-name": "warn";
|
|
276
|
+
}> = { "repro/study-registered": "error", "repro/study-plan-name": "warn" };
|