paperlint 10.2.0 → 10.3.0

package/src/studies.test.ts

@@ -9,6 +9,8 @@ import { memoryFiles, memoryListDir } from "./adapters/memory/index.ts";
 import { studyRegistryCodec } from "./adapters/study-registry/index.ts";
 import {
   indexBlock,
+  planTemplate,
+  planText,
   registryText,
   studyReadme,
   withIndex,
@@ -101,17 +103,21 @@ describe("studies new", () => {
       status: "designing",
       date: "2026-10-15",
       in_paper: "no",
+      plan: "plan.json",
     };
     const r = P1(study(), added);
     expect({ code, err }).toEqual({ code: 0, err: [] });
     expect(after).toEqual({
       ...before,
       [`${REPRO}/P1-S02-ablation-of-the-guard/README.md`]: studyReadme(added),
+      [`${REPRO}/P1-S02-ablation-of-the-guard/plan.json`]:
+        planText(planTemplate()),
       [`${REPRO}/studies.json`]: registryText(r),
       [`${REPRO}/README.md`]: `# Reproduction index\n\nIntro.\n\n${indexBlock(r)}\n`,
     });
     expect(log).toEqual([
       "created papers/example/repro/P1-S02-ablation-of-the-guard/README.md",
+      "created papers/example/repro/P1-S02-ablation-of-the-guard/plan.json, the study's plan: its fields are there to fill in before the first run",
       "added P1-S02 to papers/example/repro/studies.json: kind spike, status designing, 2026-10-15",
       "wrote the index of 2 studies in papers/example/repro/README.md",
       "Next: write the study's one-sentence question in its row (`question`) and in its README.md. When it ends, set `status`, `result`, `result_file` and `in_paper` in its row, then run `npx paperlint studies index papers/example`.",

package/src/studies.ts

@@ -3,8 +3,9 @@
  * `repro/studies.json`, and its index in `repro/README.md` (docs/design/study-registry.md).
  *
  *   new <paper-dir> "<title>" [--kind <kind>]
- *            the next id; `repro/<id>-<slug>/README.md` with the id and title in its frontmatter; the
- *            row, `status: designing`, today's date; the index regenerated
+ *            the next id; `repro/<id>-<slug>/README.md` with the id and title in its frontmatter, and
+ *            `plan.json`, the plan's template; the row, `status: designing`, today's date, its `plan`;
+ *            the index regenerated
  *   index <paper-dir>
  *            the index block of `repro/README.md`, regenerated from the registry
  *
@@ -18,11 +19,14 @@ import type { AbsolutePath } from "./domain/paths.ts";
 import { README, REPRO_DIR } from "./domain/repro-layout.ts";
 import {
   DEFAULT_KIND,
+  PLAN_FILE,
   STUDIES_FILE,
   STUDY_KINDS,
   addStudy,
   indexInSync,
   isStudyKind,
+  planTemplate,
+  planText,
   registryText,
   studyReadme,
   withIndex,
@@ -178,7 +182,7 @@ function newArgs(
       );
 }
 
-/** Write a new study's README.md, the registry, and the index; the lines that say so. */
+/** Write a new study's README.md and plan.json, the registry, and the index; the lines that say so. */
 function writeStudy(
   p: Paper,
   added: { readonly registry: StudyRegistry; readonly study: Study },
@@ -188,11 +192,16 @@ function writeStudy(
   const at = (...names: readonly string[]): AbsolutePath =>
     callerPath(join(p.dir, REPRO_DIR, ...names));
   files.writeAtomic(at(study.folder, README), text(studyReadme(study)));
+  files.writeAtomic(
+    at(study.folder, PLAN_FILE),
+    text(planText(planTemplate())),
+  );
   files.writeAtomic(at(STUDIES_FILE), text(registryText(registry)));
   writeIndex(p.dir, registry, files);
   const repro = `${p.shown}/${REPRO_DIR}`;
   return [
     `created ${repro}/${study.folder}/${README}`,
+    `created ${repro}/${study.folder}/${PLAN_FILE}, the study's plan: its fields are there to fill in before the first run`,
     `added ${study.id} to ${repro}/${STUDIES_FILE}: kind ${study.kind}, status ${study.status}, ${study.date}`,
     `wrote the index of ${studiesCount(registry)} in ${repro}/${README}`,
     `Next: write the study's one-sentence question in its row (\`question\`) and in its README.md. When it ends, set \`status\`, \`result\`, \`result_file\` and \`in_paper\` in its row, then run \`npx paperlint studies index ${p.shown}\`.`,

package/src/study-registry-rule.test.ts

@@ -9,6 +9,8 @@ import { describe, expect, it } from "vitest";
 import { memoryFiles, memoryListDir } from "./adapters/memory/index.ts";
 import { studyRegistryCodec } from "./adapters/study-registry/index.ts";
 import {
+  planTemplate,
+  planText,
   registryText,
   studyMessage,
   withIndex,
@@ -65,13 +67,14 @@ interface Reported {
 function lint(
   tree: Readonly<Record<string, string>>,
   file = `${DIR}/PIPELINE-STATUS.md`,
+  name: "study-registered" | "study-plan-name" = "study-registered",
 ): Reported[] {
   const files = memoryFiles(tree);
   const rule = studyRegistryRules({
     files,
     listDir: memoryListDir(files),
     codec: studyRegistryCodec,
-  })["study-registered"];
+  })[name];
   const out: Reported[] = [];
   const visitor = rule.create({
     filename: file,
@@ -271,6 +274,7 @@ describe("repro/study-registered — level", () => {
   it("is an error from the start", () => {
     expect(STUDY_REGISTRY_RULE_LEVELS).toEqual({
       "repro/study-registered": "error",
+      "repro/study-plan-name": "warn",
     });
   });
 });
@@ -315,3 +319,138 @@ describe("repro/study-registered — a result_file the disk cannot hold", () =>
     ).toEqual(["result-file"]);
   });
 });
+
+/** A plan as a study would hold it once the author has checked its drawn units. */
+const FILLED = planText({
+  question: "How many rules can a linter check?",
+  population: {
+    source: "the crawl of 2026-10-01",
+    draw: { kind: "all", method: "census" },
+  },
+  arms: [],
+  primary_outcome: "the share of rules a linter can check",
+  author_check: { by: "A. Author", date: "2026-10-14", quote: "Go." },
+});
+
+/** A registry with a headline measurement of one running study whose row names `plan`. */
+const HEADLINE = (plan: string): StudyRegistry => ({
+  prefix: "P1",
+  author: "A. Author",
+  measurements: [
+    {
+      id: "M1",
+      title: "Rule census",
+      role: "headline",
+      approval: { by: "A. Author", date: "2026-10-12", quote: "Yes." },
+    },
+  ],
+  studies: [study({ status: "running", measurement: "M1", plan })],
+});
+
+describe("repro/study-registered — the plan a row names, read from its folder", () => {
+  it("is silent on a plan.json the author checked", () => {
+    expect(
+      lint({
+        ...registered(HEADLINE("plan.json")),
+        [`${DIR}/repro/P1-S01-baseline/plan.json`]: FILLED,
+      }),
+    ).toEqual([]);
+  });
+
+  it("reports a plan that is not there", () => {
+    expect(messages(registered(HEADLINE("plan.json")))).toEqual([
+      studyMessage({
+        problem: "bad-plan",
+        id: "P1-S01",
+        path: "plan.json",
+        why: "missing",
+      }),
+    ]);
+  });
+
+  it("reports a plan that is not JSON, with the reason", () => {
+    expect(
+      messages({
+        ...registered(HEADLINE("plan.json")),
+        [`${DIR}/repro/P1-S01-baseline/plan.json`]: "{",
+      }),
+    ).toEqual([
+      studyMessage({
+        problem: "bad-plan",
+        id: "P1-S01",
+        path: "plan.json",
+        why: "unreadable",
+        reasons: "not JSON",
+      }),
+    ]);
+  });
+
+  it("reports a running study of the headline whose plan is still the template", () => {
+    expect(
+      lint({
+        ...registered(HEADLINE("plan.json")),
+        [`${DIR}/repro/P1-S01-baseline/plan.json`]: planText(planTemplate()),
+      }).map((r) => [r.messageId, r.line]),
+    ).toEqual([
+      ["bad-plan", 1],
+      ["author-check", 1],
+    ]);
+  });
+});
+
+describe("repro/study-registered — a plan the disk cannot hold", () => {
+  it("reads as missing, and the disk is never asked", () => {
+    const r = HEADLINE("a\u0000.json");
+    const files = memoryFiles(registered(r));
+    const probe = {
+      ...files,
+      isFile: (p: Parameters<typeof files.isFile>[0]): boolean => {
+        if (p.includes("\0")) throw new Error("ERR_INVALID_ARG_VALUE");
+        return files.isFile(p);
+      },
+    };
+    expect(
+      studyFindings(DIR, {
+        files: probe,
+        listDir: memoryListDir(files),
+        codec: studyRegistryCodec,
+      }).map((f) => f.problem),
+    ).toEqual(["bad-plan"]);
+  });
+});
+
+describe("repro/study-plan-name — a plan under its old name", () => {
+  const OLD = {
+    ...registered(HEADLINE("prereg.json")),
+    [`${DIR}/repro/P1-S01-baseline/prereg.json`]: FILLED,
+  };
+
+  it("is read as the plan: repro/study-registered is silent", () => {
+    expect(lint(OLD)).toEqual([]);
+  });
+
+  it("is reported by repro/study-plan-name, naming plan.json", () => {
+    expect(lint(OLD, `${DIR}/PIPELINE-STATUS.md`, "study-plan-name")).toEqual([
+      {
+        messageId: "legacy-plan",
+        message: studyMessage({
+          problem: "legacy-plan",
+          id: "P1-S01",
+          path: "prereg.json",
+        }),
+        line: 1,
+      },
+    ]);
+  });
+
+  it("is silent on plan.json, and on any file but PIPELINE-STATUS.md", () => {
+    const tree = {
+      ...registered(HEADLINE("plan.json")),
+      [`${DIR}/repro/P1-S01-baseline/plan.json`]: FILLED,
+    };
+    expect([
+      lint(tree, `${DIR}/PIPELINE-STATUS.md`, "study-plan-name"),
+      lint(OLD, `${DIR}/paper.tex`, "study-plan-name"),
+    ]).toEqual([[], []]);
+  });
+});

package/src/study-registry-rule.ts

@@ -1,10 +1,11 @@
 /**
  * `repro/study-registered` — every study folder at the top of a paper's `repro/` has exactly one row in
- * `repro/studies.json`, every row names a folder that exists, and the index in `repro/README.md` is the
- * one the registry generates.
+ * `repro/studies.json`, every row names a folder that exists, a row's measurement and plan hold
+ * together, and the index in `repro/README.md` is the one the registry generates. Beside it,
+ * `repro/study-plan-name`, a warning: a plan still named `prereg.json`.
  *
  * What is judged, and why, is `src/domain/study-registry.ts` and docs/rules/repro/study-registered.md.
- * This file lists `repro/`, reads the registry, the README and the result files, and reports.
+ * This file lists `repro/`, reads the registry, the README, the result files and the plans, and reports.
  *
  * ── WHY A RULE OF ITS OWN, NOT MORE OF `repro/allowed-files-only` ─────────────────────
  * `repro/allowed-files-only` judges what may stand at the top of `repro/`, by name, and accepts the
@@ -16,7 +17,7 @@
  * `PIPELINE-STATUS.md`, first line, like the layout rules: every paper has it, and ESLint lints no
  * file in `repro/`.
  */
-import { basename, dirname, join } from "node:path";
+import { basename, dirname, join, posix } from "node:path";
 import { callerPath } from "./caller-path.ts";
 import {
   README,
@@ -25,9 +26,13 @@ import {
   REPRO_DIRS,
 } from "./domain/repro-layout.ts";
 import {
+  LEGACY_PLAN_FILE,
+  PLAN_NAME_PROBLEMS,
   STUDIES_FILE,
   judgeStudies,
+  planKeys,
   studyMessage,
+  type PlanState,
   type RegistryState,
   type StudyFinding,
 } from "./domain/study-registry.ts";
@@ -97,7 +102,31 @@ const existingResults = (
       ),
   );
 
-/** Every finding of the paper folder `dir`'s study registry — THE function behind the rule. */
+/**
+ * The plan each row of `r` names, read and parsed, by `<folder>/<plan>`; a plan not there is left out.
+ * A plan still named `prereg.json` is parsed by the loose schema of its old name.
+ */
+const plansOf = (
+  dir: string,
+  r: RegistryState,
+  deps: Pick<StudyRegistryDeps, "files" | "codec">,
+): ReadonlyMap<string, PlanState> =>
+  new Map(
+    (r.kind === "read" ? planKeys(r.registry) : [])
+      // A NUL is no path the disk can hold, and probing one throws: such a plan reads as missing.
+      .filter((key) => !key.includes("\0"))
+      .flatMap((key): readonly (readonly [string, PlanState])[] => {
+        const text = textAt(deps.files, join(dir, REPRO_DIR, key));
+        if (text === null) return [];
+        const parse =
+          posix.basename(key) === LEGACY_PLAN_FILE
+            ? deps.codec.parseLegacyPlan
+            : deps.codec.parsePlan;
+        return [[key, parse(text)]];
+      }),
+  );
+
+/** Every finding of the paper folder `dir`'s study registry — THE function behind both rules. */
 export function studyFindings(
   dir: string,
   deps: StudyRegistryDeps,
@@ -108,10 +137,11 @@ export function studyFindings(
     registry,
     readme: textAt(deps.files, join(dir, REPRO_DIR, README)),
     existingResults: existingResults(dir, registry, deps.files),
+    plans: plansOf(dir, registry, deps),
   });
 }
 
-/** The problems a finding can be, each its own message id; the text is `studyMessage`. */
+/** The problems `repro/study-registered` reports, each its own message id; the text is `studyMessage`. */
 const PROBLEMS: readonly StudyFinding["problem"][] = [
   "no-registry",
   "unreadable",
@@ -126,47 +156,84 @@ const PROBLEMS: readonly StudyFinding["problem"][] = [
   "result-file",
   "no-successor",
   "stale-index",
+  "unknown-measurement",
+  "empty-measurement",
+  "unknown-requirement",
+  "headline-approval",
+  "bad-plan",
+  "author-check",
 ];
 
-const META: FolderRuleModule["meta"] = {
+/** A rule's meta: its page, and a message id per problem, the whole text being `studyMessage`. */
+const metaOf = (
+  name: string,
+  description: string,
+  problems: readonly StudyFinding["problem"][],
+): FolderRuleModule["meta"] => ({
   type: "suggestion",
-  docs: {
-    description:
-      "every study folder at the top of a paper's repro/ has exactly one row in repro/studies.json, every row names a folder that exists, and repro/README.md's index is the one the registry generates",
-    url: rulePageUrl("repro/study-registered"),
-  },
+  docs: { description, url: rulePageUrl(name) },
   schema: [],
-  // The whole text is `studyMessage` (src/domain/study-registry.ts).
-  messages: Object.fromEntries(PROBLEMS.map((p) => [p, "{{text}}"])),
-};
+  messages: Object.fromEntries(problems.map((p) => [p, "{{text}}"])),
+});
 
-/** The rule, reading the folder through `deps`. */
-export function studyRegistryRules(
+/** A rule that reports the findings of `problems`, reading the folder through `deps`. */
+const ruleOf = (
+  meta: FolderRuleModule["meta"],
+  problems: readonly StudyFinding["problem"][],
   deps: StudyRegistryDeps,
-): Readonly<Record<"study-registered", FolderRuleModule>> {
-  return {
-    "study-registered": {
-      meta: META,
-      create(context: FolderRuleContext) {
-        if (basename(context.filename) !== STATUS_FILE) return {};
-        return {
-          root() {
-            const at = context.sourceCode.getLocFromIndex(0);
-            studyFindings(dirname(context.filename), deps).forEach((f) => {
-              context.report({
-                loc: { start: at, end: at },
-                messageId: f.problem,
-                data: { text: studyMessage(f) },
-              });
+): FolderRuleModule => ({
+  meta,
+  create(context: FolderRuleContext) {
+    if (basename(context.filename) !== STATUS_FILE) return {};
+    return {
+      root() {
+        const at = context.sourceCode.getLocFromIndex(0);
+        studyFindings(dirname(context.filename), deps)
+          .filter((f) => problems.includes(f.problem))
+          .forEach((f) => {
+            context.report({
+              loc: { start: at, end: at },
+              messageId: f.problem,
+              data: { text: studyMessage(f) },
             });
-          },
-        };
+          });
       },
-    },
+    };
+  },
+});
+
+/**
+ * The two rules, reading the folder through `deps`: `study-registered`, the registry's errors, and
+ * `study-plan-name`, a warning for a plan still named `prereg.json` — read as the plan, so a paper
+ * that wrote one before `plan.json` existed is not broken, and told the name to move to.
+ */
+export function studyRegistryRules(
+  deps: StudyRegistryDeps,
+): Readonly<Record<"study-registered" | "study-plan-name", FolderRuleModule>> {
+  return {
+    "study-registered": ruleOf(
+      metaOf(
+        "repro/study-registered",
+        "every study folder at the top of a paper's repro/ has exactly one row in repro/studies.json, every row names a folder that exists, its measurement and its plan hold together, and repro/README.md's index is the one the registry generates",
+        PROBLEMS,
+      ),
+      PROBLEMS,
+      deps,
+    ),
+    "study-plan-name": ruleOf(
+      metaOf(
+        "repro/study-plan-name",
+        "a study's plan is named plan.json, not prereg.json",
+        PLAN_NAME_PROBLEMS,
+      ),
+      PLAN_NAME_PROBLEMS,
+      deps,
+    ),
   };
 }
 
-/** The level it is on at in paperlint's own config, for every paper's `PIPELINE-STATUS.md`. */
-export const STUDY_REGISTRY_RULE_LEVELS: Readonly<
-  Record<"repro/study-registered", "error">
-> = { "repro/study-registered": "error" };
+/** The levels they are on at in paperlint's own config, for every paper's `PIPELINE-STATUS.md`. */
+export const STUDY_REGISTRY_RULE_LEVELS: Readonly<{
+  "repro/study-registered": "error";
+  "repro/study-plan-name": "warn";
+}> = { "repro/study-registered": "error", "repro/study-plan-name": "warn" };