paperlint 10.3.0 → 10.4.0

package/src/adapters/study-registry/mode.test.ts

@@ -0,0 +1,162 @@
+/**
+ * The schema of a study's mode (`mode` in its plan). Both halves: what `studies new` writes parses back
+ * to itself; a measurement's plan is ready only with exactly one mode; every contradiction is refused
+ * with its reason.
+ */
+import { expect, it } from "vitest";
+import type { StudyMode } from "../../domain/study-mode.ts";
+import {
+  planTemplate,
+  planText,
+  type StudyKind,
+  type StudyPlan,
+} from "../../domain/study-registry.ts";
+import { studyRegistryCodec } from "./index.ts";
+
+const DECIDED: StudyMode = {
+  kind: "decide_before_run",
+  outputs: ["data/outcome/**", "RESULTS.md"],
+  decision: {
+    file: "repro/P1-S01-x/decision.md",
+    note: "exclusions and the sample approved",
+  },
+};
+const EXPLORATORY: StudyMode = {
+  kind: "exploratory",
+  reason: "a first look at the logs, to find the question",
+};
+
+const PLAN: StudyPlan = {
+  question: "Does the hint help?",
+  population: {
+    source: "tasks, 2026-10-01",
+    draw: { kind: "all", method: "census" },
+  },
+  arms: [{ name: "hint" }, { name: "none" }],
+  primary_outcome: "pass rate",
+  author_check: null,
+  mode: DECIDED,
+};
+
+const parse = (text: string, kind: StudyKind = "measurement") =>
+  studyRegistryCodec.parsePlan(text, kind);
+const withMode = (mode: unknown): string => JSON.stringify({ ...PLAN, mode });
+const why = (text: string): string => {
+  const r = parse(text);
+  return r.kind === "unreadable" ? r.why : r.kind;
+};
+const empty = (text: string, kind: StudyKind = "measurement") => {
+  const r = parse(text, kind);
+  return r.kind === "draft" ? r.empty : r.kind;
+};
+
+it("each mode parses to itself, ready", () => {
+  // Guards: the three modes round-trip through the text planText writes.
+  expect(
+    [DECIDED, EXPLORATORY].map((mode) => parse(planText({ ...PLAN, mode }))),
+  ).toEqual(
+    [DECIDED, EXPLORATORY].map((mode) => ({
+      kind: "ready",
+      plan: { ...PLAN, mode },
+    })),
+  );
+});
+
+it("a measurement's plan without a mode is a draft naming `mode`; a spike's is ready", () => {
+  const none: StudyPlan = { ...PLAN, mode: undefined };
+  // Guards: "none" is not a ready state for a measurement — and a spike may omit it.
+  expect(empty(planText(none))).toEqual(["mode"]);
+  expect(empty(planText(none), "spike")).toBe("ready");
+});
+
+it("the template of a measurement is a draft decided before its run, its decision file named", () => {
+  const t = planTemplate("measurement", "P1-S02-x");
+  // Guards: `studies new` scaffolds the default mode; what is left to fill is named.
+  expect(t.mode).toEqual({
+    kind: "decide_before_run",
+    outputs: [],
+    decision: { file: "repro/P1-S02-x/decision.md", note: "" },
+  });
+  expect(empty(planText(t))).toEqual([
+    "question",
+    "population.source",
+    "population.draw.method",
+    "primary_outcome",
+    "mode.outputs",
+    "mode.decision.note",
+  ]);
+  expect(planTemplate("spike", "P1-S03-y").mode).toBeUndefined();
+});
+
+it("each mode's empty texts are named: the decision's, an exploratory reason", () => {
+  // Guards: a mode left half-written is a draft, so a study that has run on it is reported.
+  expect(
+    empty(
+      withMode({ ...DECIDED, outputs: [], decision: { file: " ", note: "" } }),
+    ),
+  ).toEqual(["mode.outputs", "mode.decision.file", "mode.decision.note"]);
+  expect(empty(withMode({ ...EXPLORATORY, reason: "" }))).toEqual([
+    "mode.reason",
+  ]);
+});
+
+it("two modes at once, or a mode that is neither, cannot be written", () => {
+  // Guards: exactly one — a second mode's key is an unknown key, an unknown kind is refused.
+  expect(why(withMode({ ...DECIDED, reason: "r" }))).toBe(
+    'mode: Unrecognized key: "reason"',
+  );
+  expect(why(withMode({ kind: "preregistered" }))).toBe(
+    "mode.kind: Invalid discriminator value. Expected 'decide_before_run' | 'exploratory'",
+  );
+  expect(why(withMode({ kind: "blind" }))).toBe(
+    "mode.kind: Invalid discriminator value. Expected 'decide_before_run' | 'exploratory'",
+  );
+});
+
+it("refuses an output outside the folder or naming the study's README, and a decision outside the paper", () => {
+  const outside =
+    "is not a path inside the study's folder, or names its README.md";
+  // Guards: a glob cannot reach past the study's folder; a decision cannot leave the paper.
+  expect(
+    ["../x", "/abs", "a\\b", "README.md"].map((o) =>
+      why(withMode({ ...DECIDED, outputs: [o] })),
+    ),
+  ).toEqual(Array.from({ length: 4 }, () => `mode.outputs.0: ${outside}`));
+  expect(why(withMode({ ...DECIDED, outputs: ["a", "a"] }))).toBe(
+    "mode.outputs: the output «a» is listed more than once",
+  );
+  expect(
+    ["../d.md", "/d.md", "a\\d.md"].map((file) =>
+      why(withMode({ ...DECIDED, decision: { file, note: "n" } })),
+    ),
+  ).toEqual(
+    Array.from(
+      { length: 3 },
+      () => "mode.decision.file: is not a path inside the paper's folder",
+    ),
+  );
+});
+
+it("refuses a decision path that names the paper's folder or a directory, and keeps an empty one a draft", () => {
+  const notFile =
+    "mode.decision.file: is not a path to a file inside the paper's folder: one or more names joined by /, none empty, . or ..";
+  // Guards: a decision of `.` would make the gate ask git about the whole paper, and the order rule
+  // read the paper's first commit as the decision; such a path cannot be written down.
+  expect(
+    [".", "./", "repro/", "a//d.md", "a/./d.md", "./d.md"].map((file) =>
+      why(withMode({ ...DECIDED, decision: { file, note: "n" } })),
+    ),
+  ).toEqual(Array(6).fill(notFile));
+  expect(
+    empty(withMode({ ...DECIDED, decision: { file: "", note: "n" } })),
+  ).toEqual(["mode.decision.file"]);
+});
+
+it("mode is a field of the plan, so extra cannot carry it", () => {
+  // Guards: one place says how a study is gated.
+  expect(
+    why(JSON.stringify({ ...PLAN, mode: undefined, extra: { mode: {} } })),
+  ).toBe(
+    "extra.mode: is a field of the plan itself — set it there, not in extra",
+  );
+});

package/src/cli.ts

@@ -190,6 +190,12 @@ import {
   studyRegistryRules,
 } from "./study-registry-rule.ts";
 import { studyRegistryCodec } from "./adapters/study-registry/index.ts";
+import { nodeListEntries, spawnProcess } from "./adapters/node/index.ts";
+import { gitHistory } from "./adapters/git/index.ts";
+import {
+  ORDER_RULE_LEVELS,
+  outputsBeforeDecisionRules,
+} from "./outputs-before-decision-rule.ts";
 import { runStudies } from "./studies.ts";
 import typography from "#eslint-rules/paper-typography";
 import texBuild from "#eslint-rules/tex-build";
@@ -243,6 +249,11 @@ const USAGE = `paperlint — machine-checkable gates for a paper kept in git
                                       index in repro/README.md regenerated
   npx paperlint studies index <paper-dir>
                                       regenerate the index in repro/README.md from repro/studies.json
+  npx paperlint studies gate <paper-dir> <id>
+                                      may the study's run start? For a study decided before its run
+                                      (the default mode), exit 0 only once its decision file is
+                                      committed as it stands, printing the commit and date; else exit
+                                      1 saying why. A runner calls it first: gate … && ./run.sh
   npx paperlint hook <name>           run an editor hook (.claude/settings.json calls this)
   npx paperlint --help
 
@@ -267,7 +278,10 @@ studies:
   --kind <kind>       new: measurement (the default), spike, tooling, data, artifact or salvage.
                       A study's status, result and in_paper are edited in repro/studies.json, its
                       plan in its plan.json; the rule repro/study-registered checks them
-                      (docs/rules/repro/study-registered.md)
+                      (docs/rules/repro/study-registered.md). A measurement's plan declares its
+                      mode: decide_before_run (written by new) or exploratory;
+                      repro/outputs-before-decision checks the order in git
+                      (docs/design/study-modes.md)
 
 submission:
   The venue preset declares the portal — "portal": { "kind": "hotcrp", "url": "<site>" } — and the
@@ -383,20 +397,43 @@ const FRAGMENT_RULES = {
   "heading-case": REGISTER_AND_CASE_RULES["heading-case"],
 };
 
+/** `names` of this process's environment, and nothing else of it. */
+const envOf =
+  (names: readonly string[]) =>
+  (env: Readonly<NodeJS.ProcessEnv>): Readonly<Record<string, string>> =>
+    Object.fromEntries(
+      Object.entries(env).filter(
+        (e): e is [string, string] =>
+          names.includes(e[0]) && e[1] !== undefined,
+      ),
+    );
+
+/** The environment git runs in: PATH, and HOME for its configuration. */
+const gitEnv = envOf(["PATH", "HOME"]);
+
+/** git's history, as the decide-first rule and the gate read it. */
+const GIT = gitHistory(spawnProcess(), gitEnv(process.env));
+
 /** What the two layout rules read with: the disk and its directory listings. */
 const LAYOUT_RULE_DEPS = { files: nodeFiles, listDir: nodeListDir };
 
 /** What `repro/study-registered` and `paperlint studies` read with: the layout's, and the registry's parser. */
-const STUDY_RULE_DEPS = { ...LAYOUT_RULE_DEPS, codec: studyRegistryCodec };
+const STUDY_RULE_DEPS = {
+  ...LAYOUT_RULE_DEPS,
+  entries: nodeListEntries,
+  codec: studyRegistryCodec,
+};
 
 /** The `repro` plugin's rules, on every `PIPELINE-STATUS.md`: the layout of `repro/`, and its study registry. */
 const REPRO_RULES = {
   ...reproLayoutRules(LAYOUT_RULE_DEPS),
   ...studyRegistryRules(STUDY_RULE_DEPS),
+  ...outputsBeforeDecisionRules({ ...STUDY_RULE_DEPS, git: GIT }),
 };
 const REPRO_RULE_LEVELS = {
   ...REPRO_LAYOUT_RULE_LEVELS,
   ...STUDY_REGISTRY_RULE_LEVELS,
+  ...ORDER_RULE_LEVELS,
 };
 
 /** Today in this machine's time zone, `YYYY-MM-DD`: the date `paperlint studies new` registers a study on. */
@@ -1675,7 +1712,14 @@ const SIMPLE: Readonly<
         extra: a.paths.slice(3),
         kind: a.kind,
       },
-      { ...STUDY_RULE_DEPS, cwd, today: today(), log, err },
+      {
+        ...STUDY_RULE_DEPS,
+        git: GIT,
+        cwd,
+        today: today(),
+        log,
+        err,
+      },
     ),
   submission: (a, { log, err, cwd }) =>
     runSubmission(

package/src/domain/decide-first.test.ts

@@ -0,0 +1,258 @@
+/**
+ * What `repro/outputs-before-decision` decides from git's facts (`src/domain/decide-first.ts`): silent
+ * on outputs that came after the decision, loud on each output that did not, on outputs with no
+ * decision committed, on a decision the outputs match, and on a history it cannot read.
+ */
+import { expect, it } from "vitest";
+import {
+  judgeOrder,
+  orderMessage,
+  type DecideFirstStudy,
+  type OrderFinding,
+} from "./decide-first.ts";
+
+const D = { commit: "d".repeat(40), date: "2026-10-16T10:00:00+05:00" };
+const A = { commit: "a".repeat(40), date: "2026-10-16T11:00:00+05:00" };
+
+const STUDY: DecideFirstStudy = {
+  id: "P1-S01",
+  folder: "P1-S01-x",
+  plan: "plan.json",
+  mode: {
+    kind: "decide_before_run",
+    outputs: ["out/**", "RESULTS.md"],
+    decision: { file: "repro/P1-S01-x/decision.md", note: "exclusions" },
+  },
+  tree: ["plan.json", "README.md", "decision.md", "RESULTS.md", "notes.txt"],
+  facts: {
+    kind: "read",
+    decision: D,
+    decisionDeleted: false,
+    files: [
+      { path: "decision.md", ...D, order: "same" },
+      { path: "plan.json", ...D, order: "same" },
+      { path: "RESULTS.md", ...A, order: "after" },
+      { path: "notes.txt", ...D, order: "same" },
+    ],
+  },
+};
+
+it("silent when every output entered history after the decision; the study's own files never count", () => {
+  // Guards: the clean history — and the decision and the plan, committed together, are no outputs.
+  expect(judgeOrder(STUDY)).toEqual([]);
+});
+
+it("an output in the decision's commit, or not after it, deleted or not, is one finding each", () => {
+  const facts = {
+    kind: "read",
+    decision: D,
+    decisionDeleted: false,
+    files: [
+      { path: "RESULTS.md", ...D, order: "same" },
+      { path: "out/gone.csv", ...A, order: "not-after" },
+    ],
+  } as const;
+  // Guards: loud on each defect, the deleted early output included.
+  expect(judgeOrder({ ...STUDY, facts })).toEqual([
+    {
+      problem: "output-before-decision",
+      id: "P1-S01",
+      folder: "P1-S01-x",
+      path: "RESULTS.md",
+      output: D,
+      decision: D,
+      same: true,
+    },
+    {
+      problem: "output-before-decision",
+      id: "P1-S01",
+      folder: "P1-S01-x",
+      path: "out/gone.csv",
+      output: A,
+      decision: D,
+      same: false,
+    },
+  ]);
+});
+
+it("an output in the tree while the decision was never committed", () => {
+  // Guards: the run started before any decision existed in history.
+  expect(
+    judgeOrder({
+      ...STUDY,
+      facts: {
+        kind: "read",
+        decision: null,
+        decisionDeleted: false,
+        files: [],
+      },
+    }),
+  ).toEqual([
+    {
+      problem: "undecided-output",
+      id: "P1-S01",
+      folder: "P1-S01-x",
+      path: "RESULTS.md",
+      decision: "repro/P1-S01-x/decision.md",
+    },
+  ]);
+});
+
+it("an output committed and since deleted, while the decision was never committed, still counts", () => {
+  // Guards: history is the evidence — an output that existed before any decision is not erased by
+  // deleting it; each path is said once, whether it is in the tree, in history, or both.
+  expect(
+    judgeOrder({
+      ...STUDY,
+      tree: ["plan.json", "README.md", "RESULTS.md"],
+      facts: {
+        kind: "read",
+        decision: null,
+        decisionDeleted: false,
+        files: [
+          { path: "plan.json", ...D, order: "not-after" },
+          { path: "out/gone.csv", ...A, order: "not-after" },
+          { path: "RESULTS.md", ...A, order: "not-after" },
+        ],
+      },
+    }).map((f) => (f.problem === "undecided-output" ? f.path : f.problem)),
+  ).toEqual(["RESULTS.md", "out/gone.csv"]);
+});
+
+it("a history it cannot read, and a decision the outputs match, are said", () => {
+  const clash = {
+    ...STUDY,
+    mode: { ...STUDY.mode, outputs: ["**/*.md"] },
+  };
+  // Guards: no git and a shallow clone are findings, never a pass; the clash comes first.
+  expect([
+    judgeOrder({
+      ...STUDY,
+      facts: { kind: "unknown", why: "git is not installed" },
+    }),
+    judgeOrder({ ...clash, facts: { kind: "shallow" } }),
+  ]).toEqual([
+    [{ problem: "no-history", id: "P1-S01", why: "git is not installed" }],
+    [
+      {
+        problem: "decision-is-output",
+        id: "P1-S01",
+        file: "repro/P1-S01-x/decision.md",
+        glob: "**/*.md",
+      },
+      { problem: "shallow", id: "P1-S01" },
+    ],
+  ]);
+});
+
+it("a decision committed and since deleted is reported, and no order is judged against it", () => {
+  // Guards: outputs that came after a decision that no longer stands are not passed on it.
+  expect(
+    judgeOrder({
+      ...STUDY,
+      facts: {
+        kind: "read",
+        decision: D,
+        decisionDeleted: true,
+        files: [{ path: "RESULTS.md", ...A, order: "after" }],
+      },
+    }),
+  ).toEqual([
+    {
+      problem: "decision-deleted",
+      id: "P1-S01",
+      file: "repro/P1-S01-x/decision.md",
+    },
+  ]);
+});
+
+it("a decision that names a directory is reported, and no history is judged", () => {
+  // Guards: the outputs of such a study are never passed on a directory's commits.
+  expect(
+    judgeOrder({ ...STUDY, facts: { kind: "not-a-file", what: "directory" } }),
+  ).toEqual([
+    {
+      problem: "decision-not-file",
+      id: "P1-S01",
+      file: "repro/P1-S01-x/decision.md",
+      what: "directory",
+    },
+  ]);
+});
+
+const PAGE =
+  "\nRule repro/outputs-before-decision: https://github.com/zernie/paperlint/blob/main/docs/rules/repro/outputs-before-decision.md";
+
+/** One finding of every problem, the decision's first. */
+const ALL: readonly OrderFinding[] = [
+  {
+    problem: "decision-is-output",
+    id: "P1-S01",
+    file: "repro/P1-S01-x/decision.md",
+    glob: "**/*.md",
+  },
+  { problem: "decision-deleted", id: "P1-S01", file: "repro/d.md" },
+  {
+    problem: "decision-not-file",
+    id: "P1-S01",
+    file: "repro/d",
+    what: "directory",
+  },
+  {
+    problem: "decision-not-file",
+    id: "P1-S01",
+    file: "repro/l.md",
+    what: "symbolic link",
+  },
+  { problem: "no-history", id: "P1-S01", why: "git is not installed" },
+  { problem: "shallow", id: "P1-S01" },
+  {
+    problem: "undecided-output",
+    id: "P1-S01",
+    folder: "P1-S01-x",
+    path: "RESULTS.md",
+    decision: "repro/P1-S01-x/decision.md",
+  },
+  {
+    problem: "output-before-decision",
+    id: "P1-S01",
+    folder: "P1-S01-x",
+    path: "RESULTS.md",
+    output: D,
+    decision: D,
+    same: true,
+  },
+  {
+    problem: "output-before-decision",
+    id: "P1-S01",
+    folder: "P1-S01-x",
+    path: "RESULTS.md",
+    output: A,
+    decision: D,
+    same: false,
+  },
+];
+
+it("each finding's message: what is wrong, what to do, the rule's page", () => {
+  // Guards: the whole text a person reads, for every problem.
+  expect(ALL.map(orderMessage)).toEqual([
+    "The decision of `P1-S01`, `repro/P1-S01-x/decision.md`, is matched by its output `**/*.md`: the decision would be one of the files its own run produces. What to do: keep the decision out of the outputs — a file the globs do not match, or one elsewhere in the paper — and set `mode.decision.file`." +
+      PAGE,
+    "The decision of `P1-S01`, `repro/d.md`, was committed and then deleted: it is not in HEAD, so the study has no decision its outputs could follow. What to do: restore the decision file as it was decided and commit it — never write a new one after the outputs; if the decision no longer holds, declare the study `exploratory` with the reason." +
+      PAGE,
+    "The decision of `P1-S01`, `repro/d`, is a directory, not a file: git answers for a directory with the commits of anything under it, so no order can be read from it. What to do: name the decision file itself in `mode.decision.file`." +
+      PAGE,
+    "The decision of `P1-S01`, `repro/l.md`, is a symbolic link, not a file: git records the link, not the file it points to, so no order can be read from it. What to do: name the decision file itself in `mode.decision.file`." +
+      PAGE,
+    "`P1-S01` is decided before its run, and git's history is the proof of that order, but git could not answer: git is not installed. What to do: lint the paper inside its git repository, with git installed." +
+      PAGE,
+    "`P1-S01` is decided before its run, and this clone is shallow: every file's first commit is the clone's first, so the order of decision and outputs cannot be read. What to do: fetch the whole history (`git fetch --unshallow`; in GitHub Actions, `actions/checkout` with `fetch-depth: 0`)." +
+      PAGE,
+    "`repro/P1-S01-x/RESULTS.md` is an output of `P1-S01` — in the tree, or in its history — and the study is decided before its run, but its decision `repro/P1-S01-x/decision.md` was never committed. What to do: write and commit the decision, check it with `npx paperlint studies gate <paper-dir> P1-S01`, then run the study again; if the output was made before any decision, declare the study `exploratory` with the reason. Deleting the output does not undo it: history keeps it." +
+      PAGE,
+    `\`repro/P1-S01-x/RESULTS.md\`, an output of \`P1-S01\`, entered history in ${"d".repeat(12)} (${D.date}), the very commit that first recorded its decision, ${"d".repeat(12)} (${D.date}). The study is declared decided before its run, and its history does not show that order. What to do: if the decision really came first, nothing in this history can prove it — declare the study \`exploratory\` with the reason, or run it again after the decision; never rewrite the history to make it pass.` +
+      PAGE,
+    `\`repro/P1-S01-x/RESULTS.md\`, an output of \`P1-S01\`, entered history in ${"a".repeat(12)} (${A.date}), not after the commit that first recorded its decision, ${"d".repeat(12)} (${D.date}). The study is declared decided before its run, and its history does not show that order. What to do: if the decision really came first, nothing in this history can prove it — declare the study \`exploratory\` with the reason, or run it again after the decision; never rewrite the history to make it pass.` +
+      PAGE,
+  ]);
+});