paperlint 13.1.0 → 13.2.0

package/src/domain/decide-first.ts

@@ -12,9 +12,16 @@
  *   - a history that cannot be read: no git, or a shallow clone, whose every file starts at the clone.
  *
  * Outputs that were committed and later deleted count: an output that existed before the decision is
- * evidence after it is gone. Pure.
+ * evidence after it is gone.
+ *
+ * A plan v2's gates (src/domain/study-gates.ts) are frozen by the same history: the plan as it stood
+ * at the commit of the study's first output must declare them exactly as HEAD's does. A gate changed
+ * since is `gates-changed` (a threshold moved after the result was seen); a plan absent from that
+ * commit, or one without gates there (v1, or unreadable), is `gates-not-fixed`. Pure.
  */
+import { isDeepEqual } from "remeda";
 import { README } from "./repro-layout.ts";
+import { GATE_NAMES, type PlanGates } from "./study-gates.ts";
 import {
   decisionAsOutput,
   inStudy,
@@ -68,6 +75,23 @@ export type OrderFacts =
       readonly files: readonly FirstCommit[];
     };
 
+/** The plan as it stood at the commit of the study's first output, as git gave it. */
+export type PlanThen =
+  | { readonly kind: "gates"; readonly gates: PlanGates }
+  /** No gates were fixed there: no plan at that commit, a plan v1, or a text that is no plan. */
+  | {
+      readonly kind: "no-gates";
+      readonly why: "absent" | "v1" | "unreadable";
+    }
+  | { readonly kind: "unknown"; readonly why: string };
+
+/** A plan v2's gates now, and the plan at the study's first output (the first that came after the decision). */
+export interface GateHistory {
+  readonly now: PlanGates;
+  readonly output: FirstCommit;
+  readonly then: PlanThen;
+}
+
 /** What the judge reads of one decide-first study. */
 export interface DecideFirstStudy {
   readonly id: string;
@@ -78,6 +102,8 @@ export interface DecideFirstStudy {
   /** Every file under the study's folder now, relative to it. */
   readonly tree: readonly string[];
   readonly facts: OrderFacts;
+  /** Only for a plan v2 with an output after its decision: nothing to freeze otherwise. */
+  readonly gates?: GateHistory | undefined;
 }
 
 /** One defect `repro/outputs-before-decision` reports. */
@@ -118,6 +144,24 @@ export type OrderFinding =
       readonly path: string;
       readonly decision: string;
     }
+  | {
+      readonly problem: "gates-changed";
+      readonly id: string;
+      readonly folder: string;
+      readonly plan: string;
+      readonly gate: (typeof GATE_NAMES)[number];
+      readonly then: unknown;
+      readonly now: unknown;
+      readonly output: FirstCommit;
+    }
+  | {
+      readonly problem: "gates-not-fixed";
+      readonly id: string;
+      readonly folder: string;
+      readonly plan: string;
+      readonly why: "absent" | "v1" | "unreadable";
+      readonly output: FirstCommit;
+    }
   | {
       readonly problem: "output-before-decision";
       readonly id: string;
@@ -141,6 +185,41 @@ export const outputCommits = (
   files: readonly FirstCommit[],
 ): readonly FirstCommit[] => files.filter((f) => outputOf(s)(f.path));
 
+/**
+ * The first output of the study that entered history after its decision: the commit whose plan the
+ * gates are held to. None while no output followed the decision.
+ */
+export const firstOutputAfter = (
+  s: DecideFirstStudy,
+  files: readonly FirstCommit[],
+): FirstCommit | null =>
+  outputCommits(s, files).find((f) => f.order === "after") ?? null;
+
+/** The findings of a plan v2's gates against the plan at the study's first output. */
+const gateFindings = (s: DecideFirstStudy): readonly OrderFinding[] => {
+  const g = s.gates;
+  if (g === undefined) return [];
+  const at = { id: s.id, folder: s.folder, plan: s.plan, output: g.output };
+  switch (g.then.kind) {
+    case "unknown":
+      return [{ problem: "no-history", id: s.id, why: g.then.why }];
+    case "no-gates":
+      return [{ problem: "gates-not-fixed", ...at, why: g.then.why }];
+    case "gates": {
+      const then = g.then.gates;
+      return GATE_NAMES.filter((n) => !isDeepEqual(then[n], g.now[n])).map(
+        (gate) => ({
+          problem: "gates-changed",
+          ...at,
+          gate,
+          then: then[gate],
+          now: g.now[gate],
+        }),
+      );
+    }
+  }
+};
+
 /** The findings of a history that was read. */
 const ordered = (
   s: DecideFirstStudy,
@@ -185,7 +264,7 @@ export const judgeOrder = (s: DecideFirstStudy): readonly OrderFinding[] => {
             glob,
           },
         ];
-  return [...clash, ...historyFindings(s)];
+  return [...clash, ...historyFindings(s), ...gateFindings(s)];
 };
 
 /** The findings of a decide-first study's history, whatever git could say of it. */
@@ -227,6 +306,13 @@ const NOT_A_FILE: Readonly<Record<NotAFile, string>> = {
   "special file": "git tracks only regular files",
 };
 
+/** What the plan was at the first output, when it fixed no gates. */
+const GATES_THEN: Readonly<Record<"absent" | "v1" | "unreadable", string>> = {
+  absent: "was not in history yet",
+  v1: "was a plan v1, which declares no gates,",
+  unreadable: "was no plan paperlint could read",
+};
+
 const short = (c: CommitAt): string => `${c.commit.slice(0, 12)} (${c.date})`;
 
 const HEADS: {
@@ -246,6 +332,10 @@ const HEADS: {
     `\`${f.id}\` is decided before its run, and this clone is shallow: every file's first commit is the clone's first, so the order of decision and outputs cannot be read. What to do: fetch the whole history (\`git fetch --unshallow\`; in GitHub Actions, \`actions/checkout\` with \`fetch-depth: 0\`).`,
   "undecided-output": (f) =>
     `\`repro/${f.folder}/${f.path}\` is an output of \`${f.id}\` — in the tree, or in its history — and the study is decided before its run, but its decision \`${f.decision}\` was never committed. What to do: write and commit the decision, check it with \`npx paperlint studies gate <paper-dir> ${f.id}\`, then run the study again; if the output was made before any decision, declare the study \`exploratory\` with the reason. Deleting the output does not undo it: history keeps it.`,
+  "gates-changed": (f) =>
+    `The gate \`${f.gate}\` in the plan of \`${f.id}\`, \`repro/${f.folder}/${f.plan}\`, was ${JSON.stringify(f.then)} when its first output, \`${f.output.path}\`, entered history in ${short(f.output)}, and is ${JSON.stringify(f.now)} now. A gate is fixed before the run: moved after a result exists, it can only follow the result. What to do: put the gate back as it was; if it was wrong, say so in the paper and run the study again under the corrected plan — never edit the history to make it pass.`,
+  "gates-not-fixed": (f) =>
+    `The plan of \`${f.id}\`, \`repro/${f.folder}/${f.plan}\`, ${GATES_THEN[f.why]} when its first output, \`${f.output.path}\`, entered history in ${short(f.output)}: its gates were not fixed before the run. What to do: run the study again now that the plan v2 is committed; if the outputs must stand, declare the study \`exploratory\` with the reason.`,
   "output-before-decision": (f) =>
     `\`repro/${f.folder}/${f.path}\`, an output of \`${f.id}\`, entered history in ${short(f.output)}, ${f.same ? "the very commit" : "not after the commit"} that first recorded its decision, ${short(f.decision)}. The study is declared decided before its run, and its history does not show that order. What to do: if the decision really came first, nothing in this history can prove it — declare the study \`exploratory\` with the reason, or run it again after the decision; never rewrite the history to make it pass.`,
 };

package/src/domain/study-gates.ts

@@ -0,0 +1,125 @@
+/**
+ * THE GATES OF A PLAN V2 — what a measurement study fixes before its run so that its number can be
+ * believed, and what its run must show after (docs/design/study-gates.md, issue #332).
+ *
+ * A plan says it is v2 by its `$schema`, `PLAN_V2`. A plan without it is v1 and is judged as before;
+ * a measurement study registered on or after `PLAN_V2_FROM` must be v2 (`repro/study-registered`).
+ *
+ * The five gates, each with ONE way to say "not this study" — the literal `"none"` — and one way to
+ * say "not decided yet" — null, which keeps the plan a draft. A key left out of a v2 plan is no plan:
+ *
+ *   we_lose_if     inside a decide-first mode only: the number in `result.json` and the threshold
+ *                  at which the study's claim loses. An exploratory study has no claim to lose, and
+ *                  cannot write one.
+ *   second_check   per labelling step, the least Cohen's kappa between two raters on it.
+ *   self_test      the check the study runs is first run on inputs whose answer is known: a script
+ *                  that makes and runs them, or one input it must catch and one it must pass.
+ *   funnel         the reasons a drawn unit may be dropped; the result counts each.
+ *   recall_audit   how complete the step is that extracts units from artefacts: a person reads
+ *                  `n_files` sampled files in full, and what the extractor found of the units the
+ *                  person found must be at least `min_recall`.
+ *
+ * After the run the study writes `result.json` (`RESULT_FILE`) beside its plan, and
+ * `repro/study-result` judges it against these (src/domain/study-result.ts). What no machine sees: a
+ * `"none"` that is false. The author's check of the plan is where that is caught. Pure.
+ */
+
+/** The `$schema` of a plan v2: the JSON Schema the package ships for it. */
+export const PLAN_V2 =
+  "https://unpkg.com/paperlint/schemas/plan.v2.schema.json";
+
+/** A measurement study registered on this day or later must have a plan v2. */
+export const PLAN_V2_FROM = "2026-10-31";
+
+/** The one spelling of "this study has none of it". */
+export const NONE = "none";
+export type None = typeof NONE;
+
+/** The claim loses when the number named `result` is below, or above, a threshold: exactly one. */
+export type WeLoseIf =
+  | { readonly result: string; readonly below: number }
+  | { readonly result: string; readonly above: number };
+
+/** One labelling step, judged by two raters: the least kappa between them it needs. */
+export interface KappaStep {
+  readonly step: string;
+  readonly min_kappa: number;
+}
+
+export type SecondCheck = None | readonly KappaStep[];
+
+/** The check is first run on known answers: made by a script, or a pair of inputs. Paths are in the study's folder. */
+export type SelfTest =
+  | None
+  | { readonly made_by: string }
+  | { readonly must_catch: string; readonly must_pass: string };
+
+/** The reasons a drawn unit may be dropped, each once. */
+export type Funnel = None | readonly string[];
+
+/**
+ * The audit of the extractor: how many files a person reads in full, and the least share of the units
+ * found there that the extractor must have found too.
+ */
+export interface RecallAuditPlan {
+  readonly n_files: number;
+  readonly min_recall: number;
+}
+
+export type RecallAudit = None | RecallAuditPlan;
+
+/** The file a study of a plan v2 writes its numbers to, beside its plan. */
+export const RESULT_FILE = "result.json";
+
+/**
+ * The five gates of a plan, as the plan declares them (null while undecided): what is frozen once the
+ * study's first output exists (`repro/outputs-before-decision`, src/domain/decide-first.ts).
+ */
+export interface PlanGates {
+  readonly we_lose_if: WeLoseIf | null;
+  readonly second_check: SecondCheck | null;
+  readonly self_test: SelfTest | null;
+  readonly funnel: Funnel | null;
+  readonly recall_audit: RecallAudit | null;
+}
+
+/** The gates' names, in the order they are reported. */
+export const GATE_NAMES = [
+  "we_lose_if",
+  "second_check",
+  "self_test",
+  "funnel",
+  "recall_audit",
+] as const satisfies readonly (keyof PlanGates)[];
+
+/** A plan's gates, from the parts that hold them: none for a plan v1, which declares no gates. */
+export const gatesOf = (plan: {
+  readonly $schema?: string | undefined;
+  readonly mode?:
+    | {
+        readonly kind: string;
+        readonly we_lose_if?: WeLoseIf | null | undefined;
+      }
+    | undefined;
+  readonly second_check?: SecondCheck | null | undefined;
+  readonly self_test?: SelfTest | null | undefined;
+  readonly funnel?: Funnel | null | undefined;
+  readonly recall_audit?: RecallAudit | null | undefined;
+}): PlanGates | null =>
+  plan.$schema === PLAN_V2
+    ? {
+        we_lose_if: plan.mode?.we_lose_if ?? null,
+        second_check: plan.second_check ?? null,
+        self_test: plan.self_test ?? null,
+        funnel: plan.funnel ?? null,
+        recall_audit: plan.recall_audit ?? null,
+      }
+    : null;
+
+/** Whether the number `value` makes the claim lose by `w`. */
+export const loses = (w: WeLoseIf, value: number): boolean =>
+  "below" in w ? value < w.below : value > w.above;
+
+/** The threshold of `w`, and its side, in words: `below 0.2`. */
+export const thresholdText = (w: WeLoseIf): string =>
+  "below" in w ? `below ${String(w.below)}` : `above ${String(w.above)}`;

package/src/domain/study-measurements.test.ts

@@ -24,6 +24,7 @@ import {
   type IndexInput,
   folderKey,
 } from "./study-registry.ts";
+import { PLAN_V2, PLAN_V2_FROM } from "./study-gates.ts";
 
 /** No plan and no history: the index with every mode column `—`. */
 const NO_FACTS: IndexInput = { plans: new Map(), history: new Map() };
@@ -635,3 +636,73 @@ describe("studyMessage — the new findings stand alone", () => {
     );
   });
 });
+
+describe("PLAN_V2_FROM", () => {
+  it("is 2026-10-31: the first study the gates were written for was registered that day", () => {
+    expect(PLAN_V2_FROM).toBe("2026-10-31");
+  });
+});
+
+describe("plans — a measurement registered from PLAN_V2_FROM has a plan v2", () => {
+  const V2: StudyPlan = {
+    ...PLAN,
+    $schema: PLAN_V2,
+    second_check: "none",
+    self_test: "none",
+    funnel: "none",
+    recall_audit: "none",
+  };
+  const onDay = (date: string, kind: Study["kind"] = "measurement") =>
+    reg({
+      studies: [study({ measurement: "M1", plan: PLAN_FILE, date, kind })],
+    });
+
+  it("a measurement registered on the day, its plan v1: reported, with what to add", () => {
+    const f = findings(onDay(PLAN_V2_FROM));
+    expect(f).toEqual([
+      {
+        problem: "plan-version",
+        id: "P1-S01",
+        path: PLAN_FILE,
+        date: PLAN_V2_FROM,
+      },
+    ]);
+    expect(f.map(studyMessage).join("")).toContain(
+      `add \`"$schema": "${PLAN_V2}"\``,
+    );
+  });
+
+  it("silent on a plan v1 registered before the day, a spike's from it, and a plan v2", () => {
+    const v2 = onDay(PLAN_V2_FROM);
+    // Guards: old plans keep passing; only a measurement is asked; v2 answers it.
+    expect([
+      findings(onDay("2026-10-30")),
+      findings(onDay(PLAN_V2_FROM, "spike")),
+      findings(v2, {
+        plans: new Map([[folderKey("P1-S01-census", PLAN_FILE), read(V2)]]),
+      }),
+    ]).toEqual([[], [], []]);
+  });
+
+  it("a draft plan v1 from the day is reported too; an unreadable one is bad-plan's alone", () => {
+    const key = folderKey("P1-S01-census", PLAN_FILE);
+    const asDraft = new Map<string, PlanState>([
+      [key, draft({ ...PLAN, author_check: null })],
+    ]);
+    const unreadable = new Map<string, PlanState>([
+      [key, { kind: "unreadable", why: "x" }],
+    ]);
+    const r = reg({
+      studies: [
+        study({ plan: PLAN_FILE, date: PLAN_V2_FROM, status: "designing" }),
+      ],
+      measurements: [],
+    });
+    expect(findings(r, { plans: asDraft }).map((x) => x.problem)).toEqual([
+      "plan-version",
+    ]);
+    expect(findings(r, { plans: unreadable }).map((x) => x.problem)).toEqual([
+      "bad-plan",
+    ]);
+  });
+});

package/src/domain/study-mode.ts

@@ -18,6 +18,7 @@
 import { posix } from "node:path";
 import { REPRO_DIR } from "./repro-layout.ts";
 import type { StudyKind } from "./study-registry.ts";
+import type { WeLoseIf } from "./study-gates.ts";
 
 /** A decision recorded in a file: where, and what it decides. */
 export interface Decision {
@@ -32,6 +33,11 @@ export interface DecideFirst {
   /** Globs relative to the study's folder: the files the run produces. */
   readonly outputs: readonly string[];
   readonly decision: Decision;
+  /**
+   * A plan v2's falsifier (src/domain/study-gates.ts): when the claim loses; null while undecided.
+   * Absent from a plan v1; an exploratory mode cannot carry one.
+   */
+  readonly we_lose_if?: WeLoseIf | null | undefined;
 }
 
 /** Exploratory: no gate, and reported as exploratory. */

package/src/domain/study-registry.ts

@@ -19,6 +19,14 @@
 import { isAbsolute, posix, win32 } from "node:path";
 import { STUDIES_FILE, STUDY_SLUG } from "./repro-layout.ts";
 import { needsMode, type StudyMode } from "./study-mode.ts";
+import {
+  PLAN_V2,
+  PLAN_V2_FROM,
+  type Funnel,
+  type RecallAudit,
+  type SecondCheck,
+  type SelfTest,
+} from "./study-gates.ts";
 import {
   decisionCell,
   firstOutputCell,
@@ -187,6 +195,16 @@ export interface StudyPlan {
    * exploratory (src/domain/study-mode.ts). A measurement study's plan is not ready without it.
    */
   readonly mode?: StudyMode | undefined;
+  /** A plan v2 says so here (src/domain/study-gates.ts); a plan without it is v1. */
+  readonly $schema?: typeof PLAN_V2 | undefined;
+  /** Plan v2: the least kappa of each labelling step; `"none"`; null while undecided. */
+  readonly second_check?: SecondCheck | null | undefined;
+  /** Plan v2: how the study's check is run on known answers first; `"none"`; null while undecided. */
+  readonly self_test?: SelfTest | null | undefined;
+  /** Plan v2: the reasons a drawn unit may be dropped; `"none"`; null while undecided. */
+  readonly funnel?: Funnel | null | undefined;
+  /** Plan v2: how complete the extractor is, audited by a person reading files; `"none"`; null while undecided. */
+  readonly recall_audit?: RecallAudit | null | undefined;
 }
 
 /**
@@ -216,12 +234,20 @@ export const PLAN_FIELDS = [
   "author_check",
   "extra",
   "mode",
+  "$schema",
+  "second_check",
+  "self_test",
+  "funnel",
+  "recall_audit",
 ] as const;
 
 /** A plan with empty required texts: the author has not approved it. */
 export type DraftPlan = StudyPlan & { readonly author_check: null };
 
-/** The texts a plan must fill before the study runs, in the order they are named. */
+/**
+ * The fields a plan must fill before the study runs, in the order they are named: its texts, and a
+ * plan v2's gates, each null until decided.
+ */
 export const PLAN_TEXT_FIELDS = [
   "question",
   "population.source",
@@ -232,6 +258,11 @@ export const PLAN_TEXT_FIELDS = [
   "mode.decision.file",
   "mode.decision.note",
   "mode.reason",
+  "mode.we_lose_if",
+  "second_check",
+  "self_test",
+  "funnel",
+  "recall_audit",
 ] as const;
 export type PlanTextField = (typeof PLAN_TEXT_FIELDS)[number];
 
@@ -276,9 +307,11 @@ export const DECISION_FILE = "decision.md";
  * The plan `paperlint studies new` writes for a study of `kind` in `folder`: every field present, its
  * value to fill. Its draw is of all the population (a census); a sampled study changes it to
  * `{ "kind": "seeded", "method", "seed" }`. A study that must declare its mode gets the default one,
- * decided before the run, its decision file named and its outputs and note to fill.
+ * decided before the run, its decision file named and its outputs and note to fill; its plan is v2,
+ * its gates null until decided (src/domain/study-gates.ts).
  */
 export const planTemplate = (kind: StudyKind, folder: string): DraftPlan => ({
+  ...(needsMode(kind) ? { $schema: PLAN_V2 } : {}),
   question: "",
   population: { source: "", draw: { kind: "all", method: "" } },
   arms: [],
@@ -290,7 +323,12 @@ export const planTemplate = (kind: StudyKind, folder: string): DraftPlan => ({
           kind: "decide_before_run",
           outputs: [],
           decision: { file: decisionPath(folder), note: "" },
+          we_lose_if: null,
         },
+        second_check: null,
+        self_test: null,
+        funnel: null,
+        recall_audit: null,
       }
     : {}),
 });
@@ -453,6 +491,12 @@ export type StudyFinding =
       readonly problem: "legacy-plan";
       readonly id: string;
       readonly path: string;
+    }
+  | {
+      readonly problem: "plan-version";
+      readonly id: string;
+      readonly path: string;
+      readonly date: string;
     };
 
 /** The findings `repro/study-plan-name` reports, a warning; every other is `repro/study-registered`'s. */
@@ -765,6 +809,21 @@ const checkProblem = (
     : { why: "not-author", by: check.by, author: r.author };
 };
 
+/**
+ * A measurement registered on or after `PLAN_V2_FROM` whose plan parsed and is v1: its gates
+ * (src/domain/study-gates.ts) are not declared. A plan that does not parse is `bad-plan`'s.
+ */
+const planVersion: RowCheck = ({ s, input }) => {
+  const path = s.plan;
+  if (path === undefined || !needsMode(s.kind) || s.date < PLAN_V2_FROM)
+    return null;
+  const state = planStateOf(input, s, path);
+  return (state.kind === "ready" || state.kind === "draft") &&
+    state.plan.$schema === undefined
+    ? { problem: "plan-version", id: s.id, path, date: s.date }
+    : null;
+};
+
 /** The checks of a row, in the order their findings are reported. */
 const ROW_CHECKS: readonly RowCheck[] = [
   badId,
@@ -778,6 +837,7 @@ const ROW_CHECKS: readonly RowCheck[] = [
   noSuccessor,
   unknownMeasurement,
   badPlan,
+  planVersion,
   legacyPlan,
   authorCheck,
 ];
@@ -1170,6 +1230,8 @@ const HEADS: { readonly [P in Problem]: (f: FindingOf[P]) => string } = {
         return `${head} the \`author_check\` in its plan is by ${f.by}, not by the registry's \`author\`, ${f.author}. What to do: only the author approves the drawn units of a headline measurement; record their own answer.`;
     }
   },
+  "plan-version": (f) =>
+    `The plan of ${tick(f.id)}, ${tick(f.path)} in its folder, is a plan v1, and ${tick(f.id)} is a measurement registered on ${f.date}: from ${PLAN_V2_FROM} a measurement's plan is v2 and declares the gates its number is believed by. What to do: add \`"$schema": "${PLAN_V2}"\`, then \`we_lose_if\` in a decide-first \`mode\`, \`second_check\`, \`self_test\`, \`funnel\` and \`recall_audit\` — each \`"none"\` where the study has none of it, null until decided (docs/design/study-gates.md).`,
   "legacy-plan": (f) =>
     `The plan of ${tick(f.id)} is ${tick(f.path)}, its old name; it is read as the plan. What to do: rename it to \`${PLAN_FILE}\` in the study's folder (\`git mv\`) and set the row's \`plan\` to \`${PLAN_FILE}\`.`,
 };