paperlint 10.1.0 → 10.2.0

package/src/studies.ts

@@ -0,0 +1,259 @@
+/**
+ * `paperlint studies new|index <paper-dir>` — the two commands that write a paper's study registry,
+ * `repro/studies.json`, and its index in `repro/README.md` (docs/design/study-registry.md).
+ *
+ *   new <paper-dir> "<title>" [--kind <kind>]
+ *            the next id; `repro/<id>-<slug>/README.md` with the id and title in its frontmatter; the
+ *            row, `status: designing`, today's date; the index regenerated
+ *   index <paper-dir>
+ *            the index block of `repro/README.md`, regenerated from the registry
+ *
+ * Nothing else changes a row: a study's status, result and `in_paper` are edited in `studies.json`, and
+ * `repro/study-registered` checks them. The prefix is the paper's own choice, so a paper without a
+ * registry is refused with the file to write rather than given a guessed one.
+ */
+import { join, relative, resolve } from "node:path";
+import { callerPath } from "./caller-path.ts";
+import type { AbsolutePath } from "./domain/paths.ts";
+import { README, REPRO_DIR } from "./domain/repro-layout.ts";
+import {
+  DEFAULT_KIND,
+  STUDIES_FILE,
+  STUDY_KINDS,
+  addStudy,
+  indexInSync,
+  isStudyKind,
+  registryText,
+  studyReadme,
+  withIndex,
+  type Study,
+  type StudyKind,
+  type StudyRegistry,
+} from "./domain/study-registry.ts";
+import type { Files } from "./ports/files.ts";
+import type { StudyRegistryCodec } from "./ports/study-registry-codec.ts";
+import type { ListDir } from "./ports/talk-media.ts";
+import { registryOf } from "./study-registry-rule.ts";
+
+/** What one `studies` run was asked, after the command line. */
+export interface StudiesArgs {
+  readonly sub: string | undefined;
+  readonly paperDir: string | undefined;
+  /** `new`'s title; for `index`, an argument too many. */
+  readonly title: string | undefined;
+  /** Positional arguments past the title: refused. */
+  readonly extra: readonly string[];
+  /** `--kind <kind>`; null: the default. */
+  readonly kind: string | null;
+}
+
+export interface StudiesDeps {
+  readonly files: Files;
+  readonly listDir: ListDir;
+  readonly codec: StudyRegistryCodec;
+  readonly cwd: string;
+  /** Today, `YYYY-MM-DD`: the root reads the clock. */
+  readonly today: string;
+  readonly log: (line: string) => void;
+  readonly err: (line: string) => void;
+}
+
+const NEW_USAGE =
+  'npx paperlint studies new <paper-dir> "<title>" [--kind <kind>]';
+const INDEX_USAGE = "npx paperlint studies index <paper-dir>";
+const SUBS = `new or index — ${NEW_USAGE} · ${INDEX_USAGE}`;
+
+const STATUS_FILE = "PIPELINE-STATUS.md";
+
+/** The paper folder a run works in: absolute, and as the messages show it. */
+interface Paper {
+  readonly dir: string;
+  readonly shown: string;
+}
+
+/** A run's outcome: the exit code, and the line that explains a refusal. */
+type Refusal = { readonly code: 1 | 2; readonly line: string };
+
+const refuse = (code: 1 | 2, line: string): Refusal => ({ code, line });
+
+/** The paper folder's registry, or the refusal naming why there is none to write to. */
+function registryIn(p: Paper, deps: StudiesDeps): StudyRegistry | Refusal {
+  const r = registryOf(p.dir, deps);
+  const file = `${p.shown}/${REPRO_DIR}/${STUDIES_FILE}`;
+  switch (r.kind) {
+    case "absent":
+      return refuse(
+        1,
+        `${file} does not exist. Its prefix is the paper's own choice — a capital letter, then capitals or digits, one per paper (P1) — so write the file, then run this again:\n\n{ "prefix": "P1", "studies": [] }`,
+      );
+    case "unreadable":
+      return refuse(1, `${file} is not a study registry: ${r.why}`);
+    case "read":
+      return r.registry;
+  }
+}
+
+const isRefusal = (v: unknown): v is Refusal =>
+  typeof v === "object" && v !== null && "code" in v && "line" in v;
+
+const text = (s: string): Uint8Array => new TextEncoder().encode(s);
+
+/** The README's text; null when there is none. */
+const readmeOf = (dir: string, files: Files): string | null => {
+  const b = files.readBytes(callerPath(join(dir, REPRO_DIR, README)));
+  return b === null ? null : new TextDecoder().decode(b);
+};
+
+const studiesCount = (r: StudyRegistry): string =>
+  `${String(r.studies.length)} ${r.studies.length === 1 ? "study" : "studies"}`;
+
+/**
+ * Write the index of `r` into the README; whether it wrote. A block already in sync is left as it
+ * is, so the padding a formatter gave the table is never undone.
+ */
+function writeIndex(dir: string, r: StudyRegistry, files: Files): boolean {
+  const before = readmeOf(dir, files);
+  if (indexInSync(before, r)) return false;
+  files.writeAtomic(
+    callerPath(join(dir, REPRO_DIR, README)),
+    text(withIndex(before, r)),
+  );
+  return true;
+}
+
+/** The refusal when something that is not a file sits where `repro/README.md` goes; null when none. */
+function readmeBlocked(p: Paper, deps: StudiesDeps): Refusal | null {
+  const listed = deps.listDir(callerPath(join(p.dir, REPRO_DIR)));
+  const path = callerPath(join(p.dir, REPRO_DIR, README));
+  return listed.includes(README) && !deps.files.isFile(path)
+    ? refuse(
+        1,
+        `${p.shown}/${REPRO_DIR}/${README} is not a file, so the index cannot be written there. Move it, then run this again.`,
+      )
+    : null;
+}
+
+/** `studies index`: the README's block regenerated from the registry. */
+function runIndex(
+  p: Paper,
+  a: StudiesArgs,
+  deps: StudiesDeps,
+): Refusal | readonly string[] {
+  const [surplus] = [...(a.title === undefined ? [] : [a.title]), ...a.extra];
+  if (surplus !== undefined)
+    return refuse(
+      2,
+      `studies index takes only the paper's folder — \`${surplus}\` is one argument too many`,
+    );
+  const r = registryIn(p, deps);
+  if (isRefusal(r)) return r;
+  const blocked = readmeBlocked(p, deps);
+  if (blocked !== null) return blocked;
+  const readme = `${p.shown}/${REPRO_DIR}/${README}`;
+  return [
+    writeIndex(p.dir, r, deps.files)
+      ? `wrote the index of ${studiesCount(r)} in ${readme}`
+      : `the index in ${readme} is in sync with ${studiesCount(r)}`,
+  ];
+}
+
+/** `new`'s title and kind from the command line, or the refusal. */
+function newArgs(
+  a: StudiesArgs,
+): Refusal | { readonly title: string; readonly kind: StudyKind } {
+  if (a.title === undefined)
+    return refuse(2, `studies new needs the study's title: ${NEW_USAGE}`);
+  const [surplus] = a.extra;
+  if (surplus !== undefined)
+    return refuse(
+      2,
+      `studies new takes the paper's folder and one title — \`${surplus}\` is one argument too many (quote a title of several words)`,
+    );
+  const kind = a.kind ?? DEFAULT_KIND;
+  return isStudyKind(kind)
+    ? { title: a.title, kind }
+    : refuse(
+        2,
+        `--kind ${kind} is not a kind of study: one of ${STUDY_KINDS.join(", ")}`,
+      );
+}
+
+/** Write a new study's README.md, the registry, and the index; the lines that say so. */
+function writeStudy(
+  p: Paper,
+  added: { readonly registry: StudyRegistry; readonly study: Study },
+  files: Files,
+): readonly string[] {
+  const { study, registry } = added;
+  const at = (...names: readonly string[]): AbsolutePath =>
+    callerPath(join(p.dir, REPRO_DIR, ...names));
+  files.writeAtomic(at(study.folder, README), text(studyReadme(study)));
+  files.writeAtomic(at(STUDIES_FILE), text(registryText(registry)));
+  writeIndex(p.dir, registry, files);
+  const repro = `${p.shown}/${REPRO_DIR}`;
+  return [
+    `created ${repro}/${study.folder}/${README}`,
+    `added ${study.id} to ${repro}/${STUDIES_FILE}: kind ${study.kind}, status ${study.status}, ${study.date}`,
+    `wrote the index of ${studiesCount(registry)} in ${repro}/${README}`,
+    `Next: write the study's one-sentence question in its row (\`question\`) and in its README.md. When it ends, set \`status\`, \`result\`, \`result_file\` and \`in_paper\` in its row, then run \`npx paperlint studies index ${p.shown}\`.`,
+  ];
+}
+
+/** `studies new`: the folder, its README.md, the row, the index. */
+function runNew(
+  p: Paper,
+  a: StudiesArgs,
+  deps: StudiesDeps,
+): Refusal | readonly string[] {
+  const args = newArgs(a);
+  if (isRefusal(args)) return args;
+  const r = registryIn(p, deps);
+  if (isRefusal(r)) return r;
+  const blocked = readmeBlocked(p, deps);
+  if (blocked !== null) return blocked;
+  const added = addStudy(r, { ...args, date: deps.today });
+  if (!added.ok) return refuse(2, added.why);
+  const folder = added.study.folder;
+  return deps.listDir(callerPath(join(p.dir, REPRO_DIR))).includes(folder)
+    ? refuse(
+        1,
+        `${p.shown}/${REPRO_DIR}/${folder}/ already exists; it is never overwritten. Register it by hand, or remove it and run this again.`,
+      )
+    : writeStudy(p, added, deps.files);
+}
+
+/** The paper folder the command line names, or the refusal. */
+function paperOf(a: StudiesArgs, deps: StudiesDeps): Refusal | Paper {
+  if (a.sub === undefined)
+    return refuse(2, `studies needs a subcommand: ${SUBS}`);
+  if (a.sub !== "new" && a.sub !== "index")
+    return refuse(2, `unknown subcommand \`studies ${a.sub}\`: ${SUBS}`);
+  if (a.paperDir === undefined)
+    return refuse(
+      2,
+      `studies ${a.sub} needs the paper's folder: ${a.sub === "new" ? NEW_USAGE : INDEX_USAGE}`,
+    );
+  const dir = resolve(deps.cwd, a.paperDir);
+  const shown = relative(deps.cwd, dir) || ".";
+  return deps.files.isFile(callerPath(join(dir, STATUS_FILE)))
+    ? { dir, shown }
+    : refuse(2, `${shown} is not a paper folder: it has no ${STATUS_FILE}`);
+}
+
+/** The run: 0 done, 1 refused by the paper's state, 2 refused by the command line. */
+export function runStudies(a: StudiesArgs, deps: StudiesDeps): number {
+  const p = paperOf(a, deps);
+  const out = isRefusal(p)
+    ? p
+    : a.sub === "new"
+      ? runNew(p, a, deps)
+      : runIndex(p, a, deps);
+  if (isRefusal(out)) {
+    deps.err(out.line);
+    return out.code;
+  }
+  out.forEach((line) => {
+    deps.log(line);
+  });
+  return 0;
+}

package/src/study-registry-rule.test.ts

@@ -0,0 +1,317 @@
+/**
+ * `repro/study-registered` (`src/study-registry-rule.ts`) on paper folders held in memory, run the way
+ * ESLint runs it: `create`, then `root`. Both halves: a registered `repro/` with its index in sync is
+ * silent; each defect — an unregistered folder, a dangling row, a bad id, `done` without a result,
+ * `superseded` without a target, a stale index, a missing or unreadable registry — is reported once,
+ * on the first line of `PIPELINE-STATUS.md`, with the domain's text.
+ */
+import { describe, expect, it } from "vitest";
+import { memoryFiles, memoryListDir } from "./adapters/memory/index.ts";
+import { studyRegistryCodec } from "./adapters/study-registry/index.ts";
+import {
+  registryText,
+  studyMessage,
+  withIndex,
+  type Study,
+  type StudyRegistry,
+} from "./domain/study-registry.ts";
+import {
+  registryOf,
+  STUDY_REGISTRY_RULE_LEVELS,
+  studyFindings,
+  studyRegistryRules,
+} from "./study-registry-rule.ts";
+
+const DIR = "/work/papers/example";
+const SCAFFOLD = {
+  [`${DIR}/PIPELINE-STATUS.md`]: "# status\n",
+  [`${DIR}/paper.tex`]: "\\documentclass{article}\n",
+};
+
+const study = (over: Partial<Study> = {}): Study => ({
+  id: "P1-S01",
+  folder: "P1-S01-baseline",
+  title: "Baseline pass rate",
+  question: "How often does the agent pass?",
+  kind: "measurement",
+  status: "designing",
+  date: "2026-10-15",
+  in_paper: "no",
+  ...over,
+});
+
+/** A registered `repro/`: the registry, its index, and a README.md in each row's folder. */
+const registered = (
+  r: StudyRegistry,
+  readme: string | null = withIndex("# Reproduction index\n", r),
+): Record<string, string> => ({
+  ...SCAFFOLD,
+  [`${DIR}/repro/studies.json`]: registryText(r),
+  ...(readme === null ? {} : { [`${DIR}/repro/README.md`]: readme }),
+  ...Object.fromEntries(
+    r.studies.map((s) => [`${DIR}/repro/${s.folder}/README.md`, "# s\n"]),
+  ),
+});
+
+const P1 = (...studies: Study[]): StudyRegistry => ({ prefix: "P1", studies });
+
+interface Reported {
+  readonly messageId: string;
+  readonly message: string;
+  readonly line: number;
+}
+
+/** The rule over the folder `tree` describes, reached through `file` (default the scorecard). */
+function lint(
+  tree: Readonly<Record<string, string>>,
+  file = `${DIR}/PIPELINE-STATUS.md`,
+): Reported[] {
+  const files = memoryFiles(tree);
+  const rule = studyRegistryRules({
+    files,
+    listDir: memoryListDir(files),
+    codec: studyRegistryCodec,
+  })["study-registered"];
+  const out: Reported[] = [];
+  const visitor = rule.create({
+    filename: file,
+    sourceCode: { getLocFromIndex: () => ({ line: 1, column: 0 }) },
+    report: (d) => {
+      const template = rule.meta.messages[d.messageId] ?? "";
+      const message = Object.entries(d.data).reduce(
+        (m, [k, v]) => m.split(`{{${k}}}`).join(String(v)),
+        template,
+      );
+      out.push({ messageId: d.messageId, message, line: d.loc.start.line });
+    },
+  });
+  visitor.root?.();
+  return out;
+}
+
+const messages = (tree: Readonly<Record<string, string>>): string[] =>
+  lint(tree).map((r) => r.message);
+
+describe("repro/study-registered — silent", () => {
+  it("on a registered repro/ with its index in sync, beside tools/, data/ and byproducts", () => {
+    const done = study({
+      id: "P1-S02",
+      folder: "P1-S02-ablation",
+      status: "done",
+      result: "41 of 60 pass",
+      result_file: "out/results.tsv",
+    });
+    expect(
+      lint({
+        ...registered(P1(study(), done)),
+        [`${DIR}/repro/P1-S02-ablation/out/results.tsv`]: "x\n",
+        [`${DIR}/repro/tools/run.py`]: "x",
+        [`${DIR}/repro/data/numbers.tsv`]: "x",
+        [`${DIR}/repro/__pycache__/x.pyc`]: "x",
+        [`${DIR}/repro/node_modules/a/index.js`]: "x",
+        [`${DIR}/repro/.gitignore`]: "x",
+      }),
+    ).toEqual([]);
+  });
+
+  it("on a paper with no repro/", () => {
+    expect(lint(SCAFFOLD)).toEqual([]);
+  });
+
+  it("on a repro/ with only its shared folders and no registry", () => {
+    expect(
+      lint({
+        ...SCAFFOLD,
+        [`${DIR}/repro/README.md`]: "# index\n",
+        [`${DIR}/repro/tools/.gitkeep`]: "",
+        [`${DIR}/repro/data/.gitkeep`]: "",
+      }),
+    ).toEqual([]);
+  });
+
+  it("on any file but PIPELINE-STATUS.md", () => {
+    expect(
+      lint(
+        { ...SCAFFOLD, [`${DIR}/repro/spike-2026-10-12/README.md`]: "x" },
+        `${DIR}/paper.tex`,
+      ),
+    ).toEqual([]);
+  });
+});
+
+describe("repro/study-registered — reports, once each, on line 1", () => {
+  it("an unregistered folder", () => {
+    expect(
+      lint({
+        ...registered(P1(study())),
+        [`${DIR}/repro/spike-2026-10-12/README.md`]: "x",
+      }),
+    ).toEqual([
+      {
+        messageId: "unregistered",
+        message: studyMessage({
+          problem: "unregistered",
+          folder: "spike-2026-10-12",
+          next: "P1-S02",
+        }),
+        line: 1,
+      },
+    ]);
+  });
+
+  it("a dangling row", () => {
+    const tree = registered(P1(study()));
+    const { [`${DIR}/repro/P1-S01-baseline/README.md`]: gone, ...rest } = tree;
+    expect(gone).toBeDefined();
+    expect(messages(rest)).toEqual([
+      studyMessage({
+        problem: "no-folder",
+        id: "P1-S01",
+        folder: "P1-S01-baseline",
+      }),
+    ]);
+  });
+
+  it("a bad id", () => {
+    expect(
+      messages(registered(P1(study({ id: "P1-S1", folder: "P1-S1-x" })))),
+    ).toEqual([studyMessage({ problem: "bad-id", id: "P1-S1", prefix: "P1" })]);
+  });
+});
+
+describe("repro/study-registered — reports a row's status and the index", () => {
+  it("done without a result", () => {
+    expect(messages(registered(P1(study({ status: "done" }))))).toEqual([
+      studyMessage({
+        problem: "no-result",
+        id: "P1-S01",
+        missing: ["result", "result_file"],
+      }),
+    ]);
+  });
+
+  it("done with a result_file that is not there", () => {
+    expect(
+      messages(
+        registered(
+          P1(study({ status: "done", result: "x", result_file: "r.tsv" })),
+        ),
+      ),
+    ).toEqual([
+      studyMessage({
+        problem: "result-file",
+        id: "P1-S01",
+        path: "r.tsv",
+        why: "missing",
+      }),
+    ]);
+  });
+
+  it("superseded without a target", () => {
+    expect(messages(registered(P1(study({ status: "superseded" }))))).toEqual([
+      studyMessage({
+        problem: "no-successor",
+        id: "P1-S01",
+        superseded_by: null,
+      }),
+    ]);
+  });
+
+  it("a stale index", () => {
+    const r = P1(study());
+    expect(messages(registered(r, withIndex("# x\n", P1())))).toEqual([
+      studyMessage({ problem: "stale-index", why: "differs" }),
+    ]);
+  });
+});
+
+describe("repro/study-registered — reports the registry itself, once", () => {
+  it("study folders and no studies.json: one finding naming the file", () => {
+    expect(
+      lint({
+        ...SCAFFOLD,
+        [`${DIR}/repro/README.md`]: "# index\n",
+        [`${DIR}/repro/b-2026-10-02/README.md`]: "x",
+        [`${DIR}/repro/a-2026-10-01/README.md`]: "x",
+        [`${DIR}/repro/tools/run.py`]: "x",
+      }),
+    ).toEqual([
+      {
+        messageId: "no-registry",
+        message: studyMessage({
+          problem: "no-registry",
+          folders: ["a-2026-10-01", "b-2026-10-02"],
+        }),
+        line: 1,
+      },
+    ]);
+  });
+
+  it("a studies.json that is not a registry: one finding with the reasons", () => {
+    expect(
+      lint({
+        ...SCAFFOLD,
+        [`${DIR}/repro/studies.json`]: '{ "studies": [] }',
+        [`${DIR}/repro/P1-S01-x/README.md`]: "x",
+      }),
+    ).toEqual([
+      {
+        messageId: "unreadable",
+        message: studyMessage({
+          problem: "unreadable",
+          why: "prefix: Invalid input: expected string, received undefined",
+        }),
+        line: 1,
+      },
+    ]);
+  });
+});
+
+describe("repro/study-registered — level", () => {
+  it("is an error from the start", () => {
+    expect(STUDY_REGISTRY_RULE_LEVELS).toEqual({
+      "repro/study-registered": "error",
+    });
+  });
+});
+
+describe("repro/study-registered — a directory where the registry should be", () => {
+  it("is read as no registry, not thrown on (the disk adapter throws EISDIR on a directory)", () => {
+    const files = {
+      ...memoryFiles({}),
+      isFile: () => false,
+      readBytes: (): Uint8Array => {
+        throw Object.assign(new Error("EISDIR"), { code: "EISDIR" });
+      },
+    };
+    expect(registryOf(DIR, { files, codec: studyRegistryCodec })).toEqual({
+      kind: "absent",
+    });
+  });
+});
+
+describe("repro/study-registered — a result_file the disk cannot hold", () => {
+  it("reads as missing, and the disk is never asked (the disk adapter throws on a NUL)", () => {
+    const r = P1(study({ status: "running", result_file: "a\u0000b.tsv" }));
+    const files = memoryFiles({
+      ...SCAFFOLD,
+      [`${DIR}/repro/studies.json`]: registryText(r),
+      [`${DIR}/repro/README.md`]: withIndex("# index\n", r),
+      [`${DIR}/repro/P1-S01-baseline/README.md`]: "# s\n",
+    });
+    const probe = {
+      ...files,
+      isFile: (p: Parameters<typeof files.isFile>[0]): boolean => {
+        if (p.includes("\0")) throw new Error("ERR_INVALID_ARG_VALUE");
+        return files.isFile(p);
+      },
+    };
+    expect(
+      studyFindings(DIR, {
+        files: probe,
+        listDir: memoryListDir(files),
+        codec: studyRegistryCodec,
+      }).map((f) => f.problem),
+    ).toEqual(["result-file"]);
+  });
+});