paperlint 7.2.0 → 9.0.0

package/src/sources-rules.ts

@@ -0,0 +1,117 @@
+/**
+ * `paper/sources-fresh` — the one finding for a paper whose build record cannot be used
+ * (docs/design/paper-sources.md §1, docs/rules/paper/sources-fresh.md). Every rule that needs to know
+ * which files make up the paper, or which bibliography TeX reads, reads `_build/sources.json`
+ * (`paperRecord`, `src/paper-record.ts`); when there is none, or the paper has changed since, they
+ * are silent and this rule speaks once instead:
+ *
+ *   no record                      "not built" — run `npx paperlint build`
+ *   a record this paperlint cannot read   the reason, and the same command
+ *   a file changed since the build  which files, and the same command
+ *
+ * A warning, not an error, for the reason `pdf/measured` is one: lint often runs where nothing is
+ * built (the CI action only lints). Each message says what stays unchecked until a build: the files
+ * `paper.tex` includes are not linted, and the rules over the bibliography say nothing.
+ */
+import { basename, dirname } from "node:path";
+import { callerPath } from "./caller-path.ts";
+import { describeChanges } from "./domain/sources-record.ts";
+import { MAIN_FILE } from "./domain/paper-sources.ts";
+import { paperRecord, type RecordReadDeps } from "./paper-record.ts";
+
+/** One finding: its message, and the values the message names. */
+interface Finding {
+  readonly messageId: string;
+  readonly data?: Readonly<Record<string, string>>;
+}
+
+interface Loc {
+  readonly line: number;
+  readonly column: number;
+}
+interface RuleContext {
+  readonly filename: string;
+  readonly sourceCode: { getLocFromIndex(i: number): Loc };
+  report(
+    d: Finding & { readonly loc: Readonly<{ start: Loc; end: Loc }> },
+  ): void;
+}
+export interface SourcesRuleModule {
+  readonly meta: {
+    readonly type: "problem" | "suggestion";
+    readonly docs: {
+      readonly description: string;
+      readonly url: string;
+    };
+    readonly schema: readonly object[];
+    readonly messages: Readonly<Record<string, string>>;
+  };
+  create(context: RuleContext): { "root:exit"?: () => void };
+}
+
+const BUILD = "`npx paperlint build`";
+const UNTIL =
+  "until then the files `paper.tex` includes are not linted, and the rules that read the bibliography say nothing";
+
+const META: SourcesRuleModule["meta"] = {
+  type: "problem",
+  docs: {
+    description:
+      "the build recorded which files TeX read, and the paper has not changed since (_build/sources.json)",
+    url: "https://github.com/zernie/paperlint/blob/main/docs/rules/paper/sources-fresh.md",
+  },
+  schema: [],
+  messages: {
+    notBuilt: `the paper has not been built — run ${BUILD}, which records the files TeX reads. Until it has, the files \`paper.tex\` includes are not linted, and the rules that read the bibliography say nothing`,
+    unreadable: `the last build's record cannot be used ({{why}}) — run ${BUILD}; ${UNTIL}`,
+    stale: `the paper changed since the last build ({{changed}}) — run ${BUILD}; ${UNTIL}`,
+  },
+};
+
+/** The finding for a paper's record, or none when it is current. */
+function findingFor(dir: string, deps: RecordReadDeps): Finding | null {
+  const r = paperRecord(dir, deps);
+  switch (r.kind) {
+    case "none":
+      return r.unreadable === null
+        ? { messageId: "notBuilt" }
+        : { messageId: "unreadable", data: { why: r.unreadable } };
+    case "stale":
+      return {
+        messageId: "stale",
+        data: { changed: describeChanges(r.changed) },
+      };
+    case "fresh":
+      return null;
+  }
+}
+
+/** `paper/sources-fresh`, reading the paper through `deps`. It acts on `paper.tex` only. */
+export function sourcesRules(
+  deps: RecordReadDeps,
+): Readonly<Record<"sources-fresh", SourcesRuleModule>> {
+  return {
+    "sources-fresh": {
+      meta: META,
+      create: (context) =>
+        basename(context.filename) !== MAIN_FILE
+          ? {}
+          : {
+              "root:exit": () => {
+                const f = findingFor(
+                  dirname(callerPath(context.filename)),
+                  deps,
+                );
+                if (f === null) return;
+                const at = context.sourceCode.getLocFromIndex(0);
+                context.report({ loc: { start: at, end: at }, ...f });
+              },
+            },
+    },
+  };
+}
+
+/** The level it is on at for every `paper.tex` in paperlint's own config. */
+export const SOURCES_RULE_LEVELS = {
+  "paper/sources-fresh": "warn",
+} as const;

package/src/structure.harness.ts

@@ -47,7 +47,6 @@ const paper = (name: string, files: readonly string[]) => {
 
 try {
   paper("complete", ["PIPELINE-STATUS.md", "paper.tex", "refs.bib"]);
-  paper("complete-md", ["PIPELINE-STATUS.md", "paper.md"]);
   // the marker is present (paper.tex), the scorecard is not — the directory is linted by ZERO
   // rules and reports clean
   paper("no-scorecard", ["paper.tex", "refs.bib"]);
@@ -65,10 +64,6 @@ try {
     "a complete paper directory — NOT A SINGLE finding",
     at("complete").length === 0,
   );
-  check(
-    "and `paper.md` counts on equal footing with `paper.tex` — the corpus holds both forms",
-    at("complete-md").length === 0,
-  );
   check(
     "a missing scorecard — a finding",
     at("no-scorecard").length === 1 &&
@@ -86,10 +81,11 @@ try {
     /no-scorecard/.test(at("no-scorecard")[0]?.message ?? ""),
   );
   check(
-    "a directory with no source — a finding, and BOTH accepted forms are listed",
+    "a directory with no source — a finding naming `paper.tex`",
     at("no-source").length === 1 &&
-      /`paper\.tex`/.test(at("no-source")[0]?.message ?? "") &&
-      /`paper\.md`/.test(at("no-source")[0]?.message ?? ""),
+      /`paper\.tex` — a paper directory with no source/.test(
+        at("no-source")[0]?.message ?? "",
+      ),
   );
 
   // 🔴 THE PAIRED HALF: detection is GENEROUS. Without this the check would scream about every
@@ -124,9 +120,7 @@ try {
       {
         cwd: root,
       },
-    ).some(
-      (x) => x.file.endsWith("complete-md") && /refs\.bib/.test(x.message),
-    ),
+    ).some((x) => x.file.endsWith("no-source") && /refs\.bib/.test(x.message)),
   );
   check(
     "a nonexistent root does not crash it — that's the empty-set guard talking",

package/src/structure.ts

@@ -26,6 +26,7 @@ import { readdirSync, existsSync } from "node:fs";
 import { join, relative, basename, isAbsolute, sep } from "node:path";
 import type { StructureConfig, StructureFinding } from "./types.ts";
 import { CONFIG_FILE } from "#lib/paper-config";
+import { SOURCE_FILE } from "./new-paper.ts";
 
 /** The requirements after the consumer's config is laid over the defaults. */
 type Rules = Required<StructureConfig>;
@@ -38,9 +39,9 @@ type Rules = Required<StructureConfig>;
  * byte-compared by the `paper/stages` rule anyway.
  */
 export const STRUCTURE_DEFAULTS = {
-  markers: ["PIPELINE-STATUS.md", "paper.tex", "paper.md", CONFIG_FILE],
+  markers: ["PIPELINE-STATUS.md", SOURCE_FILE, CONFIG_FILE],
   require: ["PIPELINE-STATUS.md"],
-  requireOneOf: [["paper.tex", "paper.md"]],
+  requireOneOf: [[SOURCE_FILE]],
   ignore: [],
 };
 
@@ -118,7 +119,7 @@ export function checkStructure(
  * The message NAMES THE CONSEQUENCE rather than restating the condition. "missing
  * PIPELINE-STATUS.md" without the second half reads as nitpicking about formatting; with it you
  * can see which checks the directory silently loses — the rules that read the scorecard; the rules
- * over `paper.md` / `paper.tex` still run.
+ * over `paper.tex` still run.
  */
 function whyMissingMatters(file: string, dirName: string): string {
   if (file === "PIPELINE-STATUS.md")

package/src/tex-paper.test.ts

@@ -1,106 +1,162 @@
 /**
- * The reading every parse-tree rule shares (`readPaper`), the files of the paper's body that lint
- * reads on their own (`bodyFiles`), and where a finding of the assembled text is reported
- * (`reportInPaper`).
+ * The reading every parse-tree rule shares (`readPaper`): the paper's text with the files TeX read
+ * spliced in — the files the build's record lists, and no others — and where a finding of that text
+ * is reported (`reportInPaper`). The record fed to it is TeX's own answer for a planted paper
+ * (`fixtures/paper-sources/<name>/tex-truth.json`).
  */
 import { describe, expect, it } from "vitest";
 import { latexReader } from "./adapters/latex/index.ts";
 import { memoryFiles } from "./adapters/memory/index.ts";
+import { sourcesCodec } from "./adapters/sources-record/index.ts";
 import { join } from "node:path";
 import { texInputsDir } from "./package-dirs.ts";
-import { bodyFiles, readPaper, reportInPaper } from "./tex-paper.ts";
-
-describe("readPaper — the paper as the rules read it", () => {
-  it("reads includes from the paper's own directory", () => {
-    const p = readPaper("/p/paper.tex", "x\\input{s/a}y", {
-      files: memoryFiles({ "/p/s/a.tex": "AAA" }),
-      latex: latexReader,
-    });
-    expect(p.text).toBe("xAAAy");
+import { readPaper, reportInPaper } from "./tex-paper.ts";
+import {
+  builtFixture,
+  builtPaper,
+  fixtureFiles,
+} from "../test/recorded-fixture.ts";
+
+/** What `readPaper` reads with: the files held in memory, the LaTeX reader, the record's schema. */
+const proseDeps = (files: Record<string, string>) => ({
+  files: memoryFiles(files),
+  latex: latexReader,
+  codec: sourcesCodec,
+});
+
+const MAIN = "/p/paper.tex";
+const textOf = (files: Record<string, string>): string => files[MAIN] ?? "";
+
+describe("readPaper — the paper as the rules read it, from the files TeX read", () => {
+  it("🔴 splices a file the record lists where its include stands, and TeX's end of line after it", () => {
+    // p1: TeX read sections/intro.tex, and the file's last line ends with a newline.
+    const files = builtFixture("p1", "/p");
+    const p = readPaper(MAIN, textOf(files), proseDeps(files));
+    const main = textOf(files);
+    const intro = files["/p/sections/intro.tex"] ?? "";
+    expect(p.text).toBe(main.replace("\\input{sections/intro}", intro));
     expect(p.main).toBe("paper.tex");
   });
 
-  it("an include the paper's directory lacks is found in paperlint's own inputs, as the build finds it", () => {
-    const p = readPaper("/p/paper.tex", "x\\input{guards}y", {
-      files: memoryFiles({ [join(texInputsDir(), "guards.tex")]: "G" }),
-      latex: latexReader,
-    });
-    expect({ text: p.text, missing: p.missing }).toEqual({
-      text: "xGy",
-      missing: [],
-    });
+  it("an include behind a comment or `\\iffalse` is not spliced: the record does not list it", () => {
+    const files = builtFixture("p1", "/p");
+    const { text } = readPaper(MAIN, textOf(files), proseDeps(files));
+    expect(text).not.toContain("Dead section");
+    expect(text).not.toContain("Parked section");
+  });
+
+  it("🔴 a file the text names after \\end{document} is not spliced: TeX never read it", () => {
+    // v23-parked-include: `\input{parked}` stands past the end of the document; parked.tex is on
+    // disk, and the record lists paper.tex alone.
+    const files = builtFixture("v23-parked-include", "/p");
+    expect(files["/p/parked.tex"]).toBeDefined();
+    const p = readPaper(MAIN, textOf(files), proseDeps(files));
+    expect(p.text).toBe(textOf(files));
+    expect(p.text).not.toContain("\\bibliography{old}");
+  });
+
+  it("🔴 a .tex a filecontents block wrote is spliced where the body \\input's it: TeX typeset its text", () => {
+    // v29-generated-input: the record lists body.tex, which the block in paper.tex wrote.
+    const files = builtFixture("v29-generated-input", "/p");
+    const p = readPaper(MAIN, textOf(files), proseDeps(files));
+    expect(p.text).toBe(
+      textOf(files).replace("\\input{body}", files["/p/body.tex"] ?? ""),
+    );
+    expect([...new Set(p.segments.map((s) => s.file))]).toEqual([
+      "paper.tex",
+      "body.tex",
+    ]);
   });
 
-  it("the paper's directory comes first: its file wins over paperlint's of the same name", () => {
-    const p = readPaper("/p/paper.tex", "\\input{guards}", {
-      files: memoryFiles({
-        "/p/guards.tex": "mine",
-        [join(texInputsDir(), "guards.tex")]: "package",
-      }),
-      latex: latexReader,
-    });
-    expect(p.text).toBe("mine");
+  it("🔴 with no record, the paper is paper.tex alone: its includes are not read", () => {
+    const files = fixtureFiles("p1", "/p");
+    const p = readPaper(MAIN, textOf(files), proseDeps(files));
+    expect(p.text).toBe(textOf(files));
+    expect(p.segments).toHaveLength(1);
+  });
+
+  it("🔴 with a stale record, the paper is paper.tex alone", () => {
+    const files = {
+      ...builtFixture("p1", "/p"),
+      "/p/sections/intro.tex": "\\section{Changed}\n",
+    };
+    const p = readPaper(MAIN, textOf(files), proseDeps(files));
+    expect(p.text).toBe(textOf(files));
   });
 });
 
-describe("bodyFiles — the files of the body lint reads on their own", () => {
-  const doc = (preamble: string, body: string): string =>
-    `\\documentclass{article}\n${preamble}\n\\begin{document}\n${body}\n\\end{document}\n`;
-  const files = (extra: Record<string, string> = {}) =>
-    memoryFiles({
-      "/p/macros.tex": "\\newcommand{\\x}{y}",
-      "/p/sections/a.tex": "A\n\\input{sections/b}\n",
-      "/p/sections/b.tex": "B",
-      [join(texInputsDir(), "guards.tex")]: "G",
-      ...extra,
-    });
-
-  it("every file an include in the document body brings in, nested ones too, by absolute path", () => {
-    expect(
-      bodyFiles("/p/paper.tex", doc("", "\\input{sections/a}"), {
-        files: files(),
-        latex: latexReader,
-      }),
-    ).toEqual({
-      files: ["/p/sections/a.tex", "/p/sections/b.tex"],
-      missing: [],
-    });
+describe("readPaper — which files, in which order", () => {
+  it("a file the record lists that is not on disk (it was absent when recorded and still is) contributes nothing", () => {
+    const files = builtPaper("/p", { "paper.tex": "x\\input{gone}y" }, [
+      { path: "paper.tex", role: "body" },
+      { path: "gone.tex", role: "body" },
+    ]);
+    expect(readPaper(MAIN, textOf(files), proseDeps(files)).text).toBe("xy");
   });
 
-  it("not a preamble include (macros), and not a file found only in paperlint's inputs", () => {
-    expect(
-      bodyFiles(
-        "/p/paper.tex",
-        doc("\\input{macros}", "\\input{guards}\nText."),
-        { files: files(), latex: latexReader },
-      ),
-    ).toEqual({ files: [], missing: [] });
+  it("files are found in the paper's directory only: paperlint's inputs are not the paper's", () => {
+    const files = {
+      ...builtPaper("/p", { "paper.tex": "x\\input{guards}y" }, [
+        { path: "paper.tex", role: "body" },
+      ]),
+      [join(texInputsDir(), "guards.tex")]: "G",
+    };
+    const p = readPaper(MAIN, textOf(files), proseDeps(files));
+    expect(p.text).toBe("xy");
   });
 
-  it("an include that resolves nowhere is named, with the file that wrote it", () => {
-    expect(
-      bodyFiles("/p/paper.tex", doc("", "\\input{gone}"), {
-        files: files(),
-        latex: latexReader,
-      }),
-    ).toEqual({ files: [], missing: [{ file: "paper.tex", target: "gone" }] });
+  it("nested includes are spliced in the order TeX read them, each once", () => {
+    const files = builtPaper(
+      "/p",
+      {
+        "paper.tex": "A\n\\input{sections/a}\nZ\n",
+        "sections/a.tex": "a1\n\\input{sections/b}\na2\n",
+        "sections/b.tex": "b1",
+      },
+      [
+        { path: "paper.tex", role: "body" },
+        { path: "sections/a.tex", role: "body" },
+        { path: "sections/b.tex", role: "body" },
+      ],
+    );
+    const p = readPaper(MAIN, textOf(files), proseDeps(files));
+    expect(p.text).toBe("A\na1\nb1\n\na2\n\nZ\n");
+    expect([...new Set(p.segments.map((s) => s.file))]).toEqual([
+      "paper.tex",
+      "sections/a.tex",
+      "sections/b.tex",
+    ]);
   });
 
-  it("a main file with no document environment: every include is the body", () => {
-    expect(
-      bodyFiles("/p/paper.tex", "\\input{sections/b}", {
-        files: files(),
-        latex: latexReader,
-      }),
-    ).toEqual({ files: ["/p/sections/b.tex"], missing: [] });
+  it("a preamble file the record lists is spliced too: it is a file of the paper", () => {
+    const files = builtPaper(
+      "/p",
+      {
+        "paper.tex": "\\input{macros}\n\\begin{document}\nText.\n",
+        "macros.tex": "\\newcommand{\\x}{y}\n",
+      },
+      [
+        { path: "paper.tex", role: "body" },
+        { path: "macros.tex", role: "preamble" },
+      ],
+    );
+    const p = readPaper(MAIN, textOf(files), proseDeps(files));
+    expect(p.text).toBe("\\newcommand{\\x}{y}\n\n\\begin{document}\nText.\n");
   });
 });
 
 describe("reportInPaper — where a finding of the assembled text is reported", () => {
-  const paper = readPaper("/p/paper.tex", "x\n\\input{a}\ny", {
-    files: memoryFiles({ "/p/a.tex": "one\ntwo" }),
-    latex: latexReader,
-  });
+  const main = "x\n\\input{a}\ny";
+  const paper = readPaper(
+    "/p/paper.tex",
+    main,
+    proseDeps(
+      builtPaper("/p", { "paper.tex": main, "a.tex": "one\ntwo" }, [
+        { path: "paper.tex", role: "body" },
+        { path: "a.tex", role: "body" },
+      ]),
+    ),
+  );
   const report = (
     at: { start: number; end: number } | null,
     messageId = "m",
@@ -146,3 +202,17 @@ describe("reportInPaper — where a finding of the assembled text is reported",
     ]);
   });
 });
+
+describe("readPaper — how an include names its file", () => {
+  it("🔴 an include spelled otherwise than the disk is the file the record names: TeX opened it by either spelling", () => {
+    const files = builtPaper(
+      "/p",
+      { "paper.tex": "x\\input{Sections/Intro}y", "sections/intro.tex": "I" },
+      [
+        { path: "paper.tex", role: "body" },
+        { path: "sections/intro.tex", role: "body" },
+      ],
+    );
+    expect(readPaper(MAIN, textOf(files), proseDeps(files)).text).toBe("xI\ny");
+  });
+});

package/src/tex-paper.ts

@@ -7,9 +7,13 @@
  * file at the include in `paper.tex` that brought it in — the one place ESLint can point to in this
  * file — with the included file, line and column at the front of the message.
  *
- * An include is looked for where the build tells TeX to look (`texSearchPath`): the paper's own
- * directory, then paperlint's inputs. `bodyFiles` names the files of the paper's BODY the author
- * wrote — what the rules over ESLint's own LaTeX text read one by one, each at its own path.
+ * 🔴 WHICH FILES ARE THE PAPER'S IS TEX'S ANSWER, NOT THIS MODULE'S (docs/design/paper-sources.md §1).
+ * `readPaper` splices only the files the build's record lists (`_build/sources.json`, `paperRecord`):
+ * the files of the paper directory TeX read. An include of a file the record does not list — behind
+ * `\iffalse`, never reached — contributes nothing, as it did to TeX. With no
+ * record, or a stale one, the paper is `paper.tex` alone, and `paper/sources-fresh` says the files it
+ * includes went unlinted. Where an include stands in the text is the only thing read from the text:
+ * whether the file it names is part of the paper is the record's to say, never the disk's.
  */
 import { basename, dirname, join } from "node:path";
 import {
@@ -21,8 +25,8 @@ import type { Span } from "./domain/tex-document.ts";
 import type { LatexReader } from "./ports/latex.ts";
 import type { Files } from "./ports/files.ts";
 import { callerPath } from "./caller-path.ts";
-import type { AbsolutePath } from "./domain/paths.ts";
-import { texSearchPath } from "./package-dirs.ts";
+import { paperRecord, type RecordReadDeps } from "./paper-record.ts";
+import { spelledIn } from "./domain/sources-record.ts";
 import type { Located, TexRuleContext } from "./tex-venue-rules.ts";
 
 /** What reading a paper needs: the disk, and the LaTeX reader. */
@@ -31,75 +35,41 @@ export interface PaperDeps {
   readonly latex: LatexReader;
 }
 
-/** The paper whose main file is `filename` with the text `src`, its includes spliced. */
+/** What `readPaper` needs: the disk, the LaTeX reader (where an include stands), and the record's schema. */
+export interface ProseDeps extends PaperDeps, RecordReadDeps {}
+
+/** The text of a file read from the disk, or null when there was none. */
+export const decoded = (b: Uint8Array | null): string | null =>
+  b === null ? null : new TextDecoder().decode(b);
+
+/**
+ * The paper whose main file is `filename` with the text `src`: the files the last build's record lists
+ * spliced where their includes stand. No record, or one the paper has outgrown, is `src` alone.
+ */
 export function readPaper(
   filename: string,
   src: string,
-  deps: PaperDeps,
+  deps: ProseDeps,
 ): PaperSource {
   const dir = dirname(filename);
+  const record = paperRecord(dir, deps);
+  const listed =
+    record.kind === "fresh" ? record.record.inputs.map((i) => i.path) : [];
   return assemblePaper(basename(filename), src, {
-    includes: deps.latex.includes,
+    // No record: no include is looked at, so there is nothing to splice.
+    includes: record.kind === "fresh" ? deps.latex.includes : () => [],
     documentBody: deps.latex.documentBody,
     read: (rel) => {
-      const at = located(dir, rel, deps.files);
-      const b = at === null ? null : deps.files.readBytes(at);
-      return b === null ? null : new TextDecoder().decode(b);
+      const recorded = spelledIn(listed, rel);
+      return recorded === null
+        ? null
+        : decoded(deps.files.readBytes(callerPath(join(dir, recorded))));
     },
   });
 }
 
-/** `rel` in the first directory of the paper's TeX search path that holds it, or null. */
-const located = (dir: string, rel: string, files: Files): AbsolutePath | null =>
-  texSearchPath(dir)
-    .map((d) => callerPath(join(d, rel)))
-    .find((p) => files.isFile(p)) ?? null;
-
-/** An include that resolved nowhere: the file that wrote it, and the path as written. */
-export interface Unread {
-  readonly file: string;
-  readonly target: string;
-}
-
-/**
- * The files of the body of the paper whose main file is `filename`: every file an include inside its
- * `document` environment brings in, nested ones too, found in the paper's own directory. Not a
- * preamble include (macros are not the body), and not a file found only in paperlint's inputs (not
- * the author's text). `missing` is every include, anywhere, that resolved nowhere.
- */
-export function bodyFiles(
-  filename: string,
-  src: string,
-  deps: PaperDeps,
-): {
-  readonly files: readonly AbsolutePath[];
-  readonly missing: readonly Unread[];
-} {
-  const dir = dirname(filename);
-  const paper = readPaper(filename, src, deps);
-  const body = deps.latex.documentBody(src);
-  const inBody = (via: Span | null): boolean =>
-    via !== null &&
-    (body === null || (via.start >= body.start && via.end <= body.end));
-  const own = (rel: string): AbsolutePath | null => {
-    const at = located(dir, rel, deps.files);
-    return at === callerPath(join(dir, rel)) ? at : null;
-  };
-  return {
-    files: [
-      ...new Set(
-        paper.segments
-          .filter((s) => s.file !== paper.main && inBody(s.via))
-          .map((s) => own(s.file))
-          .filter((f): f is AbsolutePath => f !== null),
-      ),
-    ],
-    missing: paper.missing.map(({ file, target }) => ({ file, target })),
-  };
-}
-
 /** Line and column (1-based) of an offset in a text. */
-function lineColumn(text: string, at: number): string {
+export function lineColumn(text: string, at: number): string {
   const line = text.slice(0, at).split("\n").length;
   const column = at - text.lastIndexOf("\n", at - 1);
   return `${String(line)}:${String(column)}`;