paperlint 4.1.0 → 5.0.0

package/src/paper-includes.test.ts

@@ -0,0 +1,161 @@
+/**
+ * `paperlint lint` over a paper whose body is in other files: the rules over ESLint's LaTeX text read
+ * every file `paper.tex` includes from its document body, report at that file's own path and line,
+ * and fix that file. What a rule decides for the whole paper — the majority reference form and
+ * whether the build is in review mode — it decides over all of the paper's files, not over the one
+ * it is reporting in.
+ */
+import {
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join, relative } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { run } from "./cli.ts";
+import { lintReport } from "../test/lint-report.ts";
+
+const dirs: string[] = [];
+afterEach(() => {
+  for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true });
+});
+
+const doc = (cls: string, preamble: string, body: string): string =>
+  `\\documentclass${cls}\n${preamble}\n\\begin{document}\n${body}\n\\end{document}\n`;
+
+const INTRO = "See §3 and p < .05.\nAs Fig.~\\ref{f3} shows.\n";
+
+function project(files: Record<string, string>): string {
+  const root = realpathSync(mkdtempSync(join(tmpdir(), "paperlint-inc-")));
+  dirs.push(root);
+  const all: Record<string, string> = {
+    "package.json": JSON.stringify({ name: "c", private: true }),
+    "papers/a/PIPELINE-STATUS.md": "---\nstages: []\n---\n",
+    "papers/a/paper.tex": doc(
+      "{article}",
+      "\\input{macros}",
+      "Figure~\\ref{f1} and Figure~\\ref{f2}.\n\\input{sections/intro}",
+    ),
+    // Prose in a preamble include is not the body: were it linted, `§2` would be reported.
+    "papers/a/macros.tex": "% See §2.\n\\newcommand{\\half}{0.5}\nSee §2.\n",
+    "papers/a/sections/intro.tex": INTRO,
+    // A frozen version beside the paper is not the paper.
+    "papers/a/versions/2026-01-01-submitted.tex": doc(
+      "{article}",
+      "",
+      "See §9.",
+    ),
+    ...files,
+  };
+  for (const [p, text] of Object.entries(all)) {
+    mkdirSync(join(root, p, ".."), { recursive: true });
+    writeFileSync(join(root, p), text);
+  }
+  return root;
+}
+
+/** `paperlint lint --json` → rule ids per linted file (relative to the project), and stderr. */
+async function lint(
+  root: string,
+  args: string[] = [],
+): Promise<{ byFile: Record<string, string[]>; err: string }> {
+  const out: string[] = [];
+  const err: string[] = [];
+  await run(["lint", "--json", ...args], {
+    cwd: root,
+    log: (s: string) => out.push(s),
+    err: (s: string) => err.push(s),
+  });
+  const byFile = Object.fromEntries(
+    lintReport(out.join("\n")).map((r) => [
+      relative(root, r.filePath),
+      r.messages.map((m) => m.ruleId ?? "(fatal)").sort(),
+    ]),
+  );
+  return { byFile, err: err.join("\n") };
+}
+
+describe("lint reads the files paper.tex includes from its body", () => {
+  it("🔴 findings in an included file land at that file; the majority form is the paper's", async () => {
+    const { byFile } = await lint(project({}));
+    expect(byFile["papers/a/sections/intro.tex"]).toEqual([
+      "paper/figure-ref-style",
+      "paper/leading-zero",
+      "paper/section-word",
+    ]);
+    // Two `Figure` in paper.tex against one `Fig.` in the section: the section holds the minority.
+    expect(byFile["papers/a/paper.tex"]).not.toContain(
+      "paper/figure-ref-style",
+    );
+  });
+
+  it("a preamble include and a frozen version beside the paper are not linted", async () => {
+    const { byFile } = await lint(project({}));
+    expect(Object.keys(byFile).sort()).toEqual([
+      "papers/a/PIPELINE-STATUS.md",
+      "papers/a/paper.tex",
+      "papers/a/sections/intro.tex",
+    ]);
+  });
+
+  it("--fix rewrites the included file, to the paper's majority form", async () => {
+    const root = project({});
+    const before = readFileSync(join(root, "papers/a/paper.tex"), "utf8");
+    const { byFile } = await lint(root, ["--fix"]);
+    expect(byFile["papers/a/sections/intro.tex"]).toEqual([]);
+    expect(
+      readFileSync(join(root, "papers/a/sections/intro.tex"), "utf8"),
+    ).toBe("See Section 3 and p < 0.05.\nAs Figure~\\ref{f3} shows.\n");
+    expect(readFileSync(join(root, "papers/a/paper.tex"), "utf8")).toBe(before);
+  });
+});
+
+describe("what lint decides for the whole paper, it decides over every file of it", () => {
+  it("tex/future-promise in an included file follows the build mode paper.tex declares", async () => {
+    const promise = {
+      "papers/a/sections/intro.tex":
+        "The harness will be released at camera-ready.\n",
+    };
+    const final = await lint(project(promise));
+    expect(final.byFile["papers/a/sections/intro.tex"]).toEqual([
+      "tex/future-promise",
+    ]);
+    const review = await lint(
+      project({
+        ...promise,
+        "papers/a/paper.tex": doc(
+          "[review]{article}",
+          "",
+          "Text.\n\\input{sections/intro}",
+        ),
+      }),
+    );
+    expect(review.byFile["papers/a/sections/intro.tex"]).toEqual([]);
+  });
+
+  it("an include that resolves nowhere is named as a file lint did not read", async () => {
+    const { err } = await lint(
+      project({
+        "papers/a/paper.tex": doc("{article}", "", "\\input{sections/gone}"),
+      }),
+    );
+    expect(err).toMatch(
+      /papers\/a\/paper\.tex: lint did not read `sections\/gone`/,
+    );
+  });
+
+  it("the note is about the papers this run lints, not another paper of the project", async () => {
+    const root = project({
+      "papers/b/PIPELINE-STATUS.md": "---\nstages: []\n---\n",
+      "papers/b/paper.tex": doc("{article}", "", "\\input{gone}"),
+    });
+    expect((await lint(root, ["papers/a"])).err).not.toMatch(/did not read/);
+    expect((await lint(root)).err).toMatch(
+      /papers\/b\/paper\.tex: lint did not read `gone`/,
+    );
+  });
+});

package/src/paper-includes.ts

@@ -0,0 +1,77 @@
+/**
+ * THE FILES OF A PAPER'S BODY, handed to ESLint — so the rules over ESLint's LaTeX text read a paper
+ * whose body is in `sections/*.tex`, and report each finding at its own file and line (#144).
+ *
+ * The set is what TeX reads, resolved by the reader the parse-tree rules use (`bodyFiles`,
+ * src/tex-paper.ts): `paper.tex` and every file an include in its document body brings in, found in
+ * the paper's directory. Not a glob over every `.tex`: a frozen version under `versions/`, a macro
+ * file the preamble includes and a scratch file nothing includes are not the paper.
+ *
+ * Two blocks per paper, both relative to the paper's directory (`basePath`):
+ *
+ * 1. a global un-ignore of each body file by name. paperlint's scope ignores every file it does not
+ *    claim (src/paper-files.ts); the fragment block (`buildConfig`) claims every `.tex` but
+ *    `paper.tex`, so only the files named here are ever enumerated;
+ * 2. `settings.paperlint.paper` for `paper.tex` and its body files: the main file and the whole set,
+ *    for the rules whose verdict is the paper's (eslint-rules/paper-context.ts).
+ */
+import { join, relative } from "node:path";
+import { PAPERLINT_SETTINGS } from "#eslint-rules/paper-context";
+import { callerPath } from "./caller-path.ts";
+import type { AbsolutePath } from "./domain/paths.ts";
+import type { ConfigBlock } from "./rules-config.ts";
+import { bodyFiles, type PaperDeps, type Unread } from "./tex-paper.ts";
+
+/** One paper's main file and body files, and the includes that resolved nowhere. */
+export interface PaperBody {
+  readonly dir: string;
+  readonly main: AbsolutePath;
+  readonly files: readonly AbsolutePath[];
+  readonly missing: readonly Unread[];
+}
+
+/** The body of each paper directory in `dirs` that has a `paper.tex`. */
+export function paperBodies(
+  dirs: readonly string[],
+  deps: PaperDeps,
+): readonly PaperBody[] {
+  return dirs.flatMap((dir) => {
+    const main = callerPath(join(dir, "paper.tex"));
+    const bytes = deps.files.readBytes(main);
+    if (bytes === null) return [];
+    const body = bodyFiles(main, new TextDecoder().decode(bytes), deps);
+    return [{ dir, main, files: body.files, missing: body.missing }];
+  });
+}
+
+/** The config blocks that hand ESLint each paper's body files, and the paper to every one of them. */
+export const includeBlocks = (
+  bodies: readonly PaperBody[],
+): readonly ConfigBlock[] =>
+  bodies.flatMap((b) => {
+    const rel = b.files.map((f) => relative(b.dir, f));
+    const paper = { main: b.main, files: [b.main, ...b.files] };
+    return [
+      ...(rel.length > 0
+        ? [{ basePath: b.dir, ignores: rel.map((r) => `!${r}`) }]
+        : []),
+      {
+        basePath: b.dir,
+        files: ["paper.tex", ...rel],
+        settings: { [PAPERLINT_SETTINGS]: { paper } },
+      },
+    ];
+  });
+
+/** One line per include that resolved nowhere: which text lint did not read, and why. */
+export const unreadLines = (
+  bodies: readonly PaperBody[],
+  shown: (path: string) => string,
+): readonly string[] =>
+  bodies.flatMap((b) =>
+    b.missing.map(
+      (m) =>
+        `${shown(b.main)}: lint did not read \`${m.target}\` (included by ${m.file}): ` +
+        "no such file in the paper's directory or paperlint's inputs",
+    ),
+  );

package/src/ports/latex.ts

@@ -2,10 +2,13 @@
  * What paperlint needs from a LaTeX source, shaped by its rules and `paperlint new` — not by the
  * parser. The one implementation is `src/adapters/latex/` (unified-latex); `src/cli.ts` wires it.
  */
+import type { Include } from "../domain/paper-source.ts";
 import type {
   ClassLine,
   DocumentClass,
   Outline,
+  Passage,
+  Span,
   TextRun,
 } from "../domain/tex-document.ts";
 
@@ -16,6 +19,16 @@ export interface LatexReader {
   readonly outline: (src: string) => Outline;
   /** The text a reader sees: prose, the author block, footnotes — not comments, keys, math, code. */
   readonly renderedRuns: (src: string) => readonly TextRun[];
+  /**
+   * The body's prose, as passages a sentence cannot cross: from the start of the document (the
+   * abstract included) to the appendix or the bibliography, without headings, captions, floats,
+   * code or the title block. Citations, links and cross-references stay in place as marks.
+   */
+  readonly bodyProse: (src: string) => readonly Passage[];
+  /** Every `\input`, `\include` and `\subfile`, in source order: the path as written, where it stands. */
+  readonly includes: (src: string) => readonly Include[];
+  /** The span of the `document` environment's body, or null when the source has none. */
+  readonly documentBody: (src: string) => Span | null;
   /**
    * A preset's `template` → the class it names: a whole `\documentclass[…]{…}` line, or a bare
    * class name (no option). Null for anything else.

package/src/register.test.ts

@@ -0,0 +1,165 @@
+/**
+ * `tex/register` on real parses: sentences of the body opening with And, So, But, Nor, Or or Yet per
+ * 1000 words, reported once for the whole body when the rate runs above what accepted papers show.
+ * Both halves, what counts as a word and as a conjunction start, the floor under which a rate means
+ * nothing, and the option.
+ */
+import { describe, expect, it } from "vitest";
+import { latexReader } from "./adapters/latex/index.ts";
+import { memoryFiles } from "./adapters/memory/index.ts";
+import {
+  judgeRegister,
+  measureRegister,
+  MIN_WORDS,
+  registerRule,
+} from "./register.ts";
+
+const doc = (body: string): string =>
+  `\\documentclass{article}\n\\begin{document}\n${body}\n\\end{document}\n`;
+
+/** Fourteen words. */
+const PLAIN =
+  "The rule reads the body of the paper and counts every sentence it holds.";
+/** Thirteen words, opening with a conjunction. */
+const CONJ = "But the rule reads the body of the paper and counts each one.";
+
+/** A body of `plain` plain sentences and `conj` conjunction starts, in paragraphs. */
+const body = (plain: number, conj: number): string =>
+  [
+    ...Array.from({ length: plain }, () => PLAIN),
+    ...Array.from({ length: conj }, () => CONJ),
+  ]
+    .map((s, i) => (i % 10 === 9 ? `${s}\n\n` : `${s} `))
+    .join("");
+
+const measured = (src: string) =>
+  measureRegister(latexReader.bodyProse(doc(src)));
+
+describe("measureRegister — what the body is counted as", () => {
+  it("its words, and its sentences opening with a conjunction", () => {
+    expect(measured(`${PLAIN} ${CONJ} Yet it holds.`)).toMatchObject({
+      words: 14 + 13 + 3,
+      conjunctionStarts: 2,
+    });
+  });
+
+  it("a citation, a reference and math each read as a word", () => {
+    expect(
+      measured(
+        "We show a sample in Table~\\ref{t}. It follows~\\cite{k} with $n$ runs.",
+      ),
+    ).toMatchObject({ words: 7 + 6 });
+  });
+
+  it("🔴 a conjunction opens a sentence only as its first word and capitalised: a list item's «and», «Android», «Oracle» do not", () => {
+    expect(
+      measured(
+        "And it holds. Android ships it. Oracle runs it. Some sentences, and more.\n\\begin{itemize}\n\\item and the rest.\n\\end{itemize}",
+      ),
+    ).toMatchObject({ conjunctionStarts: 1 });
+  });
+
+  it("a sentence with no word in it — a stray mark, a lone full stop — adds nothing", () => {
+    expect(measured("\\cite{k}. . It holds here.")).toMatchObject({
+      words: 3,
+    });
+    expect(measured(". !")).toMatchObject({ words: 0 });
+    expect(measureRegister([])).toMatchObject({ words: 0, at: null });
+  });
+
+  it("the examples are the first three, as the reader sees them", () => {
+    expect(
+      measured(`${CONJ} So it goes. Or not. And again. Yet more.`).examples,
+    ).toEqual([
+      "But the rule reads the body of the paper and counts each one.",
+      "So it goes.",
+      "Or not.",
+    ]);
+  });
+});
+
+describe("judgeRegister — one finding when the rate runs above the limit", () => {
+  const judged = (src: string, limit?: number) =>
+    judgeRegister(measured(src), limit).map((f) => f.messageId);
+
+  it("a body like an accepted paper's is silent", () => {
+    // 200 × 14 + 13 = 2 813 words, 1 start: 0.36 per 1000.
+    expect(judged(body(200, 1))).toEqual([]);
+  });
+
+  it("🔴 conjunction starts above 0.8 per 1000 words are reported, once", () => {
+    // 200 × 14 + 3 × 13 = 2 839 words, 3 starts: 1.06 per 1000.
+    expect(judged(body(200, 3))).toEqual(["conjunctionStarts"]);
+  });
+
+  it("at the limit is not above it", () => {
+    expect(judged(body(200, 3), 1000 * (3 / (200 * 14 + 3 * 13)))).toEqual([]);
+  });
+
+  it("🔴 under the floor of words a rate means nothing, and nothing is reported", () => {
+    const tiny = "It holds here. But it holds.";
+    expect(measured(tiny).words).toBeLessThan(MIN_WORDS);
+    expect(judged(tiny)).toEqual([]);
+  });
+
+  it("the limit is an option: raised, the same body is silent; lowered, it reports", () => {
+    expect(judged(body(200, 3), 2)).toEqual([]);
+    expect(judged(body(200, 1), 0.2)).toEqual(["conjunctionStarts"]);
+  });
+
+  it("the finding carries the rate, the count, the limit and the examples", () => {
+    const [f] = judgeRegister(measured(body(200, 3)));
+    expect(f?.data).toMatchObject({
+      count: 3,
+      rate: "1.06",
+      limit: 0.8,
+      examples: `«${CONJ}», «${CONJ}», «${CONJ}»`,
+    });
+  });
+});
+
+describe("the rule — reported at the start of the body, the option read from the config", () => {
+  interface Seen {
+    readonly line: number;
+    readonly messageId: string;
+  }
+  const run = (src: string, options?: readonly unknown[]): Seen[] => {
+    const seen: Seen[] = [];
+    registerRule({ files: memoryFiles({}), latex: latexReader })
+      .create({
+        filename: "/p/paper.tex",
+        ...(options === undefined ? {} : { options }),
+        sourceCode: {
+          text: src,
+          getLocFromIndex: (i) => {
+            const line = src.slice(0, i).split("\n").length;
+            return { line, column: i - src.lastIndexOf("\n", i - 1) };
+          },
+        },
+        report: (d) => {
+          seen.push({
+            line: d.loc.start.line,
+            messageId: "messageId" in d ? d.messageId : d.message,
+          });
+        },
+      })
+      .root?.();
+    return seen;
+  };
+  const FOUND = { messageId: "conjunctionStarts" };
+
+  it("🔴 one finding, where the body's first sentence starts", () => {
+    expect(run(doc(body(200, 4)))).toEqual([{ line: 3, ...FOUND }]);
+  });
+
+  it("the option in the config changes the limit; an option object without it keeps the default", () => {
+    expect(run(doc(body(200, 4)), [{ conjunctionStartsPer1000: 2 }])).toEqual(
+      [],
+    );
+    expect(run(doc(body(200, 4)), [{}])).toEqual([{ line: 3, ...FOUND }]);
+  });
+
+  it("a body read without a document environment is reported at the top", () => {
+    expect(run(body(200, 4))).toEqual([{ line: 1, ...FOUND }]);
+  });
+});

package/src/register.ts

@@ -0,0 +1,177 @@
+/**
+ * `tex/register` — the body reads at the register of a paper, not of a post: sentences that open
+ * with a coordinating conjunction stay as rare as they are in accepted papers.
+ *
+ *   warn  more than 0.8 sentences per 1000 words of the body open with And, So, But, Nor, Or or Yet
+ *
+ * ── WHY ──────────────────────────────────────────────────────────────────────────
+ * Reviewers called our prose "too informal" and "a blogpost" twice. Measured against eleven accepted
+ * papers, sentence-initial conjunctions set our drafts apart (accepted 0–0.58 per 1000 words). The
+ * limit is an option; its default and the measurement behind it are on the rule's page.
+ *
+ * ── WHAT IT READS ────────────────────────────────────────────────────────────────
+ * The body's prose through the `LatexReader` port (`bodyProse`) — the parse tree of the whole paper,
+ * includes spliced — cut into sentences by the same splitter `tex/claim-provenance` uses
+ * (src/domain/sentences.ts). A citation, a cross-reference or math reads as one word, as it does on
+ * the page. A body under 2 000 words is not judged: there one sentence decides the rate.
+ */
+import { runText, type Passage, type Span } from "./domain/tex-document.ts";
+import { runOf, sentences } from "./domain/sentences.ts";
+import { readPaper, reportInPaper, type PaperDeps } from "./tex-paper.ts";
+import {
+  ruleDocsUrl,
+  type Located,
+  type TexRuleModule,
+} from "./tex-venue-rules.ts";
+
+/** Under this many words in the body, the rate is not judged. */
+export const MIN_WORDS = 2000;
+
+/** The limit when the config sets none, from the measurement on the rule's page. */
+export const DEFAULT_LIMIT = 0.8;
+
+/** The words that, opening a sentence, make it a conjunction start. Capitalised: only a first word. */
+const CONJUNCTIONS: ReadonlySet<string> = new Set([
+  "And",
+  "So",
+  "But",
+  "Nor",
+  "Or",
+  "Yet",
+]);
+
+/** The characters a mark or math stands as in a sentence's text (src/domain/sentences.ts). */
+const MARK = /[-]/gu;
+
+const WORDS = new Intl.Segmenter("en", { granularity: "word" });
+
+/** A sentence as the rate sees it: its text, its words, and its first word. */
+interface Sentence {
+  readonly text: string;
+  readonly words: number;
+  readonly first: string;
+}
+
+/** A sentence's words: the word-like ones, and one for each mark or math. Null when it has none of its own. */
+function sentenceOf(text: string): Sentence | null {
+  const words = [...WORDS.segment(text)].filter((w) => w.isWordLike);
+  const [first] = words;
+  if (first === undefined) return null;
+  return {
+    text: text.replace(MARK, "").replace(/\s+/gu, " ").trim(),
+    words: words.length + (text.match(MARK)?.length ?? 0),
+    first: first.segment,
+  };
+}
+
+/** What the body measures. */
+export interface RegisterMeasure {
+  readonly words: number;
+  readonly conjunctionStarts: number;
+  /** The first three, as the reader sees them. */
+  readonly examples: readonly string[];
+  /** Where the body's first passage starts, for a finding about the whole body. */
+  readonly at: Span | null;
+}
+
+/** Where a passage's first piece stands. */
+function startOf(p: Passage): Span {
+  const first = p.pieces[0];
+  return first.kind === "text"
+    ? {
+        start: first.segment.at,
+        end: first.segment.at + first.segment.text.length,
+      }
+    : first.span;
+}
+
+/** The body's words, and its sentences that open with a conjunction. */
+export function measureRegister(passages: readonly Passage[]): RegisterMeasure {
+  const all = passages.flatMap((p) => {
+    const text = runText(runOf(p));
+    return sentences(text).flatMap(([from, to]) => {
+      const s = sentenceOf(text.slice(from, to));
+      return s === null ? [] : [s];
+    });
+  });
+  const conj = all.filter((s) => CONJUNCTIONS.has(s.first));
+  const [first] = passages;
+  return {
+    words: all.reduce((n, s) => n + s.words, 0),
+    conjunctionStarts: conj.length,
+    examples: conj.slice(0, 3).map((s) => s.text),
+    at: first === undefined ? null : startOf(first),
+  };
+}
+
+/** One finding, at the start of the body, when the rate runs above `limit` per 1000 words. */
+export function judgeRegister(
+  m: RegisterMeasure,
+  limit: number = DEFAULT_LIMIT,
+): readonly Located[] {
+  if (m.words < MIN_WORDS) return [];
+  const rate = (1000 * m.conjunctionStarts) / m.words;
+  if (rate <= limit) return [];
+  const data = {
+    count: m.conjunctionStarts,
+    rate: rate.toFixed(2),
+    limit,
+    examples: m.examples.map((x) => `«${x}»`).join(", "),
+  };
+  return [{ messageId: "conjunctionStarts", data, at: m.at }];
+}
+
+/** The limit the config sets, or the default: ESLint has validated the option against the schema. */
+function limitOf(options: readonly unknown[] | undefined): number {
+  const [o] = options ?? [];
+  const given: Readonly<Record<string, unknown>> =
+    typeof o === "object" && o !== null
+      ? Object.fromEntries(Object.entries(o))
+      : {};
+  const v = given["conjunctionStartsPer1000"];
+  return typeof v === "number" ? v : DEFAULT_LIMIT;
+}
+
+const META: TexRuleModule["meta"] = {
+  type: "suggestion",
+  docs: {
+    description:
+      "the body reads at a paper's register: few sentences open with And, So, But, Nor, Or or Yet",
+    url: ruleDocsUrl("register"),
+  },
+  schema: [
+    {
+      type: "object",
+      properties: {
+        conjunctionStartsPer1000: { type: "number", minimum: 0 },
+      },
+      additionalProperties: false,
+    },
+  ],
+  messages: {
+    conjunctionStarts:
+      "{{count}} sentences open with And, So, But, Nor, Or or Yet — {{rate}} per 1000 words (first: {{examples}}). Accepted papers measured 0–0.58; this rule warns above {{limit}}. Join the sentence to the one before, or open with the connective a paper uses (However, Therefore, Moreover)",
+  },
+};
+
+/** The rule, over the body's prose of the whole paper, its includes spliced (`readPaper`). */
+export const registerRule = (deps: PaperDeps): TexRuleModule => ({
+  meta: META,
+  create(context) {
+    return {
+      root() {
+        const sc = context.sourceCode;
+        const paper = readPaper(context.filename, sc.raw ?? sc.text, deps);
+        reportInPaper(
+          context,
+          META.messages,
+          paper,
+          judgeRegister(
+            measureRegister(deps.latex.bodyProse(paper.text)),
+            limitOf(context.options),
+          ),
+        );
+      },
+    };
+  },
+});