paperlint 8.0.0 → 9.0.0

package/src/adapters/latex/filecontents.ts

@@ -0,0 +1,114 @@
+/**
+ * THE `filecontents` BLOCKS OF A SOURCE — where a `.bib` TeX wrote stands in the text that wrote it.
+ * Read from the parse tree, never decided: which file a block writes, and whether TeX ran it, is
+ * TeX's answer (`written` in `_build/sources.json`); this reader only says where each block's text
+ * is, so a finding about an entry TeX read can be shown at the block that holds it
+ * (docs/design/paper-sources.md §1).
+ *
+ * unified-latex reads a `filecontents` environment as verbatim; its option and file name are the
+ * first things in that node, white space between them — a line break too: `\begin{filecontents*}` with
+ * `{refs.bib}` on the next line writes `refs.bib` (measured with pdflatex) — and its body starts on the
+ * line after the name, as LaTeX's own reader starts it.
+ * A block is live when the tree keeps it: a comment is a node of its own, `\iffalse … \fi` is cut at
+ * parse (`conditionals.ts`), and nothing after `\end{document}` is read (`liveRoot`).
+ */
+import type * as Ast from "@unified-latex/unified-latex-types";
+import type { Filecontents } from "../../ports/latex.ts";
+import {
+  argumentPieces,
+  inPlace,
+  isNode,
+  placeOf,
+  visited,
+  type ArgumentPiece,
+  type Node,
+} from "./nodes.ts";
+import { liveRoot, parseLatex, type ParsedTex } from "./parse.ts";
+
+const FILECONTENTS: ReadonlySet<string> = new Set([
+  "filecontents",
+  "filecontents*",
+]);
+
+/** Names and options as TeX reads them: off the tree, a comma between two names. */
+type Name = readonly ArgumentPiece[];
+
+/** What a name or an option is written as: its pieces joined, white space trimmed off. */
+export const written = (name: Name): string =>
+  name
+    .map((p) => p.text)
+    .join("")
+    .trim();
+
+/** What may stand before the name: nothing, or one `[options]` — anything else is a block with no name. */
+const OPTION = /^(?:\[[^\]]*\])?$/;
+
+/** A block's head: its name's group, and its options. */
+interface Head {
+  readonly name: Node;
+  readonly options: readonly string[];
+}
+
+/** The head of a block whose text is `text`: the first group, with only white space and an option before it. */
+function headOf(text: string): Head | null {
+  const nodes = parseLatex(text).root.content;
+  const at = nodes.findIndex((x) => x.type === "group");
+  const option = written(argumentPieces(nodes.slice(0, Math.max(at, 0)), null));
+  const name = nodes[at];
+  if (name === undefined || !OPTION.test(option)) return null;
+  return {
+    name,
+    options: option
+      .replace(/^\[|\]$/g, "")
+      .split(",")
+      .map((o) => o.trim()),
+  };
+}
+
+/**
+ * One verbatim `filecontents` node as a block: its name, option, and body. unified-latex reads the
+ * whole environment as verbatim, so its text is parsed on its own and the name read off that tree like
+ * a declaration's (`headOf`). The body starts on the line after the one the name ends on; what TeX
+ * ignores after the name on that line is no body.
+ */
+export function blockOf(
+  src: string,
+  n: Readonly<Ast.VerbatimEnvironment>,
+  jobname: string,
+): readonly Filecontents[] {
+  return inPlace(placeOf(n), (span) => {
+    // The node runs from `\begin{env}` to past `\end{env}`.
+    const headAt = span.start + `\\begin{${n.env}}`.length;
+    const end = span.end - `\\end{${n.env}}`.length;
+    const head = headOf(src.slice(headAt, end));
+    if (head === null) return [];
+    return inPlace(placeOf(head.name), (named) => {
+      const after = headAt + named.end;
+      // What TeX ignores after the name, up to the end of its line.
+      const rest = src.slice(after, end).split("\n", 1).join("");
+      return [
+        {
+          writes: written(argumentPieces([head.name], jobname)),
+          overwrite: head.options.some(
+            (o) => o === "overwrite" || o === "force",
+          ),
+          span,
+          body: { start: Math.min(after + rest.length + 1, end), end },
+        },
+      ];
+    });
+  });
+}
+
+const isBlock = (n: Node): n is Readonly<Ast.VerbatimEnvironment> =>
+  n.type === "verbatim" && FILECONTENTS.has(n.env);
+
+/** Every live `filecontents` block of a source whose main file's name is `jobname`, in source order. */
+export const filecontentsOf = (
+  t: ParsedTex,
+  jobname: string,
+): readonly Filecontents[] =>
+  visited(liveRoot(t), () => false)
+    .filter(isNode)
+    .filter(isBlock)
+    .flatMap((n) => blockOf(t.src, n, jobname));

package/src/adapters/latex/includes.ts

@@ -4,6 +4,7 @@ import type { Include } from "../../domain/paper-source.ts";
 import type { Span } from "../../domain/tex-document.ts";
 import { isDefinition } from "./hidden.ts";
 import {
+  argumentPieces,
   inPlace,
   isNode,
   macroPlace,
@@ -13,7 +14,7 @@ import {
   type Macro,
   type Node,
 } from "./nodes.ts";
-import type { ParsedTex } from "./parse.ts";
+import { liveRoot, type ParsedTex } from "./parse.ts";
 
 const INCLUDE_MACROS: ReadonlyMap<string, Include["macro"]> = new Map([
   ["input", "input"],
@@ -22,15 +23,22 @@ const INCLUDE_MACROS: ReadonlyMap<string, Include["macro"]> = new Map([
 ]);
 
 /**
- * The path an include names, as written: its braced argument, or — TeX's own form, `\input name` —
- * the name up to the next space. unified-latex gives the unbraced form an argument of the first
- * token only (`sections` of `sections/a`), so the source is read for it.
+ * The path an include names: its braced argument read off the tree as TeX reads it (a comment is no
+ * part of it; a macro stays as written: nothing here expands one), or — TeX's own form, `\\input name`
+ * — the name up to the next space. unified-latex gives the unbraced form an argument of the first
+ * token only (`sections` of `sections/a`), so the source is read for that one.
  */
 function argumentText(src: string, m: Macro): string | null {
-  const start = mandatory(m)[0]?.content[0]?.position?.start.offset;
-  if (start === undefined) return null;
+  const arg = mandatory(m)[0];
+  const start = arg?.content[0]?.position?.start.offset;
+  if (arg === undefined || start === undefined) return null;
+  // The parser gives both forms `{` as their open mark; the source says which one was written.
+  if (src[start - 1] === "{")
+    return argumentPieces(arg.content, null)
+      .map((p) => p.text)
+      .join("");
   const rest = src.slice(start);
-  const stop = rest.search(src[start - 1] === "{" ? /\}/u : /[\s{}%]/u);
+  const stop = rest.search(/[\s{}%]/u);
   return stop < 0 ? rest : rest.slice(0, stop);
 }
 
@@ -44,11 +52,11 @@ function includeOf(src: string, m: Macro): readonly Include[] {
 
 /**
  * Every `\input`, `\include` and `\subfile` of the source, in source order — not in a comment (a
- * comment is a node of its own) and not in a macro definition's body, which is read where the macro
- * is used, and nothing here expands macros.
+ * comment is a node of its own), not in a macro definition's body, which is read where the macro
+ * is used, and nothing here expands macros; not after `\end{document}`, which TeX never reads.
  */
 export const includesOf = (t: ParsedTex): readonly Include[] =>
-  visited(t.root, isDefinition)
+  visited(liveRoot(t), isDefinition)
     .filter(isNode)
     .flatMap((n) => (n.type === "macro" ? includeOf(t.src, n) : []));
 

package/src/adapters/latex/index.test.ts

@@ -697,3 +697,76 @@ describe("bodyEmphasis — each style, and edge cases", () => {
     expect(e && src.slice(e.span.start, e.span.end)).toBe("\\textbf{see it}");
   });
 });
+
+describe("latexReader.filecontents — the blocks that write a file, where they stand", () => {
+  const SRC =
+    "\\documentclass{x}\n" +
+    "\\begin{filecontents*}[overwrite]{refs.bib}\n@misc{a, title={A}}\n\\end{filecontents*}\n" +
+    "\\begin{document}\n" +
+    "\\begin{filecontents}{\\jobname.bib}\n@misc{b, title={B}}\n\\end{filecontents}\n" +
+    "\\end{document}\n" +
+    "\\begin{filecontents}{late.bib}\n@misc{c}\n\\end{filecontents}\n";
+
+  it("each live block, in order: the file it writes (\\jobname expanded), [overwrite], its span and body", () => {
+    const blocks = latexReader.filecontents(SRC, "paper");
+    expect(
+      blocks.map((b) => [
+        b.writes,
+        b.overwrite,
+        SRC.slice(b.body.start, b.body.end),
+      ]),
+    ).toEqual([
+      ["refs.bib", true, "@misc{a, title={A}}\n"],
+      ["paper.bib", false, "@misc{b, title={B}}\n"],
+    ]);
+    expect(SRC.slice(blocks[0]?.span.start, blocks[0]?.span.end)).toMatch(
+      /^\\begin\{filecontents\*\}.*\\end\{filecontents\*\}$/s,
+    );
+  });
+
+  it("a block inside a comment, or after \\end{document}, is not one TeX runs", () => {
+    expect(
+      latexReader.filecontents(
+        "% \\begin{filecontents}{x.bib}\n% @misc{z}\n% \\end{filecontents}\n\\begin{document}\n\\end{document}\n",
+        "paper",
+      ),
+    ).toEqual([]);
+  });
+
+  it("a block that names no file writes none: a group after its body's first text is no name, and a body may hold none", () => {
+    expect(
+      [
+        "\\begin{filecontents}\n@misc{z}\n\\end{filecontents}\n",
+        "\\begin{filecontents}\nplain text\n\\end{filecontents}\n",
+      ].flatMap((block) =>
+        latexReader.filecontents(
+          `${block}\\begin{document}\n\\end{document}\n`,
+          "paper",
+        ),
+      ),
+    ).toEqual([]);
+  });
+});
+
+describe("latexReader.filecontents — a file name on a line of its own", () => {
+  it("🔴 a name on the line after \\begin, the option before it or not: TeX takes it, and the body starts on the line after the name", () => {
+    // Measured with pdflatex: `\\begin{filecontents*}` then `{refs.bib}` on the next line writes
+    // refs.bib from the lines after the name (fixtures/paper-sources/v30-name-next-line).
+    const src =
+      "\\begin{filecontents*}\n{refs.bib}\n@misc{a, title={A}}\n\\end{filecontents*}\n" +
+      "\\begin{filecontents}[overwrite]\n  {two.bib}\n@misc{b}\n\\end{filecontents}\n" +
+      "\\begin{document}\n\\end{document}\n";
+    expect(
+      latexReader
+        .filecontents(src, "paper")
+        .map((b) => [
+          b.writes,
+          b.overwrite,
+          src.slice(b.body.start, b.body.end),
+        ]),
+    ).toEqual([
+      ["refs.bib", false, "@misc{a, title={A}}\n"],
+      ["two.bib", true, "@misc{b}\n"],
+    ]);
+  });
+});

package/src/adapters/latex/index.ts

@@ -33,6 +33,7 @@ import { parseLatex } from "./parse.ts";
 import { bodyEmphasis, bodyProse } from "./prose.ts";
 import { renderedRuns } from "./rendered.ts";
 import { layoutOverridesOf } from "./layout.ts";
+import { filecontentsOf } from "./filecontents.ts";
 
 export {
   documentClassOf,
@@ -69,4 +70,5 @@ export const latexReader: LatexReader = {
   template: parseTemplate,
   withDocumentClass: (src, want) => replaceDocumentClass(parseLatex(src), want),
   layoutOverrides: (src) => layoutOverridesOf(parseLatex(src)),
+  filecontents: (src, jobname) => filecontentsOf(parseLatex(src), jobname),
 };

package/src/adapters/latex/nodes.ts

@@ -156,3 +156,67 @@ export const isArgument = (v: Visited): v is Argument =>
 /** A node, as opposed to a node list or an argument (which visit also reports). */
 export const isNode = (v: Visited): v is Node =>
   !isList(v) && v.type !== "argument" && v.type !== "root";
+
+/** A piece of an argument as TeX reads it: text, or a macro this reader leaves as written (`\x`). */
+export interface ArgumentPiece {
+  readonly text: string;
+  readonly macro: boolean;
+}
+
+/**
+ * An argument's nodes as TeX reads them, off the tree: a comment is no part of it, a group's braces
+ * are none, and white space is one space. `\jobname` is the main file's name when `jobname` is given;
+ * every other macro — `\jobnamebib` included, a control sequence of its own — is a piece of its own,
+ * written as it stands, and so is anything else (math, verbatim) a name cannot hold.
+ */
+export function argumentPieces(
+  nodes: readonly Node[],
+  jobname: string | null,
+): readonly ArgumentPiece[] {
+  return nodes.flatMap((n) => pieceOf(n, jobname));
+}
+
+/** The nodes a name cannot hold as text: an environment, math, verbatim. */
+type Nameless = Extract<
+  Node,
+  {
+    readonly type:
+      | "root"
+      | "environment"
+      | "mathenv"
+      | "inlinemath"
+      | "displaymath"
+      | "verb"
+      | "verbatim";
+  }
+>;
+const NAMELESS: ReadonlySet<string> = new Set([
+  "root",
+  "environment",
+  "mathenv",
+  "inlinemath",
+  "displaymath",
+  "verb",
+  "verbatim",
+]);
+const isNameless = (n: Node): n is Nameless => NAMELESS.has(n.type);
+
+/** One node of an argument as `argumentPieces` reads it. */
+function pieceOf(n: Node, jobname: string | null): readonly ArgumentPiece[] {
+  if (isNameless(n)) return [{ text: `\\${n.type}`, macro: true }];
+  switch (n.type) {
+    case "comment":
+      return [];
+    case "string":
+      return [{ text: n.content, macro: false }];
+    case "whitespace":
+    case "parbreak":
+      return [{ text: " ", macro: false }];
+    case "group":
+      return argumentPieces(n.content, jobname);
+    case "macro":
+      return n.content === "jobname" && jobname !== null
+        ? [{ text: jobname, macro: false }]
+        : [{ text: `\\${n.content}`, macro: true }];
+  }
+}

package/src/adapters/latex/parse.test.ts

@@ -0,0 +1,38 @@
+/**
+ * `parseLatex` keeps the trees of a bounded amount of source text: an editor session must not hold a
+ * tree for every version of the paper it has seen.
+ */
+import { describe, expect, it } from "vitest";
+import { parseLatex } from "./index.ts";
+
+describe("parseLatex's memo", () => {
+  /** A distinct source of `kib` KiB that parses quickly: one long comment. */
+  const source = (tag: string, kib: number) =>
+    `${tag}\n%${"x".repeat(kib * 1024)}`;
+
+  it("returns the same tree for the same text while it is kept", () => {
+    const s = source("same", 1);
+    expect(parseLatex(s)).toBe(parseLatex(s));
+  });
+
+  it("keeps up to 512 KiB of source: a paper's ten files stay, read in a cycle", () => {
+    const files = Array.from({ length: 11 }, (_, i) =>
+      source(`f${String(i)}`, 30),
+    );
+    const first = files.map((f) => parseLatex(f));
+    expect(files.map((f) => parseLatex(f))).toEqual(first);
+    expect(files.every((f, i) => parseLatex(f) === first[i])).toBe(true);
+  });
+
+  it("past 512 KiB the text used least recently goes first, a recently used one stays", () => {
+    const big = Array.from({ length: 6 }, (_, i) =>
+      source(`b${String(i)}`, 100),
+    );
+    const trees = big.slice(0, 5).map((f) => parseLatex(f));
+    // Touch the oldest: it becomes the most recent, so the second is the one evicted.
+    expect(parseLatex(big[0] ?? "")).toBe(trees[0]);
+    parseLatex(big[5] ?? "");
+    expect(parseLatex(big[0] ?? "")).toBe(trees[0]);
+    expect(parseLatex(big[1] ?? "")).not.toBe(trees[1]);
+  });
+});

package/src/adapters/latex/parse.ts

@@ -59,10 +59,13 @@ export const KEY_SIGNATURES: Readonly<
       "subfile",
       "bibliography",
       "bibliographystyle",
-      "addbibresource",
     ],
     "m",
   ),
+  // biblatex's resources take options (`[location=remote]`) before the name.
+  ...signed(["addbibresource", "addglobalbib", "addsectionbib"], "o m"),
+  // bibentry's: writes `\bibdata` as `\bibliography` does, and typesets nothing.
+  ...signed(["nobibliography"], "m"),
   ...signed(["includegraphics"], "s o o m"),
   ...signed(["usepackage", "RequirePackage", "documentclass"], "o m o"),
 ]);
@@ -134,9 +137,67 @@ const parser = once(() =>
   }),
 );
 
+/**
+ * The trees parsed last, by their source: a tree is a pure function of its text, and one lint of a
+ * paper asks for the same texts many times — every rule that reads the paper (`readPaper`)
+ * parses `paper.tex`, its includes and the assembled whole again. Measured 2026-10-06 on the
+ * accepted ACM paper of the corpus: 0.9 s per reading of the paper's sources, seven of them per lint,
+ * 50–76 ms with the memo.
+ *
+ * Bounded by the SOURCE TEXT it holds (UTF-16 units, about bytes for TeX), least recently used out.
+ * A count bound is wrong both ways: 64 trees held 64 versions of one file in an editor (186 MiB for a
+ * 48 KiB paper), and 8 trees thrashed on a paper with ten includes — the rules read them in a cycle,
+ * so every read missed and the corpus lint timed out. `MEMO_BYTES` holds the largest paper of the corpus
+ * (150 KB of sources, and the assembled whole) and about ten versions of a 48 KiB file.
+ */
+const MEMO_BYTES = 512 * 1024;
+const memo = new Map<string, ParsedTex>();
+
+/** The texts to evict, least recent first, so that `adding` more fits under `MEMO_BYTES`. */
+const evicted = (adding: number): readonly string[] => {
+  const keys = [...memo.keys()];
+  const total = keys.reduce((n, k) => n + k.length, adding);
+  const { out } = keys.reduce<{
+    readonly left: number;
+    readonly out: readonly string[];
+  }>(
+    (acc, k) =>
+      acc.left > MEMO_BYTES
+        ? { left: acc.left - k.length, out: [...acc.out, k] }
+        : acc,
+    { left: total, out: [] },
+  );
+  return out;
+};
+
 /** A LaTeX source → its tree, without the branches `\iffalse … \fi` hides (`conditionals.ts`). */
-export const parseLatex = (src: string): ParsedTex => ({
-  src,
-  root: withoutFalseBranches(src, parser().parse(src)),
-  [PARSED]: true,
-});
+export function parseLatex(src: string): ParsedTex {
+  const hit = memo.get(src);
+  const parsed: ParsedTex = hit ?? {
+    src,
+    root: withoutFalseBranches(src, parser().parse(src)),
+    [PARSED]: true,
+  };
+  // Most recent last: a hit moves to the end, and the first keys are the ones used least recently.
+  // eslint-disable-next-line functional/immutable-data -- the bounded memo, see above
+  memo.delete(src);
+  // eslint-disable-next-line functional/immutable-data -- the bounded memo, see above
+  evicted(src.length).forEach((k) => memo.delete(k));
+  // eslint-disable-next-line functional/immutable-data -- the bounded memo, see above
+  memo.set(src, parsed);
+  return parsed;
+}
+
+/**
+ * The part of a source's tree TeX reads: everything up to and with the `document` environment.
+ * TeX stops at `\end{document}`, so a `\bibliography`, a `thebibliography`, a block or an `\input`
+ * parked after it is never read. A source with no `document` environment is read whole.
+ */
+export function liveRoot(t: ParsedTex): Readonly<Ast.Root> {
+  const end = t.root.content.findIndex(
+    (n) => n.type === "environment" && n.env === "document",
+  );
+  return end < 0
+    ? t.root
+    : { ...t.root, content: t.root.content.slice(0, end + 1) };
+}

package/src/adapters/memory/index.ts

@@ -5,6 +5,7 @@
 import { absolutePath } from "../../domain/paths.ts";
 import type { Download } from "../../ports/download.ts";
 import type { Files } from "../../ports/files.ts";
+import type { ListDir } from "../../ports/talk-media.ts";
 import type { Command, ProcessExit, RunProcess } from "../../ports/process.ts";
 import type { NotAPromise, Scratch, Workspace } from "../../ports/workspace.ts";
 import { err, ok } from "../../domain/result.ts";
@@ -55,6 +56,22 @@ export function memoryFiles(
   };
 }
 
+/** `ListDir` over `memoryFiles`: the names directly under a directory, none when it holds no file. */
+export const memoryListDir =
+  (files: MemoryFiles): ListDir =>
+  (dir) => [
+    ...new Set(
+      [...files.map.keys()]
+        .filter((k) => k.startsWith(`${dir}/`))
+        .map((k) =>
+          k
+            .slice(dir.length + 1)
+            .split("/", 1)
+            .join(""),
+        ),
+    ),
+  ];
+
 export interface MemoryWorkspace extends Workspace {
   /** Every file written into a scratch directory, in order. */
   readonly written: {