paperlint 8.0.0 → 9.0.0

package/src/adapters/references/index.test.ts

@@ -11,10 +11,16 @@ import {
   type LookupCache,
 } from "../../domain/lookup-cache.ts";
 import * as cites from "../../../skills/verify-citations/scripts/verify-cites.mjs";
+import { absolutePath } from "../../domain/paths.ts";
+import { bibReader } from "../bibtex/index.ts";
+
+/** A BibTeX text's entries, as the bibtex reader reads them. */
+const read = (bib: string) =>
+  bibReader.readFile(absolutePath("/p/refs.bib"), bib).entries;
 
 /** The verdicts of a cold run — an empty cache, the adapter the CLI wires. */
 const checkOnly = async (bib: string) =>
-  (await onlineReferences(bib, EMPTY_LOOKUP_CACHE)).check;
+  (await onlineReferences(read(bib), EMPTY_LOOKUP_CACHE)).check;
 
 afterEach(() => {
   vi.unstubAllGlobals();
@@ -131,23 +137,75 @@ test("each entry gets its existence and its authors: confirmed, fabricated, mism
   );
 });
 
-test("an author finding names what is extra and what is out of order; an entry only the author check read is unresolvable", async () => {
+test("one key in two databases bibtex opened: each entry gets its own verdict, in the order given", async () => {
+  vi.useFakeTimers();
+  fakeFetch({
+    "https://api.crossref.org/works/10.1%2Fgood": () =>
+      json(200, {
+        message: { title: ["Good Paper"], issued: { "date-parts": [[2024]] } },
+      }),
+    "https://doi.org/api/handles/10.1/good": () =>
+      json(200, { responseCode: 1 }),
+    "https://doi.org/api/handles/10.1/fake": () => json(404),
+    // Crossref knows nothing of the fake: a 404, not a refusal that would stop the run asking it.
+    "https://api.crossref.org/works/10.1%2Ffake": () => json(404),
+    "https://dblp.org/search/publ/api/?q=Good%20Paper": () =>
+      dblp("Good Paper", ["Ada Lovelace"]),
+    "https://api.crossref.org/": () => json(200),
+    "https://dblp.org/": () => json(200, {}),
+  });
+  // Two databases of one `\bibliography`: both define `k` (bibtex takes the first), and only one
+  // copy fails — its existence, or its authors. The failing verdict is the failing copy's own.
+  const good =
+    "@inproceedings{k, author={Ada Lovelace}, title={Good Paper}, booktitle={ICSE}, doi={10.1/good}}";
+  const fake =
+    "@inproceedings{k, author={Ada Lovelace}, title={Fake Paper}, booktitle={ICSE}, doi={10.1/fake}}";
+  const drift =
+    "@inproceedings{k, author={Ada Lovelace and Alan Turing}, title={Good Paper}, booktitle={ICSE}, doi={10.1/good}}";
+  const verdicts = async (bib: string) => {
+    const r = await settle(checkOnly(bib));
+    return (r.kind === "checked" ? r.entries : []).map((e) => [
+      e.key,
+      e.exists,
+      e.authors,
+    ]);
+  };
+  assert.deepEqual(
+    [
+      await verdicts(`${good}\n${fake}`),
+      await verdicts(`${fake}\n${good}`),
+      await verdicts(`${good}\n${drift}`),
+    ],
+    [
+      [
+        ["k", "true", "match"],
+        ["k", "false", "skipped"],
+      ],
+      [
+        ["k", "false", "skipped"],
+        ["k", "true", "match"],
+      ],
+      [
+        ["k", "true", "match"],
+        ["k", "true", "mismatch"],
+      ],
+    ],
+  );
+});
+
+test("an author finding names what is extra and what is out of order", async () => {
   vi.useFakeTimers();
   fakeFetch({
     "https://dblp.org/search/publ/api/?q=Extra%20Paper": () =>
       dblp("Extra Paper", ["Ada Lovelace"]),
     "https://dblp.org/search/publ/api/?q=Swapped%20Paper": () =>
       dblp("Swapped Paper", ["Ada Lovelace", "Alan Turing"]),
-    "https://dblp.org/search/publ/api/?q=Comment%20Paper": () =>
-      dblp("Comment Paper", ["Grace Hopper"]),
     "https://dblp.org/": () => json(200, {}),
     "https://api.crossref.org/": () => json(200, { message: { items: [] } }),
   });
   const bib = [
     "@inproceedings{extra, author={Ada Lovelace and Grace Hopper}, title={Extra Paper}, booktitle={ICSE}}",
     "@inproceedings{swapped, author={Alan Turing and Ada Lovelace}, title={Swapped Paper}, booktitle={ICSE}}",
-    // verify-cites skips @comment blocks; bib-authors reads them — so only one checker sees this key.
-    "@comment{commented, author={Ada Lovelace}, title={Comment Paper}, booktitle={ICSE}}",
   ].join("\n");
   const r = await settle(checkOnly(bib));
   const why = Object.fromEntries(
@@ -165,10 +223,6 @@ test("an author finding names what is extra and what is out of order; an entry o
       "unresolvable",
       "not found by title in any database (no resolvable id to disprove) — could be a legit unindexed/regional/pre-digital work, NOT fabrication; order differs (DBLP: ICSE 2024)",
     ],
-    commented: [
-      "unresolvable",
-      "missing hopper; extra lovelace (DBLP: ICSE 2024)",
-    ],
   });
 });
 
@@ -208,7 +262,7 @@ test(
     slowFetch(40);
     const bib = manyBib(20);
     const serial: string[] = [];
-    for (const c of cites.parseBib(bib))
+    for (const c of cites.citationsOf(read(bib)))
       serial.push((await cites.verifyCitationLive(c, { cache: {} })).verdict);
     const t0 = Date.now();
     const r = await checkOnly(bib);
@@ -265,7 +319,7 @@ const answering = () =>
 async function cold(bib: string) {
   vi.useFakeTimers();
   answering();
-  return settle(cached(bib, EMPTY_LOOKUP_CACHE));
+  return settle(cached(read(bib), EMPTY_LOOKUP_CACHE));
 }
 
 test("cold: every answer is stored with the day it was fetched — responses, never verdicts", async () => {
@@ -324,7 +378,7 @@ test("🔴 warm: a fully cached bibliography asks NOTHING — no lookup, no reac
   // Fake timers with NO advancing: a single DBLP pause would never resolve and time the test out.
   vi.useFakeTimers();
   const calls = fakeFetch({});
-  const second = await cached(bib, first.cache);
+  const second = await cached(read(bib), first.cache);
   assert.deepEqual(
     // Nothing fetched, so the cache comes back as the very object passed in — nothing to write.
     { calls, check: second.check, sameCache: second.cache === first.cache },
@@ -338,7 +392,7 @@ test("🔴 one entry edited: only that entry is asked again, and the old answers
   vi.useFakeTimers();
   const calls = answering();
   const edited = PAPERS.join("\n").replace("Other Paper", "Renamed Paper");
-  const second = await settle(cached(edited, first.cache));
+  const second = await settle(cached(read(edited), first.cache));
   assert.deepEqual(
     {
       // The lookups run concurrently, so the order of requests is not the subject.
@@ -373,14 +427,14 @@ test("an answer older than 30 days is asked again and re-dated; at 30 days it is
   vi.useFakeTimers();
   const quiet = fakeFetch({});
   const at30 = await referencesChecker({ today: () => "2026-10-27" })(
-    bib,
+    read(bib),
     first.cache,
   );
   vi.useRealTimers();
   vi.useFakeTimers();
   const calls = answering();
   const at31 = await settle(
-    referencesChecker({ today: () => "2026-10-28" })(bib, first.cache),
+    referencesChecker({ today: () => "2026-10-28" })(read(bib), first.cache),
   );
   assert.deepEqual(
     {
@@ -408,7 +462,7 @@ test("a failed lookup is not cached: the next run asks it again", async () => {
     "https://dblp.org/": () => json(500),
   });
   const bib = PAPERS[1];
-  const r = await settle(cached(bib, EMPTY_LOOKUP_CACHE));
+  const r = await settle(cached(read(bib), EMPTY_LOOKUP_CACHE));
   const failed: LookupCache = r.cache;
   assert.deepEqual(
     {
@@ -423,7 +477,7 @@ test("offline with an incomplete cache: not-checked, and the cache comes back un
   vi.stubGlobal("fetch", () =>
     Promise.reject(new Error("getaddrinfo ENOTFOUND api.crossref.org")),
   );
-  const r = await cached(PAPERS[0], EMPTY_LOOKUP_CACHE);
+  const r = await cached(read(PAPERS[0]), EMPTY_LOOKUP_CACHE);
   assert.deepEqual(r, {
     check: {
       kind: "not-checked",
@@ -461,7 +515,7 @@ test("🔴 #120: DBLP answering HTML is asked ONCE — no retries, no pause —
     () => new Response("<!doctype html><title>429</title>"),
     () => json(429),
   );
-  const { check } = await cached(PUBLISHED, EMPTY_LOOKUP_CACHE);
+  const { check } = await cached(read(PUBLISHED), EMPTY_LOOKUP_CACHE);
   const entries = check.kind === "checked" ? check.entries : [];
   assert.deepEqual(
     {
@@ -507,14 +561,14 @@ test("#120: a DBLP 429 trips it as well; DBLP answering keeps its 900 ms pace be
     () => json(429),
     () => json(200, { data: [] }),
   );
-  await cached(PUBLISHED, EMPTY_LOOKUP_CACHE);
+  await cached(read(PUBLISHED), EMPTY_LOOKUP_CACHE);
   vi.useRealTimers();
   vi.useFakeTimers();
   const answering = refusing(
     () => dblp("First Paper", ["Ada Lovelace"]),
     () => json(200, { data: [] }),
   );
-  const pending = cached(PUBLISHED, EMPTY_LOOKUP_CACHE);
+  const pending = cached(read(PUBLISHED), EMPTY_LOOKUP_CACHE);
   // One compared entry, then the pause: without advancing time, the run cannot finish.
   let done = false;
   void pending.finally(() => (done = true));

package/src/adapters/references/index.ts

@@ -43,6 +43,7 @@ import type {
   AuthorFinding,
   BibEntry,
 } from "../../../skills/verify-citations/scripts/bib-authors.mjs";
+import { zip } from "remeda";
 import { messageOf } from "../../domain/text.ts";
 import { isRecord } from "../../domain/record.ts";
 
@@ -55,12 +56,13 @@ const describeAuthors = (f: AuthorFinding): string =>
     .filter(Boolean)
     .join("; ") + ` (DBLP: ${f.venue})`;
 
-const authorsOf = (key: string, a: AuthorBuckets): EntryVerdict["authors"] =>
-  a.findings.some((f) => f.key === key)
+/** bib-authors' answer for one entry, from its buckets for that entry alone. */
+const authorsOf = (a: AuthorBuckets): EntryVerdict["authors"] =>
+  a.findings.length > 0
     ? "mismatch"
-    : a.unchecked.some((u) => u.key === key)
+    : a.unchecked.length > 0
       ? "unchecked"
-      : a.matched.includes(key)
+      : a.matched.length > 0
         ? "match"
         : "skipped";
 
@@ -69,36 +71,45 @@ const authorsOf = (key: string, a: AuthorBuckets): EntryVerdict["authors"] =>
  * one, so "unresolvable" is not read as "every registry looked and found nothing". Nothing for a
  * confirmed work: a refusal after a confirmation changed nothing.
  */
-const unconfirmed = (
-  c: CiteVerdict | undefined,
-): readonly (string | undefined)[] =>
-  c && c.verdict !== "true"
+const unconfirmed = (c: CiteVerdict): readonly (string | undefined)[] =>
+  c.verdict !== "true"
     ? [c.reason, ...(c.refused ?? []).map((r) => `not asked: ${r}`)]
     : [];
 
-/** One entry's verdict from the two checkers' answers. */
-function entryVerdict(
-  key: string,
-  found: readonly CiteVerdict[],
-  a: AuthorBuckets,
-): EntryVerdict {
-  const c = found.find((x) => x.id === key);
-  const mismatch = a.findings.find((f) => f.key === key);
+/** One entry's verdict: verify-cites' answer for it, and bib-authors' buckets for it alone. */
+function entryVerdict(c: CiteVerdict, a: AuthorBuckets): EntryVerdict {
   const why = [
     ...unconfirmed(c),
-    mismatch ? describeAuthors(mismatch) : undefined,
-    a.unchecked.find((u) => u.key === key)?.why,
+    ...a.findings.map(describeAuthors),
+    ...a.unchecked.map((u) => u.why),
   ]
     .filter(Boolean)
     .join("; ");
   return {
-    key,
-    exists: c?.verdict ?? "unresolvable",
-    authors: authorsOf(key, a),
+    key: c.id,
+    exists: c.verdict,
+    authors: authorsOf(a),
     ...(why ? { why } : {}),
   };
 }
 
+/**
+ * bib-authors over each entry on its own, one after another as it would run them: its buckets name
+ * an entry by key, and two entries of a bibliography may share a key (two databases bibtex
+ * opened may each define it, with other metadata).
+ */
+const authorsOfEach = (
+  parsed: readonly BibEntry[],
+  d: Pick<ReturnType<typeof cachedDblp>, "lookup" | "pause">,
+): Promise<readonly AuthorBuckets[]> =>
+  parsed.reduce<Promise<readonly AuthorBuckets[]>>(
+    async (done, e) => [
+      ...(await done),
+      await authors.checkAuthors([e], { lookup: d.lookup, pause: d.pause }),
+    ],
+    Promise.resolve([]),
+  );
+
 /** Whether bib-authors will ask DBLP about this entry — the filter its `checkAuthors` applies. */
 const asksDblp = (e: BibEntry): boolean =>
   Boolean(e.title && e.author) &&
@@ -212,8 +223,8 @@ export const referencesChecker =
   async (bib, cache) => {
     // Answers past MAX_AGE_DAYS are left out, so the run asks them again (and `grown` re-dates them).
     const usable = freshPart(cache, today());
-    const citations = cites.parseBib(bib);
-    const parsed = authors.parseBib(bib);
+    const citations = cites.citationsOf(bib);
+    const parsed = authors.authorEntries(bib);
     const store: Record<string, unknown> = Object.fromEntries(
       [...usable.citations].map(([k, v]) => [k, v.response]),
     );
@@ -237,16 +248,10 @@ export const referencesChecker =
           breaker,
         }),
       ),
-      authors.checkAuthors(parsed, {
-        lookup: d.lookup,
-        pause: d.pause,
-      }),
-    ]);
-    const keys = new Set([
-      ...found.map((c) => c.id),
-      ...a.findings.map((f) => f.key),
+      authorsOfEach(parsed, d),
     ]);
-    const entries = [...keys].map((key) => entryVerdict(key, found, a));
+    // Both lists are one per entry, in its order: so is the verdict list.
+    const entries = zip(found, a).map(([c, b]) => entryVerdict(c, b));
     return {
       check: { kind: "checked", entries },
       cache: grown({ cache, usable }, { store, dblp: d.dblp }, today),

package/src/adapters/sources-record/index.test.ts

@@ -0,0 +1,87 @@
+/**
+ * The schema of `_build/sources.json`: the one text it is parsed from, and every way a file can fail to
+ * be a record of this schema — refused by name, never read as a record.
+ */
+import { describe, expect, it } from "vitest";
+import {
+  serializeSourcesRecord,
+  type SourcesRecord,
+} from "../../domain/sources-record.ts";
+import { sourcesCodec } from "./index.ts";
+
+const parseSourcesRecord = sourcesCodec.parse;
+const A = "a".repeat(64);
+
+const RECORD: SourcesRecord = {
+  schema: 1,
+  inputs: [
+    { path: "paper.tex", role: "body" },
+    { path: "macros.tex", role: "preamble" },
+  ],
+  written: ["refs.bib"],
+  bibdata: ["refs"],
+  bibtex: {
+    ran: true,
+    databases: ["refs.bib"],
+    keys: ["k1"],
+    exit: 2,
+    errors: [{ message: "bad", file: "refs.bib", line: 4 }],
+  },
+  sha256: { "paper.tex": A, "macros.tex": null },
+};
+
+describe("parsing a sources.json", () => {
+  it("what serialize wrote parses back to the same record, and the text ends with a newline", () => {
+    const text = serializeSourcesRecord(RECORD);
+    expect(text.endsWith("}\n")).toBe(true);
+    expect(parseSourcesRecord(text)).toEqual({ ok: true, record: RECORD });
+  });
+
+  it("a build that ran no bibtex parses too", () => {
+    const record: SourcesRecord = { ...RECORD, bibtex: { ran: false } };
+    expect(parseSourcesRecord(serializeSourcesRecord(record))).toEqual({
+      ok: true,
+      record,
+    });
+  });
+
+  it("text that is not JSON is refused by name", () => {
+    expect(parseSourcesRecord("{")).toEqual({
+      ok: false,
+      why: "sources.json is not JSON",
+    });
+  });
+
+  it("another schema number is refused by name, not read as this one", () => {
+    expect(parseSourcesRecord('{"schema":2}')).toEqual({
+      ok: false,
+      why: "sources.json is schema 2, this paperlint reads schema 1",
+    });
+    expect(parseSourcesRecord("[]")).toEqual({
+      ok: false,
+      why: "sources.json is schema unknown, this paperlint reads schema 1",
+    });
+    expect(parseSourcesRecord('{"schema":"1"}')).toEqual({
+      ok: false,
+      why: "sources.json is schema unknown, this paperlint reads schema 1",
+    });
+  });
+
+  it("a record of this schema that is not shaped like one names where", () => {
+    const bad = (patch: object): string => {
+      const r = parseSourcesRecord(JSON.stringify({ ...RECORD, ...patch }));
+      return r.ok ? "accepted" : r.why;
+    };
+    expect(bad({ inputs: "paper.tex" })).toContain("inputs");
+    expect(bad({ sha256: { "paper.tex": "not a digest" } })).toContain(
+      "sha256.paper.tex",
+    );
+    expect(bad({ bibtex: { ran: "yes" } })).toContain("bibtex");
+    expect(bad({ inputs: [{ path: "a.tex", role: "main" }] })).toContain(
+      "inputs.0.role",
+    );
+    expect(parseSourcesRecord(JSON.stringify({ schema: 1 }))).toMatchObject({
+      ok: false,
+    });
+  });
+});

package/src/adapters/sources-record/index.ts

@@ -0,0 +1,82 @@
+/**
+ * The schema of `_build/sources.json` — the one place its shape is checked (docs/design/paper-sources.md
+ * §1). The text is parsed once, here, into the domain's `SourcesRecord`; whatever reads a record after
+ * it never re-validates. A record of another schema number is refused by name, not read as this one.
+ */
+import { z } from "zod";
+import {
+  SOURCES_FILE,
+  SOURCES_SCHEMA,
+  type ParsedRecord,
+} from "../../domain/sources-record.ts";
+import type { SourcesCodec } from "../../ports/sources-codec.ts";
+
+const SHA256 = /^[0-9a-f]{64}$/;
+
+const bibtexError = z.object({
+  message: z.string(),
+  file: z.string(),
+  line: z.number().int().nullable(),
+});
+
+const recordSchema = z.object({
+  schema: z.literal(SOURCES_SCHEMA),
+  inputs: z.array(
+    z.object({ path: z.string(), role: z.enum(["preamble", "body"]) }),
+  ),
+  written: z.array(z.string()),
+  bibdata: z.array(z.string()),
+  bibtex: z.discriminatedUnion("ran", [
+    z.object({ ran: z.literal(false) }),
+    z.object({
+      ran: z.literal(true),
+      databases: z.array(z.string()),
+      keys: z.array(z.string()),
+      exit: z.number().int(),
+      errors: z.array(bibtexError),
+    }),
+  ]),
+  sha256: z.record(z.string(), z.string().regex(SHA256).nullable()),
+});
+
+/** The JSON `text` holds, or undefined when it is not JSON. */
+function jsonOf(text: string): unknown {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+}
+
+/** The `schema` field of a parsed file, whatever it holds, or undefined when there is none. */
+const schemaOf = (json: unknown): unknown =>
+  typeof json === "object" && json !== null && "schema" in json
+    ? json.schema
+    : undefined;
+
+/**
+ * A `sources.json`'s text, parsed once at the boundary: the record, or the one reason it is not one.
+ * A record of another schema number is refused by name, not read as this one.
+ */
+export function parseSourcesRecord(text: string): ParsedRecord {
+  const json = jsonOf(text);
+  if (json === undefined)
+    return { ok: false, why: `${SOURCES_FILE} is not JSON` };
+  const schema = schemaOf(json);
+  if (schema !== SOURCES_SCHEMA)
+    return {
+      ok: false,
+      why: `${SOURCES_FILE} is schema ${typeof schema === "number" ? String(schema) : "unknown"}, this paperlint reads schema ${String(SOURCES_SCHEMA)}`,
+    };
+  const parsed = recordSchema.safeParse(json);
+  if (parsed.success) return { ok: true, record: parsed.data };
+  const issues = parsed.error.issues.map(
+    (i) => `${i.path.join(".")} ${i.message}`,
+  );
+  return {
+    ok: false,
+    why: `${SOURCES_FILE} does not match schema ${String(SOURCES_SCHEMA)}: ${issues.join("; ")}`,
+  };
+}
+
+export const sourcesCodec: SourcesCodec = { parse: parseSourcesRecord };

package/src/adapters/tex-output/blg.ts

@@ -0,0 +1,27 @@
+/**
+ * The errors in bibtex's log (`paper.blg`). bibtex ends an error with the place it was reading —
+ * `---line 4 of file refs.bib`, or `---while reading file paper.aux` when it names no line — and a
+ * warning with a single dash (`--line 1 of file x.bib`), so the three dashes are the whole grammar.
+ * The message is what stands before them on the line, or, when they start the line, the line before.
+ */
+import type { BibtexError } from "../../domain/sources-record.ts";
+
+const PLACE = /^(.*?)---(?:line (\d+) of file|while reading file) (.+)$/;
+const LINE_NUMBER = /---line (\d+) of file/;
+
+export function parseBlgErrors(blg: string): readonly BibtexError[] {
+  const rows = blg.split(/\r?\n/);
+  return rows.flatMap((row, i): readonly BibtexError[] => {
+    const m = PLACE.exec(row);
+    if (m === null) return [];
+    const [, before = "", , file = ""] = m;
+    const line = LINE_NUMBER.exec(row)?.[1];
+    return [
+      {
+        message: (before === "" ? (rows[i - 1] ?? "") : before).trim(),
+        file: file.trim(),
+        line: line === undefined ? null : Number(line),
+      },
+    ];
+  });
+}

package/src/adapters/tex-output/fls.ts

@@ -0,0 +1,24 @@
+/**
+ * pdflatex's `-recorder` file (`paper.fls`): one line per file TeX opened — `INPUT <path>` to read
+ * it, `OUTPUT <path>` to write it — in the order TeX opened them, and a `PWD <dir>` line first. The
+ * format has no quoting: a path is the rest of its line.
+ */
+import type { Fls, FlsLine } from "../../domain/tex-run.ts";
+
+const OPS = ["INPUT", "OUTPUT"] as const;
+
+/** `INPUT a.tex` → the line; anything else (`PWD …`, a blank, a word with no path) → none. */
+function lineOf(raw: string): readonly FlsLine[] {
+  const op = OPS.find((o) => raw.startsWith(`${o} `));
+  const path = op === undefined ? "" : raw.slice(op.length + 1).trimEnd();
+  return op === undefined || path === "" ? [] : [{ op, path }];
+}
+
+export function parseFls(text: string): Fls {
+  const rows = text.split(/\r?\n/);
+  const pwd = rows.find((r) => r.startsWith("PWD "));
+  return {
+    pwd: pwd === undefined ? null : pwd.slice("PWD ".length).trimEnd(),
+    lines: rows.flatMap(lineOf),
+  };
+}