paperlint 7.2.0 → 9.0.0

package/src/reference-rules.ts

@@ -5,25 +5,47 @@
  *
  *   entry whose authors are the preprint's   paper/author-list (error), on the entry
  *   entry whose identifier provably fails    paper/cite-exists (error), on the entry
- *   no record, or recorded "not checked"     paper/refs-checked (warn) — like pdf/measured
- *   bibliography edited after the build      paper/refs-fresh (error) — like pdf/fresh
+ *   no verdicts, or recorded "not checked"   paper/refs-checked (warn) — like pdf/measured
+ *   a key bibtex typeset that no entry the  paper/refs-checked (warn): that reference was not
+ *   reader read has (`unseenKeys`)           checked
+ *   a database bibtex found outside the      paper/refs-checked (warn): its references were not checked
+ *   paper directory (BIBINPUTS)
+ *   databases edited after the build         paper/refs-fresh (error) — like pdf/fresh
  *
- * While the record is stale, the per-entry rules are silent: they would judge a bibliography
- * that no longer exists, and refs-fresh already says so once. A paper with no bibliography gets
- * nothing.
+ * The databases are the ones the last build's bibtex opened (`_build/sources.json`,
+ * docs/design/paper-sources.md §1) — `readBibliography` answers only from a record the paper has not
+ * changed since. With no record, or a stale one, every rule here is silent: `paper/sources-fresh`
+ * speaks once. While the verdicts are stale, the per-entry rules are silent too: they would judge
+ * databases that no longer exist, and refs-fresh already says so once. A paper whose build ran no
+ * bibtex gets nothing.
+ *
+ * `paper/author-list` and `paper/cite-exists` lay the verdicts on the entries of those databases. ESLint
+ * does not lint a `.bib`, so a finding is a place in `paper.tex`: an entry of a `.bib` TeX wrote from a
+ * block of `paper.tex` is reported on its line there (`entryReports`); any other entry at the top of
+ * `paper.tex`, with the entry's file, line and column at the front of the message (`refs.bib:12:1:`).
  *
  * `paper/author-list` used to live on PIPELINE-STATUS.md and ask whether the scorecard mentioned
  * a run. It lives here now because its subject is the bibliography, and the record of the run is
  * the run's own output.
  */
 import { basename, dirname } from "node:path";
-import type { Files } from "./ports/files.ts";
+import { callerPath } from "./caller-path.ts";
+import { MAIN_FILE } from "./domain/paper-sources.ts";
+import type { EntryVerdict } from "./ports/check-references.ts";
+import {
+  entryReports,
+  type EntryReport,
+  type RecordedDeps,
+  type RecordedEntry,
+} from "./recorded-bibliography.ts";
 import {
   bibHash,
-  bibliographyOf,
+  readBibliography,
   readReferences,
   REFERENCES_FILE,
-  type Bibliography,
+  unseenKeys,
+  type CheckedBibliography,
+  type ReferencesDocument,
 } from "./references.ts";
 
 interface Loc {
@@ -34,6 +56,8 @@ interface RuleContext {
   readonly filename: string;
   readonly cwd: string;
   readonly sourceCode: {
+    /** The file as written (the `tex/latex` language's source code). */
+    readonly raw: string;
     getLocFromIndex(i: number): Loc;
   };
   report(d: {
@@ -54,50 +78,77 @@ export interface ReferenceRuleModule {
 
 type Name = "author-list" | "cite-exists" | "refs-checked" | "refs-fresh";
 
+/** The checked entries of the databases, each with the verdict the build recorded for it. */
+interface Judged {
+  readonly bib: CheckedBibliography;
+  readonly verdicts: readonly (readonly [RecordedEntry, EntryVerdict])[];
+}
+
 /** What one lint run knows about a paper's references. */
 type Assessment =
-  | { readonly kind: "no-bibliography" }
+  /** No usable record of the build, or a build that ran no bibtex: nothing to judge. */
+  | { readonly kind: "silent" }
+  | { readonly kind: "unread"; readonly databases: readonly string[] }
   | { readonly kind: "unrecorded" }
   | { readonly kind: "not-checked"; readonly why: string }
   | { readonly kind: "stale" }
-  | {
-      readonly kind: "ready";
-      readonly bib: Bibliography;
-      readonly failing: readonly {
-        readonly key: string;
-        readonly rule: "author-list" | "cite-exists";
-        readonly why: string;
-      }[];
-    };
-
-function assess(files: Files, paperDir: string): Assessment {
-  const bib = bibliographyOf(files, paperDir);
-  if (bib === null) return { kind: "no-bibliography" };
-  const doc = readReferences(files, paperDir);
+  | { readonly kind: "ready"; readonly judged: Judged };
+
+/**
+ * Each entry checked with its verdict — when the record holds one verdict per entry, in its order, as
+ * the build writes it. One that does not (written by hand) is not about these databases: null.
+ */
+function laid(
+  verdicts: readonly EntryVerdict[],
+  entries: readonly RecordedEntry[],
+): Judged["verdicts"] | null {
+  const pairs = entries.flatMap((f, i) => {
+    const v = verdicts[i];
+    return v?.key === f.entry.key ? [[f, v] as const] : [];
+  });
+  return pairs.length === entries.length && verdicts.length === entries.length
+    ? pairs
+    : null;
+}
+
+/** What the verdicts on disk say about the databases of `bib`. */
+function judgedVerdicts(
+  bib: CheckedBibliography,
+  doc: ReferencesDocument | null,
+): Assessment {
   if (doc === null) return { kind: "unrecorded" };
   if (doc.bib.sha256 !== bibHash(bib)) return { kind: "stale" };
   if (doc.status === "not-checked")
     return { kind: "not-checked", why: doc.why ?? "no reason recorded" };
-  const failing = doc.entries.flatMap((e) => [
-    ...(e.authors === "mismatch"
-      ? [{ key: e.key, rule: "author-list" as const, why: e.why ?? "" }]
-      : []),
-    ...(e.exists === "false"
-      ? [{ key: e.key, rule: "cite-exists" as const, why: e.why ?? "" }]
-      : []),
-  ]);
-  return { kind: "ready", bib, failing };
+  const verdicts = laid(doc.entries, bib.entries);
+  return verdicts === null
+    ? { kind: "stale" }
+    : { kind: "ready", judged: { bib, verdicts } };
 }
 
-/** Where entry `key` starts in the source file: its `@type{key,` line, or the file's start. */
-function entryOffset(bib: Bibliography, key: string): number {
-  const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-  const m = new RegExp(`^@\\w+\\s*\\{\\s*${escaped}\\s*,`, "m").exec(bib.text);
-  return m ? bib.offset + m.index : 0;
+function assess(deps: RecordedDeps, paperDir: string): Assessment {
+  const read = readBibliography(paperDir, deps);
+  switch (read.kind) {
+    case "no-record":
+    case "nothing":
+    case "not-wired":
+      return { kind: "silent" };
+    case "unread":
+      return { kind: "unread", databases: read.databases };
+    case "read":
+      return judgedVerdicts(
+        read.bibliography,
+        readReferences(deps.files, paperDir),
+      );
+  }
 }
 
 const BUILD = "`npx paperlint build`";
 
+const MISMATCH =
+  "`{{key}}`: the authors are not those of the version cited — {{why}}. The citation resolves and the id resolves, and the list is the PREPRINT's under a published venue";
+const MISSING = "`{{key}}` does not resolve to the work cited — {{why}}";
+
 const META: Readonly<Record<Name, ReferenceRuleModule["meta"]>> = {
   "author-list": {
     type: "problem",
@@ -106,10 +157,7 @@ const META: Readonly<Record<Name, ReferenceRuleModule["meta"]>> = {
         "each entry's authors are those of the version it cites, not the preprint's (recorded by paperlint build)",
     },
     schema: [],
-    messages: {
-      mismatch:
-        "`{{key}}`: the authors are not those of the version cited — {{why}}. The citation resolves and the id resolves, and the list is the PREPRINT's under a published venue",
-    },
+    messages: { mismatch: MISMATCH, mismatchIn: `{{where}}: ${MISMATCH}` },
   },
   "cite-exists": {
     type: "problem",
@@ -118,9 +166,7 @@ const META: Readonly<Record<Name, ReferenceRuleModule["meta"]>> = {
         "each entry's identifier resolves to the work cited (recorded by paperlint build)",
     },
     schema: [],
-    messages: {
-      missing: "`{{key}}` does not resolve to the work cited — {{why}}",
-    },
+    messages: { missing: MISSING, missingIn: `{{where}}: ${MISSING}` },
   },
   "refs-checked": {
     type: "suggestion",
@@ -132,13 +178,16 @@ const META: Readonly<Record<Name, ReferenceRuleModule["meta"]>> = {
     messages: {
       unrecorded: `the references have not been checked — ${BUILD} checks them online and records the result in _build/${REFERENCES_FILE}`,
       notChecked: `the last build could not check the references ({{why}}) — run ${BUILD} with network`,
+      unseen:
+        "the last build's bibtex typeset {{what}}, which paperlint's reader did not read from the databases bibtex opened, so those references were not checked: an entry behind `%` or inside `@comment{…}` — bibtex has no comment syntax and reads it, the reader skips it. Delete the entry, or remove its `@`",
+      unread: `{{what}}, which the last build's bibtex opened, is not in the paper directory (bibtex found it on its search path, BIBINPUTS), so its references were not checked — move it beside paper.tex`,
     },
   },
   "refs-fresh": {
     type: "problem",
     docs: {
       description:
-        "the recorded reference verdicts are about the bibliography on disk, not an earlier one",
+        "the recorded reference verdicts are about the databases on disk, not an earlier set",
     },
     schema: [],
     messages: {
@@ -147,76 +196,140 @@ const META: Readonly<Record<Name, ReferenceRuleModule["meta"]>> = {
   },
 };
 
-type Report = { at: number; messageId: string; data?: Record<string, string> };
-
-const JUDGES: Readonly<Record<Name, (a: Assessment) => Report[]>> = {
-  "author-list": (a) =>
-    a.kind === "ready"
-      ? a.failing
-          .filter((f) => f.rule === "author-list")
-          .map((f) => ({
-            at: entryOffset(a.bib, f.key),
-            messageId: "mismatch",
-            data: { key: f.key, why: f.why },
-          }))
-      : [],
-  "cite-exists": (a) =>
-    a.kind === "ready"
-      ? a.failing
-          .filter((f) => f.rule === "cite-exists")
-          .map((f) => ({
-            at: entryOffset(a.bib, f.key),
-            messageId: "missing",
-            data: { key: f.key, why: f.why },
-          }))
-      : [],
-  "refs-checked": (a) =>
-    a.kind === "unrecorded"
-      ? [{ at: 0, messageId: "unrecorded" }]
-      : a.kind === "not-checked"
-        ? [{ at: 0, messageId: "notChecked", data: { why: a.why } }]
-        : [],
-  "refs-fresh": (a) =>
-    a.kind === "stale" ? [{ at: 0, messageId: "stale" }] : [],
+interface Report {
+  /** Where in paper.tex; the start of the file when absent. */
+  readonly at?: EntryReport | undefined;
+  readonly messageId: string;
+  readonly data?: Record<string, string>;
+}
+
+const AT_START: EntryReport = { kind: "here", span: { start: 0, end: 0 } };
+
+const JUDGES: Readonly<
+  Record<"refs-checked" | "refs-fresh", (a: Assessment) => readonly Report[]>
+> = {
+  "refs-checked": (a) => {
+    switch (a.kind) {
+      case "unrecorded":
+        return [{ messageId: "unrecorded" }];
+      case "not-checked":
+        return [{ messageId: "notChecked", data: { why: a.why } }];
+      case "unread":
+        return [
+          { messageId: "unread", data: { what: a.databases.join(", ") } },
+        ];
+      case "ready": {
+        const unseen = unseenKeys(a.judged.bib);
+        return unseen.length === 0
+          ? []
+          : [
+              {
+                messageId: "unseen",
+                data: { what: unseen.map((k) => `\`${k}\``).join(", ") },
+              },
+            ];
+      }
+      case "silent":
+      case "stale":
+        return [];
+    }
+  },
+  "refs-fresh": (a) => (a.kind === "stale" ? [{ messageId: "stale" }] : []),
 };
 
-/** The four rules, reading through `files`. They act on `paper.tex` only. */
-export function referenceRules({
-  files,
-}: {
-  files: Files;
-}): Record<Name, ReferenceRuleModule> {
-  const make = (name: Name): ReferenceRuleModule => ({
+/**
+ * One finding, at its entry in paper.tex — or, for an entry in a `.bib`, at the top, with the `…In`
+ * message that names the entry's file, line and column first.
+ */
+function report(context: RuleContext, r: Report): void {
+  const loc = (i: number) => context.sourceCode.getLocFromIndex(i);
+  const at = r.at ?? AT_START;
+  context.report({
+    loc: { start: loc(at.span.start), end: loc(at.span.end) },
+    ...(at.kind === "here"
+      ? { messageId: r.messageId, ...(r.data ? { data: r.data } : {}) }
+      : {
+          messageId: `${r.messageId}In`,
+          data: { ...r.data, where: at.where },
+        }),
+  });
+}
+
+/** The verdicts a rule reports, and the message each makes. */
+const VERDICT_RULES = {
+  "author-list": {
+    failed: (v: EntryVerdict) => v.authors === "mismatch",
+    messageId: "mismatch",
+  },
+  "cite-exists": {
+    failed: (v: EntryVerdict) => v.exists === "false",
+    messageId: "missing",
+  },
+} as const;
+
+/**
+ * The entries the verdicts judged failing for `rule`, each where it is reported (`entryReports`): the
+ * line of its `filecontents` block in the paper's text when TeX wrote its `.bib` from one, else the
+ * top of `paper.tex`.
+ */
+function verdictReports(
+  rule: keyof typeof VERDICT_RULES,
+  context: RuleContext,
+  deps: RecordedDeps,
+): readonly Report[] {
+  const main = callerPath(context.filename);
+  const dir = dirname(main);
+  const a = assess(deps, dir);
+  if (a.kind !== "ready") return [];
+  const { failed, messageId } = VERDICT_RULES[rule];
+  const flagged = a.judged.verdicts.filter(([, v]) => failed(v));
+  const at = entryReports(
+    {
+      dir,
+      record: a.judged.bib.record,
+      main: { path: main, text: context.sourceCode.raw },
+    },
+    flagged.map(([f]) => f),
+    deps,
+  );
+  return flagged.map(([, v], i) => ({
+    at: at[i],
+    messageId,
+    data: { key: v.key, why: v.why ?? "" },
+  }));
+}
+
+/** The four rules, reading the paper through `deps`. They act on `paper.tex` only. */
+export function referenceRules(
+  deps: RecordedDeps,
+): Record<Name, ReferenceRuleModule> {
+  const make = (
+    name: Name,
+    judge: (context: RuleContext) => readonly Report[],
+  ): ReferenceRuleModule => ({
     meta: META[name],
     create: (context) =>
-      basename(context.filename) !== "paper.tex"
+      basename(context.filename) !== MAIN_FILE
         ? {}
         : {
             "root:exit": () => {
-              const paperDir = dirname(context.filename);
-              const a = assess(files, paperDir);
-              // An external refs.bib: the entry is not in this file, so the finding sits at its
-              // start and names the key.
-              const external =
-                a.kind === "ready" && a.bib.source !== "paper.tex";
-              for (const r of JUDGES[name](a)) {
-                const loc = context.sourceCode.getLocFromIndex(
-                  external ? 0 : r.at,
-                );
-                context.report({
-                  loc: { start: loc, end: loc },
-                  messageId: r.messageId,
-                  ...(r.data ? { data: r.data } : {}),
-                });
-              }
+              for (const r of judge(context)) report(context, r);
             },
           },
   });
+  const assessed =
+    (name: "refs-checked" | "refs-fresh") =>
+    (context: RuleContext): readonly Report[] =>
+      JUDGES[name](assess(deps, dirname(callerPath(context.filename))));
   return {
-    "author-list": make("author-list"),
-    "cite-exists": make("cite-exists"),
-    "refs-checked": make("refs-checked"),
-    "refs-fresh": make("refs-fresh"),
+    "author-list": make("author-list", (c) =>
+      verdictReports("author-list", c, deps),
+    ),
+    "cite-exists": make("cite-exists", (c) =>
+      verdictReports("cite-exists", c, deps),
+    ),
+    "refs-checked": make("refs-checked", assessed("refs-checked")),
+    "refs-fresh": make("refs-fresh", assessed("refs-fresh")),
   };
 }