paperlint 3.1.1 → 3.1.3

package/src/ports/check-references.ts

@@ -3,6 +3,7 @@
  * Crossref answer. The adapter (`adapters/references/`) runs the package's two checkers; a test
  * passes a function.
  */
+import type { LookupCache } from "../domain/lookup-cache.ts";
 
 /** What one bibliography entry came to. */
 export interface EntryVerdict {
@@ -24,5 +25,18 @@ export type ReferencesCheck =
   /** Nothing could be asked — no network, every service down. Never a pass. */
   | { readonly kind: "not-checked"; readonly why: string };
 
-/** The bibliography's text (BibTeX) → the verdicts. Never throws: a failure is `not-checked`. */
-export type CheckReferences = (bib: string) => Promise<ReferencesCheck>;
+/** A check's verdicts, and the lookup cache grown by whatever it had to fetch. */
+export interface ReferencesRun {
+  readonly check: ReferencesCheck;
+  /** The cache passed in, plus every new successful answer; the same object when nothing was fetched. */
+  readonly cache: LookupCache;
+}
+
+/**
+ * The bibliography's text (BibTeX) and the paper's lookup cache → the verdicts, derived afresh from
+ * cached and newly fetched answers. Never throws: a failure is `not-checked`.
+ */
+export type CheckReferences = (
+  bib: string,
+  cache: LookupCache,
+) => Promise<ReferencesRun>;

package/src/references.test.ts

@@ -11,6 +11,7 @@ import { describe, expect, it } from "vitest";
 import { ESLint } from "eslint";
 import { join } from "node:path";
 import { memoryFiles } from "./adapters/memory/index.ts";
+import { ok } from "./domain/result.ts";
 import type { AbsolutePath } from "./domain/paths.ts";
 import type {
   CheckReferences,
@@ -20,9 +21,16 @@ import { referencesStep } from "./build.ts";
 import {
   bibHash,
   bibliographyOf,
+  lookupCachePath,
   readReferences,
   referencesPath,
 } from "./references.ts";
+import {
+  parseLookupCache,
+  serializeLookupCache,
+  type CachedResponse,
+  type LookupCache,
+} from "./domain/lookup-cache.ts";
 import { referenceRules, REFERENCE_RULE_LEVELS } from "./reference-rules.ts";
 // @ts-expect-error — an ESLint language in .mjs, it has no types
 import { texLanguage } from "../eslint-rules/latex-language.mjs";
@@ -55,22 +63,30 @@ const verdict = (
 
 const checker =
   (entries: readonly EntryVerdict[]): CheckReferences =>
-  async () => ({ kind: "checked", entries });
-const offline: CheckReferences = async () => ({
-  kind: "not-checked",
-  why: "the citation services cannot be reached (fetch failed)",
+  async (_bib, cache) => ({ check: { kind: "checked", entries }, cache });
+const offline: CheckReferences = async (_bib, cache) => ({
+  check: {
+    kind: "not-checked",
+    why: "the citation services cannot be reached (fetch failed)",
+  },
+  cache,
+});
+
+/** The step's context over `files`, with every port it does not use stubbed. */
+const ctx = (files: ReturnType<typeof memoryFiles>) => ({
+  paperDir: PAPER,
+  env: {},
+  run: (() => ({})) as never,
+  readPdf: (() => null) as never,
+  measure: (() => null) as never,
+  files,
 });
 
 /** Run the build step on a paper held in memory; returns the files and the step's outcome. */
 async function build(tex: string, check: CheckReferences) {
   const files = memoryFiles({ [`${PAPER}/paper.tex`]: tex });
   const out = await referencesStep.run({
-    paperDir: PAPER,
-    env: {},
-    run: (() => ({})) as never,
-    readPdf: (() => null) as never,
-    measure: (() => null) as never,
-    files,
+    ...ctx(files),
     checkReferences: check,
   });
   return { files, out };
@@ -144,6 +160,108 @@ describe("the references build step", () => {
   });
 });
 
+const ANSWER: CachedResponse = {
+  fetched: "2026-09-27",
+  response: { db: "crossref", transport: "ok", query: "doi" },
+};
+/** A checker that fetches `key` unless the cache has it, recording the cache it was handed. */
+function fetching(key: string) {
+  const seen: LookupCache[] = [];
+  const check: CheckReferences = async (_bib, cache) => {
+    seen.push(cache);
+    return {
+      check: { kind: "checked", entries: [verdict("schick2023")] },
+      cache: cache.citations.has(key)
+        ? cache
+        : {
+            citations: new Map([...cache.citations, [key, ANSWER]]),
+            dblp: cache.dblp,
+          },
+    };
+  };
+  return { check, seen };
+}
+const cacheText = (files: ReturnType<typeof memoryFiles>): string | null => {
+  const b = files.readBytes(lookupCachePath(PAPER) as AbsolutePath);
+  return b === null ? null : new TextDecoder().decode(b);
+};
+
+describe("the lookup cache — <paper>/repro/references-cache.json", () => {
+  it("a build that fetched writes the cache; the next build is handed it and writes nothing", async () => {
+    const f = fetching("crossref:doi:10.1/x");
+    const { files, out } = await build(TEX(ENTRIES), f.check);
+    expect(out).toEqual({
+      ok: true,
+      note: "references: 1 checked, 0 failing → _build/references.json; 1 new answer → repro/references-cache.json",
+    });
+    const written = cacheText(files);
+    expect(written).toBe(
+      serializeLookupCache({
+        citations: new Map([["crossref:doi:10.1/x", ANSWER]]),
+        dblp: new Map(),
+      }),
+    );
+    const again = await referencesStep.run({
+      ...ctx(files),
+      checkReferences: f.check,
+    });
+    expect(again).toEqual({
+      ok: true,
+      note: "references: 1 checked, 0 failing → _build/references.json; nothing fetched",
+    });
+    // The second build was handed exactly what the first one wrote.
+    expect(parseLookupCache(written!)).toEqual(ok(f.seen[1]));
+    expect(cacheText(files)).toBe(written);
+  });
+});
+
+describe("the lookup cache — counting and refusing", () => {
+  it("two new answers are counted as two", async () => {
+    const two: CheckReferences = async (_bib, cache) => ({
+      check: { kind: "checked", entries: [verdict("schick2023")] },
+      cache: {
+        citations: new Map([
+          ["a", ANSWER],
+          ["b", ANSWER],
+        ]),
+        dblp: cache.dblp,
+      },
+    });
+    const { out } = await build(TEX(ENTRIES), two);
+    expect(out).toEqual({
+      ok: true,
+      note: "references: 1 checked, 0 failing → _build/references.json; 2 new answers → repro/references-cache.json",
+    });
+  });
+
+  it("no answer fetched and no cache on disk: no cache file is created", async () => {
+    const { files } = await build(
+      TEX(ENTRIES),
+      checker([verdict("schick2023")]),
+    );
+    expect(cacheText(files)).toBeNull();
+  });
+
+  it("🔴 a cache that does not parse is refused BY NAME and left alone — the check is not run over half a cache", async () => {
+    const files = memoryFiles({
+      [`${PAPER}/paper.tex`]: TEX(ENTRIES),
+      [lookupCachePath(PAPER)]: '{"schema": 7}',
+    });
+    const f = fetching("k");
+    const out = await referencesStep.run({
+      ...ctx(files),
+      checkReferences: f.check,
+    });
+    expect(out).toEqual({
+      ok: true,
+      note: "references NOT checked — repro/references-cache.json cannot be read (schema 7: this paperlint reads schema 1) — fix it, or delete it to ask every question again; lint will say so",
+    });
+    expect(f.seen).toEqual([]);
+    expect(cacheText(files)).toBe('{"schema": 7}');
+    expect(readReferences(files, PAPER)?.status).toBe("not-checked");
+  });
+});
+
 describe("the reference rules", () => {
   it("one mismatched entry → paper/author-list on that entry's own line", async () => {
     const tex = TEX(ENTRIES);
@@ -304,3 +422,35 @@ describe("the reference rules over refs.bib, and on other files", () => {
       );
   });
 });
+
+describe("the lookup cache — refreshed answers", () => {
+  // Guards: an answer asked again after MAX_AGE_DAYS keeps its key, so the key count does not change.
+  // Counted by size, the refreshed answer was not written, and every later build asked it again.
+  it("an answer refreshed under the same key is counted and written", async () => {
+    const OLD: CachedResponse = { ...ANSWER, fetched: "2026-08-01" };
+    const put =
+      (answer: CachedResponse): CheckReferences =>
+      async (_bib, cache) => ({
+        check: { kind: "checked", entries: [verdict("schick2023")] },
+        cache: {
+          citations: new Map([...cache.citations, ["a", answer]]),
+          dblp: cache.dblp,
+        },
+      });
+    const { files } = await build(TEX(ENTRIES), put(OLD));
+    const again = await referencesStep.run({
+      ...ctx(files),
+      checkReferences: put(ANSWER),
+    });
+    expect({ again, file: cacheText(files) }).toEqual({
+      again: {
+        ok: true,
+        note: "references: 1 checked, 0 failing → _build/references.json; 1 new answer → repro/references-cache.json",
+      },
+      file: serializeLookupCache({
+        citations: new Map([["a", ANSWER]]),
+        dblp: new Map(),
+      }),
+    });
+  });
+});

package/src/references.ts

@@ -17,6 +17,12 @@
  * stale when it changes (`pdf_sha256`); these are about the bibliography and are stale when IT
  * changes (`bib.sha256`). One staleness key per file, the way `pdf/fresh` already works.
  *
+ * 🔴 WHAT THE CHECKERS ANSWERED IS KEPT, COMMITTED, IN `<paper>/repro/references-cache.json`
+ * (`domain/lookup-cache.ts`). Loaded before the check and saved after it — only when something new
+ * was fetched, a refreshed answer included — so a build asks each question at most once every
+ * MAX_AGE_DAYS, not once per build (#107: 217 s for 27 references, every build). The verdicts above are NOT cached: they are derived from the cached
+ * answers on every build, so a fix to the checkers reaches every paper at once.
+ *
  * 🔴 THE STEP NEVER FAILS THE BUILD. A build without network still builds the PDF; the reference
  * step records `not-checked` and the lint rule says so, as a warning. Recording "not checked" as
  * a pass would be the counter that counts what it never looked at.
@@ -25,10 +31,18 @@ import { join } from "node:path";
 import type { AbsolutePath } from "./domain/paths.ts";
 import { sha256Hex } from "./domain/sha256.ts";
 import type { Files } from "./ports/files.ts";
+import {
+  EMPTY_LOOKUP_CACHE,
+  parseLookupCache,
+  serializeLookupCache,
+  type LookupCache,
+} from "./domain/lookup-cache.ts";
+import { ok, type Result } from "./domain/result.ts";
 import type {
   CheckReferences,
   EntryVerdict,
   ReferencesCheck,
+  ReferencesRun,
 } from "./ports/check-references.ts";
 // @ts-expect-error — an ESLint rule module in .mjs, it has no types
 import { bibRange } from "../eslint-rules/paper-typography.mjs";
@@ -36,6 +50,12 @@ import { bibRange } from "../eslint-rules/paper-typography.mjs";
 export const REFERENCES_SCHEMA = 1;
 export const REFERENCES_FILE = "references.json";
 
+/** The lookup cache's place in a paper: committed, beside the paper's other reproduction files. */
+export const LOOKUP_CACHE_FILE = "repro/references-cache.json";
+
+export const lookupCachePath = (paperDir: string): string =>
+  join(paperDir, LOOKUP_CACHE_FILE);
+
 /** Where a paper's reference verdicts live — beside `paper.facts.json`. */
 export const referencesPath = (paperDir: string): string =>
   join(paperDir, "_build", REFERENCES_FILE);
@@ -106,10 +126,48 @@ export function readReferences(
   }
 }
 
+/** The paper's lookup cache: empty when there is none, or why the file on disk cannot be read. */
+export function readLookupCache(
+  files: Files,
+  paperDir: string,
+): Result<LookupCache, string> {
+  const raw = text(files, lookupCachePath(paperDir));
+  return raw === null ? ok(EMPTY_LOOKUP_CACHE) : parseLookupCache(raw);
+}
+
 /**
- * The build step's work: check the bibliography and record the verdicts. Returns the one-line
- * note the build prints. A checker that throws is recorded as `not-checked` — the step reports,
- * it never refuses.
+ * How many answers the run fetched: entries of `after` that are not the very object `before` held
+ * under that key. The checker keeps every answer it did not fetch as the same object, so this counts
+ * both new keys and answers asked again past MAX_AGE_DAYS under an existing key — which a count of
+ * keys would miss, leaving the refreshed answer unwritten and asked again on every build.
+ */
+const newAnswers = (before: LookupCache, after: LookupCache): number => {
+  const changed = <V>(b: ReadonlyMap<string, V>, a: ReadonlyMap<string, V>) =>
+    [...a].filter(([k, v]) => b.get(k) !== v).length;
+  return (
+    changed(before.citations, after.citations) +
+    changed(before.dblp, after.dblp)
+  );
+};
+
+/** Run the checker; a throw is `not-checked`, with the cache as it was. */
+async function run(
+  check: CheckReferences,
+  bib: string,
+  cache: LookupCache,
+): Promise<ReferencesRun> {
+  try {
+    return await check(bib, cache);
+  } catch (e) {
+    return { check: { kind: "not-checked", why: (e as Error).message }, cache };
+  }
+}
+
+/**
+ * The build step's work: check the bibliography, serving what it can from the paper's lookup
+ * cache, and record the verdicts. Returns the one-line note the build prints. A checker that
+ * throws, or a cache that does not parse, is recorded as `not-checked` — the step reports, it
+ * never refuses. A cache that does not parse is left as it is, and named.
  */
 export async function recordReferences(
   files: Files,
@@ -118,21 +176,39 @@ export async function recordReferences(
 ): Promise<string> {
   const bib = bibliographyOf(files, paperDir);
   if (bib === null) return "no bibliography — nothing to check";
-  let result: ReferencesCheck;
-  try {
-    result = await check(bib.text);
-  } catch (e) {
-    result = { kind: "not-checked", why: (e as Error).message };
-  }
-  const doc = documentOf(bib, result);
-  files.writeAtomic(
-    at(referencesPath(paperDir)),
-    new TextEncoder().encode(`${JSON.stringify(doc, null, 2)}\n`),
-  );
-  if (result.kind === "not-checked")
-    return `references NOT checked — ${result.why}; lint will say so`;
+  const record = (c: ReferencesCheck): ReferencesDocument => {
+    const doc = documentOf(bib, c);
+    files.writeAtomic(
+      at(referencesPath(paperDir)),
+      new TextEncoder().encode(`${JSON.stringify(doc, null, 2)}\n`),
+    );
+    return doc;
+  };
+  const notChecked = (why: string): string => {
+    record({ kind: "not-checked", why });
+    return `references NOT checked — ${why}; lint will say so`;
+  };
+  const cache = readLookupCache(files, paperDir);
+  if (!cache.ok)
+    return notChecked(
+      `${LOOKUP_CACHE_FILE} cannot be read (${cache.error}) — fix it, or delete it to ask every question again`,
+    );
+  const result = await run(check, bib.text, cache.value);
+  if (result.check.kind === "not-checked") return notChecked(result.check.why);
+  const doc = record(result.check);
+  const fetched = newAnswers(cache.value, result.cache);
+  if (fetched > 0)
+    files.writeAtomic(
+      at(lookupCachePath(paperDir)),
+      new TextEncoder().encode(serializeLookupCache(result.cache)),
+    );
   const bad = doc.entries.filter(
     (e) => e.exists === "false" || e.authors === "mismatch",
   ).length;
-  return `references: ${String(doc.entries.length)} checked, ${String(bad)} failing → _build/${REFERENCES_FILE}`;
+  return (
+    `references: ${String(doc.entries.length)} checked, ${String(bad)} failing → _build/${REFERENCES_FILE}; ` +
+    (fetched > 0
+      ? `${String(fetched)} new answer${fetched === 1 ? "" : "s"} → ${LOOKUP_CACHE_FILE}`
+      : "nothing fetched")
+  );
 }