paperlint 3.1.0 → 3.1.2

package/src/adapters/references/index.ts

@@ -2,17 +2,39 @@
  * The online reference checks, as the `CheckReferences` port: `verify-cites` (the cited work
  * exists, the title matches — Crossref, OpenAlex, Semantic Scholar, arXiv) and `bib-authors` (the
  * authors are the published version's — DBLP). Both ship in this package's verify-citations skill;
- * this adapter only runs them and shapes their answers.
+ * this adapter only runs them, shapes their answers, and serves them from the paper's lookup cache.
+ *
+ * 🔴 THE CACHE HOLDS ANSWERS, AND ONLY THE CHECKERS' OWN SEAMS TOUCH IT. verify-cites takes a
+ * `cache` object keyed per resolver × identifier and stores only successful responses; it is
+ * handed the paper's cached responses and whatever it adds is new. bib-authors takes a `lookup`
+ * (title → DBLP hits) and a `pause`; the lookup is wrapped to answer from the cache, and the pause
+ * is skipped after a cached answer — DBLP's 900 ms pace and its retries run only for titles never
+ * answered. Neither checker's logic changes. Verdicts are derived from the answers on every build
+ * (`domain/lookup-cache.ts` says why).
  *
  * "Not checked" is decided up front, by one request: when Crossref cannot be reached at all, the
  * checkers would degrade every entry to `unresolvable` / `unchecked`, which reads like a result.
- * Saying `not-checked` with the reason is the honest record.
+ * Saying `not-checked` with the reason is the honest record. That request is skipped when every
+ * question this bibliography would ask is already answered in the cache: a warm build is offline.
  */
 import type {
   CheckReferences,
   EntryVerdict,
 } from "../../ports/check-references.ts";
+import {
+  freshPart,
+  type CachedDblp,
+  type CachedResponse,
+  type DblpHit,
+  type LookupCache,
+} from "../../domain/lookup-cache.ts";
+import { sha256Hex } from "../../domain/sha256.ts";
+import { mapLimit } from "./pool.ts";
 import { unreachable } from "./reach.io.ts";
+import { sleep, todayUtc } from "./clock.io.ts";
+
+/** Citations looked up at once (#107). */
+const LOOKUPS_IN_FLIGHT = 6;
 // @ts-expect-error — a skill script in .mjs, it has no types
 import * as cites from "../../../skills/verify-citations/scripts/verify-cites.mjs";
 // @ts-expect-error — a skill script in .mjs, it has no types
@@ -22,6 +44,8 @@ interface CiteResult {
   readonly id: string;
   readonly verdict: "true" | "false" | "unresolvable";
   readonly reason?: string;
+  /** "<service>: <reason>" for each question a refusing service did not answer (#120). */
+  readonly refused?: readonly string[];
 }
 interface AuthorFinding {
   readonly key: string;
@@ -60,6 +84,18 @@ const authorsOf = (key: string, a: AuthorBuckets): EntryVerdict["authors"] =>
         ? "match"
         : "skipped";
 
+/**
+ * Why a work was not confirmed: verify-cites' reason, then — when some registry refused — which
+ * one, so "unresolvable" is not read as "every registry looked and found nothing". Nothing for a
+ * confirmed work: a refusal after a confirmation changed nothing.
+ */
+const unconfirmed = (
+  c: CiteResult | undefined,
+): readonly (string | undefined)[] =>
+  c && c.verdict !== "true"
+    ? [c.reason, ...(c.refused ?? []).map((r) => `not asked: ${r}`)]
+    : [];
+
 /** One entry's verdict from the two checkers' answers. */
 function entryVerdict(
   key: string,
@@ -69,7 +105,7 @@ function entryVerdict(
   const c = found.find((x) => x.id === key);
   const mismatch = a.findings.find((f) => f.key === key);
   const why = [
-    c && c.verdict !== "true" ? c.reason : undefined,
+    ...unconfirmed(c),
     mismatch ? describeAuthors(mismatch) : undefined,
     a.unchecked.find((u) => u.key === key)?.why,
   ]
@@ -83,20 +119,168 @@ function entryVerdict(
   };
 }
 
-export const onlineReferences: CheckReferences = async (bib) => {
-  const why = await unreachable();
-  if (why !== null) return { kind: "not-checked", why };
-  const cache = {};
-  const found: CiteResult[] = [];
-  for (const c of cites.parseBib(bib) as { id?: string }[])
-    if (c.id) found.push(await cites.verifyCitationLive(c, { cache }));
-  const a = (await authors.checkAuthors(
-    authors.parseBib(bib),
-  )) as AuthorBuckets;
-  const keys = new Set([
-    ...found.map((c) => c.id),
-    ...a.findings.map((f) => f.key),
+interface BibAuthorsEntry {
+  readonly key: string;
+  readonly title: string;
+  readonly author: string;
+}
+
+/** Whether bib-authors will ask DBLP about this entry — the filter its `checkAuthors` applies. */
+const asksDblp = (e: BibAuthorsEntry): boolean =>
+  Boolean(e.title && e.author) &&
+  (authors.claimsPublished(e) as boolean) &&
+  !(authors.truncated(e.author) as boolean);
+
+/** Is every question this bibliography would ask already answered in `cache`? */
+async function fullyCached(
+  citations: readonly object[],
+  entries: readonly BibAuthorsEntry[],
+  cache: LookupCache,
+  store: Readonly<Record<string, unknown>>,
+): Promise<boolean> {
+  const asks = await Promise.all(
+    citations.map((c) => cites.wouldAsk(c, store) as Promise<boolean>),
+  );
+  return (
+    !asks.includes(true) &&
+    entries.filter(asksDblp).every((e) => cache.dblp.has(dblpTitleKey(e.title)))
+  );
+}
+
+/**
+ * The key a DBLP answer is stored under in the lookup cache: the title's identity (verify-cites'
+ * `titleIdentity` — the same rule its own title keys use, so the two caches cannot drift), hashed.
+ * A re-cased or re-braced title keeps its answer; `C` and `C++` do not share one.
+ */
+export const dblpTitleKey = (title: string): string =>
+  sha256Hex(new TextEncoder().encode(cites.titleIdentity(title))).slice(0, 16);
+
+export interface ReferencesCheckerOptions {
+  /** The day a new answer is stamped with, YYYY-MM-DD. */
+  readonly today: () => string;
+}
+
+/**
+ * bib-authors' `lookup` and `pause`, answering from the cached DBLP hits first.
+ *
+ * 🔴 DBLP IS ASKED UNTIL IT REFUSES ONCE, THEN NOT AT ALL FOR THE REST OF THE RUN (#120). A 429, an
+ * HTML page where JSON was asked, a timeout: bib-authors would retry each title three times with
+ * backoff, and move on to the next title to do it again — 51 titles × 3 × up to 15 s. After the
+ * first refusal every lookup fails at once with the reason, and no pause is waited, so each entry
+ * lands in bib-authors' `unchecked` bucket ("never a pass") for the price of ONE request.
+ *
+ * The 900 ms pace protects DBLP, so it is kept only after an answer DBLP actually gave: not after a
+ * cached answer, and not once DBLP is off.
+ */
+function cachedDblp(cache: LookupCache, today: () => string) {
+  const dblp = new Map<string, CachedDblp>(cache.dblp);
+  let lastWasCached = false;
+  let refused: string | null = null;
+  return {
+    dblp,
+    lookup: async (title: string): Promise<readonly DblpHit[]> => {
+      const hit = dblp.get(dblpTitleKey(title));
+      lastWasCached = hit !== undefined;
+      if (hit) return hit.hits;
+      if (refused !== null)
+        throw new Error(`DBLP refused earlier in this run (${refused})`);
+      try {
+        const hits = (await authors.dblpHits(title)) as DblpHit[];
+        dblp.set(dblpTitleKey(title), { fetched: today(), title, hits });
+        return hits;
+      } catch (e) {
+        refused = (e as Error).message;
+        throw new Error(`DBLP refused earlier in this run (${refused})`);
+      }
+    },
+    pause: (ms: number): Promise<void> =>
+      lastWasCached || refused !== null ? Promise.resolve() : sleep(ms),
+  };
+}
+
+/**
+ * The cache after a run: the old answers with every new one laid over them, dated today — or the
+ * very object passed in when nothing was fetched, so the caller can tell there is nothing to write.
+ * "New" is measured against `usable`, the part the run answered from: an answer too old to use was
+ * asked again, and its fresh copy replaces the old one. An old answer the run could not refresh
+ * (the service refused) stays, and is asked again next build.
+ */
+function grown(
+  loaded: { readonly cache: LookupCache; readonly usable: LookupCache },
+  run: {
+    readonly store: Readonly<Record<string, unknown>>;
+    readonly dblp: ReadonlyMap<string, CachedDblp>;
+  },
+  today: () => string,
+): LookupCache {
+  const { cache, usable } = loaded;
+  const { store, dblp } = run;
+  const fresh = Object.entries(store).filter(([k]) => !usable.citations.has(k));
+  const freshDblp = [...dblp].filter(([k]) => !usable.dblp.has(k));
+  if (fresh.length === 0 && freshDblp.length === 0) return cache;
+  const dated = fresh.map(([k, response]): [string, CachedResponse] => [
+    k,
+    { fetched: today(), response: response as CachedResponse["response"] },
   ]);
-  const entries = [...keys].map((key) => entryVerdict(key, found, a));
-  return { kind: "checked", entries };
-};
+  return {
+    citations: new Map([...cache.citations, ...dated]),
+    dblp: new Map([...cache.dblp, ...freshDblp]),
+  };
+}
+
+/** The checker, with its clock injected; `onlineReferences` is the one the CLI wires. */
+export const referencesChecker =
+  ({ today }: ReferencesCheckerOptions): CheckReferences =>
+  async (bib, cache) => {
+    // Answers past MAX_AGE_DAYS are left out, so the run asks them again (and `grown` re-dates them).
+    const usable = freshPart(cache, today());
+    const citations = (cites.parseBib(bib) as { id?: string }[]).filter(
+      (c) => c.id,
+    );
+    const parsed = authors.parseBib(bib) as BibAuthorsEntry[];
+    const store: Record<string, unknown> = Object.fromEntries(
+      [...usable.citations].map(([k, v]) => [k, v.response]),
+    );
+    if (!(await fullyCached(citations, parsed, usable, store))) {
+      const why = await unreachable();
+      if (why !== null) return { check: { kind: "not-checked", why }, cache };
+    }
+    // 🔴 #107: the lookups run LOOKUPS_IN_FLIGHT at a time, and the DBLP author pass runs beside
+    // them. Each citation's own requests go to four different services one after another, so a
+    // pool of 6 is at most 6 requests to any one of them. The store is shared: two citations with
+    // the same identifier may both miss it and ask twice — an extra request, never a different
+    // answer.
+    // One breaker for the run: a service that refuses is not asked again, not once per citation
+    // (#120). Requests already in flight when it starts refusing still land — at most six.
+    const breaker = cites.createBreaker() as unknown;
+    const d = cachedDblp(usable, today);
+    const [found, a] = await Promise.all([
+      mapLimit(
+        citations,
+        LOOKUPS_IN_FLIGHT,
+        (c) =>
+          cites.verifyCitationLive(c, {
+            cache: store,
+            breaker,
+          }) as Promise<CiteResult>,
+      ),
+      authors.checkAuthors(parsed, {
+        lookup: d.lookup,
+        pause: d.pause,
+      }) as Promise<AuthorBuckets>,
+    ]);
+    const keys = new Set([
+      ...found.map((c) => c.id),
+      ...a.findings.map((f) => f.key),
+    ]);
+    const entries = [...keys].map((key) => entryVerdict(key, found, a));
+    return {
+      check: { kind: "checked", entries },
+      cache: grown({ cache, usable }, { store, dblp: d.dblp }, today),
+    };
+  };
+
+/** The checker the CLI wires: the real clock. */
+export const onlineReferences: CheckReferences = referencesChecker({
+  today: todayUtc,
+});

package/src/adapters/references/pool.test.ts

@@ -0,0 +1,31 @@
+/** mapLimit: results in input order, never more than `limit` in flight, an empty list at once. */
+import assert from "node:assert/strict";
+import { test } from "vitest";
+import { mapLimit } from "./pool.ts";
+
+test("results keep the input order although they finish out of order", async () => {
+  const out = await mapLimit([30, 5, 20, 1], 2, async (ms) => {
+    await new Promise((r) => {
+      setTimeout(r, ms);
+    });
+    return ms * 10;
+  });
+  assert.deepEqual(out, [300, 50, 200, 10]);
+});
+
+test("never more than `limit` calls in flight; fewer items than the limit start fewer workers", async () => {
+  let now = 0;
+  let peak = 0;
+  const track = async () => {
+    peak = Math.max(peak, ++now);
+    await new Promise((r) => {
+      setTimeout(r, 2);
+    });
+    now--;
+  };
+  await mapLimit(Array.from({ length: 10 }), 3, track);
+  const three = peak;
+  peak = 0;
+  await mapLimit([1, 2], 6, track);
+  assert.deepEqual([three, peak, await mapLimit([], 6, track)], [3, 2, []]);
+});

package/src/adapters/references/pool.ts

@@ -0,0 +1,24 @@
+/**
+ * `fn` over `items` with at most `limit` calls in flight, results in `items`' order — a promise
+ * pool, the one piece of concurrency this adapter needs, so it is written here rather than
+ * depended on. A rejection rejects the whole map, as `Promise.all` would: callers that must not
+ * lose the other results catch inside `fn`.
+ */
+export async function mapLimit<T, R>(
+  items: readonly T[],
+  limit: number,
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const out: R[] = new Array<R>(items.length);
+  let next = 0;
+  const worker = async (): Promise<void> => {
+    while (next < items.length) {
+      const i = next++;
+      out[i] = await fn(items[i] as T);
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, worker),
+  );
+  return out;
+}

package/src/build-engine.harness.mjs

@@ -8,7 +8,6 @@
  *   2. prepareEngine: use the cache · use the system · refuse without a human · ask, decline,
  *      accept and install · dry-run never asks or installs.
  */
-import assert from "node:assert/strict";
 import {
   chmodSync,
   mkdirSync,
@@ -20,15 +19,12 @@ import {
 import { tmpdir } from "node:os";
 import { basename, delimiter, dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { createChecker } from "../lib/check.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const E = await import(join(HERE, "build-engine.ts"));
 
-let n = 0;
-const check = (label, cond, detail = "") => {
-  assert.ok(cond, `${label}${detail ? ` — ${detail}` : ""}`);
-  n++;
-};
+const check = createChecker();
 
 // ── 1. texts ────────────────────────────────────────────────────────────────────────────
 const noCache = {
@@ -311,4 +307,4 @@ const x_run = { found: {} };
 }
 
 rmSync(work, { recursive: true, force: true });
-console.log(`build-engine: ${n} checks passed`);
+console.log(`build-engine: ${check.count} checks passed`);

package/src/build-engine.test.ts

@@ -0,0 +1,137 @@
+/**
+ * build-engine.ts arms the harness does not reach: a system pdflatex that lacks packages (said
+ * before asking), an install that fails, an install that leaves packages missing, a cache used from
+ * an environment with no PATH, and the refusal line for a decision that is neither refuse nor ask.
+ */
+import assert from "node:assert/strict";
+import { chmodSync, mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { test } from "vitest";
+import { useTempDir } from "../test/support.mjs";
+import { prepareEngine, refusal } from "./build-engine.ts";
+
+const work = useTempDir("paperlint-build-engine-test-");
+const TEX = { packages: { acmart: ["acmart.cls"] }, tools: {} };
+/** A runnable pdflatex in `dir`. */
+const texAt = (dir: string): string => {
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(join(dir, "pdflatex"), "");
+  chmodSync(join(dir, "pdflatex"), 0o755);
+  return dir;
+};
+/** kpsewhich answers per bin directory. */
+const fakeRun = (found: Record<string, string>) =>
+  ((cmd: string) => ({
+    stdout: found[dirname(cmd)] ?? "",
+    status: 0,
+  })) as never;
+
+async function go(o: Parameters<typeof prepareEngine>[0]) {
+  const out: string[] = [];
+  const err: string[] = [];
+  const r = await prepareEngine({
+    log: (l) => out.push(l),
+    err: (l) => err.push(l),
+    platform: "linux",
+    interactive: true,
+    ask: async () => "y",
+    ...o,
+  });
+  return { r, out, err };
+}
+
+test("a system pdflatex lacking packages is named before asking; a failed install stops the run", async () => {
+  const sys = texAt(join(work, "sys", "bin"));
+  const x = await go({
+    tex: TEX,
+    env: { PAPERLINT_TEXLIVE_DIR: join(work, "empty-cache"), PATH: sys },
+    run: fakeRun({}),
+    install: () => ({ ok: false }),
+  });
+  assert.deepEqual(
+    [x.r, x.out],
+    [
+      { ok: false, code: 1 },
+      [`${join(sys, "pdflatex")} is not used: it lacks acmart`],
+    ],
+  );
+});
+
+test("an install that verifies but still lacks a package is refused, naming it", async () => {
+  const cache = join(work, "cache-lacking");
+  const x = await go({
+    tex: TEX,
+    env: { PAPERLINT_TEXLIVE_DIR: cache, PATH: join(work, "nothing") },
+    run: fakeRun({}),
+    install: () => {
+      texAt(join(cache, "2026", "bin", "x86_64-linux"));
+      return { ok: true, tree: { dir: "", year: "2026", bin: "" } };
+    },
+  });
+  assert.deepEqual(x.r, { ok: false, code: 1 });
+  assert.match(x.err.join("\n"), /missing: 1 package\(s\): acmart/);
+});
+
+test("an install that leaves no tree at all is refused with nothing named missing", async () => {
+  const x = await go({
+    tex: TEX,
+    env: {
+      PAPERLINT_TEXLIVE_DIR: join(work, "cache-none"),
+      PATH: join(work, "nothing"),
+    },
+    run: fakeRun({}),
+    install: () => ({ ok: true, tree: { dir: "", year: "2026", bin: "" } }),
+  });
+  assert.deepEqual(x.r, { ok: false, code: 1 });
+  assert.match(x.err.join("\n"), /missing: 0 package\(s\): \)/);
+});
+
+test("a complete cache is used from an environment with no PATH at all", async () => {
+  const bin = texAt(join(work, "cache-ok", "2026", "bin", "x86_64-linux"));
+  const x = await go({
+    tex: TEX,
+    env: { PAPERLINT_TEXLIVE_DIR: join(work, "cache-ok") },
+    run: fakeRun({ [bin]: "/t/acmart.cls\n" }),
+  });
+  assert.deepEqual(x.r, {
+    ok: true,
+    env: { PAPERLINT_TEXLIVE_DIR: join(work, "cache-ok"), PATH: `${bin}:` },
+  });
+});
+
+test("the refusal line for a decision to use a tree names nothing missing", () => {
+  assert.equal(
+    refusal({ kind: "use-cache" } as never, { system: null } as never),
+    "✗ paperlint build: no TeX Live with every package these papers need — run `npx paperlint toolchain` (missing: 0 package(s): )",
+  );
+});
+
+test("without an ask or an install given: Enter is yes, and the default install runs (here: every mirror fails)", async () => {
+  const cache = join(work, "cache-default");
+  const calls: string[] = [];
+  const out: string[] = [];
+  const err: string[] = [];
+  const r = await prepareEngine({
+    tex: TEX,
+    log: (l) => out.push(l),
+    err: (l) => err.push(l),
+    platform: "linux",
+    interactive: true,
+    env: {
+      PAPERLINT_TEXLIVE_DIR: cache,
+      PAPERLINT_CTAN_MIRROR: "https://mirror.invalid/tlnet",
+      PATH: join(work, "nothing"),
+    },
+    run: ((cmd: string) => {
+      calls.push(cmd.split("/").pop() ?? cmd);
+      return {
+        status: 22,
+        stdout: "",
+        stderr: "curl: (6) could not resolve\n",
+      };
+    }) as never,
+  });
+  assert.deepEqual(r, { ok: false, code: 1 });
+  assert.ok(calls.includes("curl"), calls.join(","));
+  assert.match(err.join("\n"), /no CTAN mirror returned install-tl/);
+});

package/src/build.harness.mjs

@@ -10,7 +10,6 @@
  * `build.sh` is never executed (checked by the trace it would leave on disk and by the list of
  * processes started), and a failed build removes the stale `paper.pdf` (checked on disk).
  */
-import assert from "node:assert/strict";
 import {
   mkdtempSync,
   mkdirSync,
@@ -23,6 +22,7 @@ import {
 import { tmpdir } from "node:os";
 import { join, dirname, delimiter } from "node:path";
 import { fileURLToPath } from "node:url";
+import { createChecker } from "../lib/check.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const {
@@ -44,11 +44,7 @@ const { packageVenuesDir } = await import(
   join(HERE, "..", "skills", "paper-pipeline", "scripts", "consumer.mjs")
 );
 
-let n = 0;
-const check = (label, cond) => {
-  assert.ok(cond, label);
-  n++;
-};
+const check = createChecker();
 
 const root = realpathSync(mkdtempSync(join(tmpdir(), "paperlint-build-")));
 // 🔴 banal is looked for under the temp root, never in the developer's cache: with the real
@@ -511,8 +507,8 @@ try {
   check("on full success there is no remedy", remedyFor([r]) === "");
 
   // ── 🔴 A STALE PDF NEVER SURVIVES A RUN THAT DID NOT REPLACE IT — every path of buildPapers ──
-  // The order matters for the battery: the up-front removal is killed by the no-engine case, the
-  // existence check by the empty document, and neither case depends on the other defence.
+  // The order matters here: the up-front removal is exercised by the no-engine case below, the
+  // existence check by the empty-document case, and neither depends on the other defence.
   const REMOVED =
     "paper.pdf removed — a stale PDF must not pass for this build";
 
@@ -680,5 +676,5 @@ try {
 }
 
 console.log(
-  `✓ ${String(n)} assertions passed — build: paperlint compiles, build.sh never runs, a red build leaves no PDF`,
+  `✓ ${String(check.count)} assertions passed — build: paperlint compiles, build.sh never runs, a red build leaves no PDF`,
 );