paperlint 3.1.0 → 3.1.1

package/src/adapters/node/files.test.ts

@@ -5,7 +5,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll as after, test } from "vitest";
 import type { AbsolutePath } from "../../domain/paths.ts";
-import { nodeFiles } from "./files.io.ts";
+import { errorCode, nodeFiles } from "./files.io.ts";
 
 const root = realpathSync(mkdtempSync(join(tmpdir(), "paperlint-files-test-")));
 after(() => rmSync(root, { recursive: true, force: true }));
@@ -26,3 +26,15 @@ test("Files: writeAtomic creates the directory and leaves no .part behind", () =
   );
   assert.equal(existsSync(`${p}.part`), false);
 });
+
+test("Files: a read that fails for another reason than absence throws — a directory is not an empty file", () => {
+  assert.throws(() => nodeFiles.readBytes(at()), { code: "EISDIR" });
+});
+
+test("errorCode: an Error's errno code; a thrown non-Error has none", () => {
+  assert.equal(
+    errorCode(Object.assign(new Error("x"), { code: "ENOENT" })),
+    "ENOENT",
+  );
+  assert.equal(errorCode("ENOENT"), undefined);
+});

package/src/adapters/node/process.test.ts

@@ -5,6 +5,7 @@
  */
 import assert from "node:assert/strict";
 import { test } from "vitest";
+import type { AbsolutePath } from "../../domain/paths.ts";
 import type { Command } from "../../ports/process.ts";
 import { spawnProcess } from "./process.io.ts";
 
@@ -44,3 +45,40 @@ test("the environment is the Command's, whole: nothing leaks in from the parent"
   // Guards: `Command.env` is the child's entire environment — the adapter merges nothing.
   assert.deepEqual(r.kind === "exited" && r.stdout, "[][1]\n");
 });
+
+test("a program killed by a signal is `signalled`, with what it printed first", () => {
+  assert.deepEqual(run.run(sh("echo before; kill -TERM $$")), {
+    kind: "signalled",
+    signal: "SIGTERM",
+    stdout: "before\n",
+    stderr: "",
+  });
+});
+
+test("output past maxOutputBytes is a spawn failure, named by Node's own message", () => {
+  const r = run.run({ ...sh("head -c 100000 /dev/zero"), maxOutputBytes: 10 });
+  assert.equal(r.kind, "spawn-failed");
+  assert.match(r.kind === "spawn-failed" ? r.message : "", /ENOBUFS/);
+});
+
+test("cwd is the Command's when it names one", () => {
+  const r = run.run({ ...sh("pwd"), cwd: "/" as AbsolutePath });
+  assert.deepEqual(r.kind === "exited" && r.stdout, "/\n");
+});
+
+test("a spawn that reports no status, no signal and no error reads as exit 1, not success", () => {
+  const silent = spawnProcess((() => ({
+    pid: 0,
+    output: [],
+    stdout: "",
+    stderr: "",
+    status: null,
+    signal: null,
+  })) as never);
+  assert.deepEqual(silent.run(sh("true")), {
+    kind: "exited",
+    status: 1,
+    stdout: "",
+    stderr: "",
+  });
+});

package/src/adapters/pdfjs/fill.test.ts

@@ -109,3 +109,37 @@ check(
   "fillsOf: an empty item is `unknown`",
   L.fillsOf(OPS, ops(), [" "])[0]?.kind === "unknown",
 );
+
+// ── argument shapes pdf.js versions differ on ────────────────────────────────────────────
+test("isUpright on a short transform; hexOf on a non-array and on non-numbers", () => {
+  assert.deepEqual(
+    [L.isUpright([]), L.hexOf(null), L.hexOf([255.4, "x", undefined])],
+    [true, "#000000", "#ff0000"],
+  );
+});
+
+test("the walk reads odd show-text arguments as no characters, and odd state operators as defaults", () => {
+  const drawn = L.drawnCharacters(
+    OPS,
+    ops(
+      [OPS.restore], // a restore with nothing saved: the default state
+      [OPS.setTextRenderingMode], // a mode with no argument: 0
+      [99], // an operator the walk does not read
+      [OPS.showText, "not an array"],
+      [OPS.showText, ["not glyphs"]],
+      [OPS.showText, [[7, null, { unicode: undefined }, { unicode: "A" }]]],
+    ),
+  );
+  assert.deepEqual(drawn, [{ ch: "A", fill: { kind: "unknown" } }]);
+});
+
+test("an item that matches only up to the end of the drawn text is not found", () => {
+  const list = ops(
+    [OPS.setFillRGBColor, ["#112233"]],
+    [OPS.showText, glyphs("ab")],
+  );
+  assert.deepEqual(L.fillsOf(OPS, list, ["bc", "b"]), [
+    { kind: "unknown" },
+    { kind: "rgb", hex: "#112233" },
+  ]);
+});

package/src/adapters/pdfjs/fill.ts

@@ -81,40 +81,45 @@ function applyState(
   return true;
 }
 
+/** One drawn character and the fill it was drawn in. */
+interface Drawn {
+  readonly ch: string;
+  readonly fill: Fill;
+}
+
 /** Every drawn character, in content order, with the fill it was drawn in. */
-export function drawnCharacters(
-  ops: ColourOps,
-  list: OperatorList,
-): { chars: string[]; fills: Fill[] } {
+export function drawnCharacters(ops: ColourOps, list: OperatorList): Drawn[] {
   const s = { cur: { fill: { kind: "unknown" } as Fill, mode: 0 }, stack: [] };
-  const chars: string[] = [];
-  const fills: Fill[] = [];
+  const drawn: Drawn[] = [];
   list.fnArray.forEach((fn, i) => {
     const args = list.argsArray[i];
     if (applyState(ops, fn, args, s)) return;
     if (fn !== ops.showText && fn !== ops.showSpacedText) return;
     const fill: Fill = s.cur.mode === 3 ? { kind: "invisible" } : s.cur.fill;
-    for (const ch of squeeze(glyphText(args))) {
-      chars.push(ch);
-      fills.push(fill);
-    }
+    for (const ch of squeeze(glyphText(args))) drawn.push({ ch, fill });
   });
-  return { chars, fills };
+  return drawn;
 }
 
 /** How far ahead the walk looks for an item it cannot find where it expects it. */
 export const LOOKAHEAD = 4000;
 
-/** The position at or after `from` where `want` starts in `chars`, or -1 within the look-ahead. */
+/**
+ * Where `want` starts in `drawn`, at or after `from` within the look-ahead, and the fill it was
+ * drawn in — or null. The fill comes from the matched element itself, not a second lookup.
+ */
 function findRun(
-  chars: readonly string[],
+  drawn: readonly Drawn[],
   want: readonly string[],
   from: number,
-): number {
-  const limit = Math.min(chars.length, from + LOOKAHEAD);
-  for (let q = from; q < limit; q++)
-    if (want.every((ch, k) => chars[q + k] === ch)) return q;
-  return -1;
+): { readonly at: number; readonly fill: Fill } | null {
+  const window = drawn.slice(from, from + LOOKAHEAD);
+  for (const [i, d] of window.entries()) {
+    const at = from + i;
+    if (want.every((ch, k) => drawn[at + k]?.ch === ch))
+      return { at, fill: d.fill };
+  }
+  return null;
 }
 
 /**
@@ -128,14 +133,28 @@ export function fillsOf(
   list: OperatorList,
   items: readonly string[],
 ): Fill[] {
-  const { chars, fills } = drawnCharacters(ops, list);
+  return fillsFor(ops, list, items, (text) => text).map((p) => p.fill);
+}
+
+/**
+ * Each item paired with its fill — `fillsOf` for items that are not bare strings, so a caller never
+ * matches the two lists up by index.
+ */
+export function fillsFor<T>(
+  ops: ColourOps,
+  list: OperatorList,
+  items: readonly T[],
+  textOf: (item: T) => string,
+): { readonly item: T; readonly fill: Fill }[] {
+  const drawn = drawnCharacters(ops, list);
   let at = 0;
-  return items.map((text): Fill => {
+  const fillOf = (text: string): Fill => {
     const want = [...squeeze(text)];
     if (want.length === 0) return { kind: "unknown" };
-    const q = findRun(chars, want, at);
-    if (q < 0) return { kind: "unknown" };
-    at = q + want.length;
-    return fills[q] ?? { kind: "unknown" };
-  });
+    const hit = findRun(drawn, want, at);
+    if (hit === null) return { kind: "unknown" };
+    at = hit.at + want.length;
+    return hit.fill;
+  };
+  return items.map((item) => ({ item, fill: fillOf(textOf(item)) }));
 }

package/src/adapters/references/index.test.ts

@@ -0,0 +1,228 @@
+/**
+ * The online references adapter — verify-cites' existence check and bib-authors' author check
+ * over one bibliography, merged per entry — driven with a fake `fetch` that answers every service
+ * by URL. DBLP's gentle-client pauses run under fake timers.
+ */
+import assert from "node:assert/strict";
+import { afterEach, test, vi } from "vitest";
+import { onlineReferences } from "./index.ts";
+// @ts-expect-error — a skill script in .mjs, it has no types
+import * as cites from "../../../skills/verify-citations/scripts/verify-cites.mjs";
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+  vi.useRealTimers();
+});
+
+const json = (status: number, body?: unknown) =>
+  new Response(body === undefined ? null : JSON.stringify(body), { status });
+const dblp = (title: string, authors: string[]) =>
+  json(200, {
+    result: {
+      hits: {
+        hit: [
+          {
+            info: {
+              title,
+              venue: "ICSE",
+              year: "2024",
+              type: "Conference and Workshop Papers",
+              authors: { author: authors.map((text) => ({ text })) },
+            },
+          },
+        ],
+      },
+    },
+  });
+
+/** Route by URL prefix; the longest prefix wins. */
+function fakeFetch(routes: Record<string, () => Response>) {
+  vi.stubGlobal("fetch", async (url: string) => {
+    const hit = Object.keys(routes)
+      .filter((p) => url.startsWith(p))
+      .sort((a, b) => b.length - a.length)[0];
+    if (hit === undefined) return json(404);
+    return routes[hit]!();
+  });
+}
+
+/** Await `p` while fake time runs, so DBLP's pauses between requests elapse at once. */
+async function settle<T>(p: Promise<T>): Promise<T> {
+  let done = false;
+  const out = p.finally(() => (done = true));
+  while (!done) await vi.advanceTimersByTimeAsync(1000);
+  return out;
+}
+
+test("no service answers: not-checked, with the reason", async () => {
+  vi.stubGlobal("fetch", async () => {
+    throw new Error("getaddrinfo ENOTFOUND api.crossref.org");
+  });
+  assert.deepEqual(await onlineReferences("@misc{k, title={T}}"), {
+    kind: "not-checked",
+    why: "the citation services cannot be reached (getaddrinfo ENOTFOUND api.crossref.org)",
+  });
+});
+
+test("each entry gets its existence and its authors: confirmed, fabricated, mismatched, unchecked, skipped", async () => {
+  vi.useFakeTimers();
+  fakeFetch({
+    "https://api.crossref.org/works/10.1%2Fgood": () =>
+      json(200, {
+        message: { title: ["Good Paper"], issued: { "date-parts": [[2024]] } },
+      }),
+    "https://doi.org/api/handles/10.1/good": () =>
+      json(200, { responseCode: 1 }),
+    "https://doi.org/api/handles/10.1/fake": () => json(404),
+    "https://dblp.org/search/publ/api/?q=Good%20Paper": () =>
+      dblp("Good Paper", ["Ada Lovelace"]),
+    "https://dblp.org/search/publ/api/?q=Drift%20Paper": () =>
+      dblp("Drift Paper", ["Ada Lovelace", "Alan Turing"]),
+    "https://dblp.org/search/publ/api/?q=Flaky%20Paper": () => json(500),
+    "https://api.crossref.org/": () => json(200),
+    // Any other title: DBLP knows nothing, so the author check does not apply.
+    "https://dblp.org/": () => json(200, {}),
+  });
+  const bib = [
+    "@inproceedings{good, author={Ada Lovelace}, title={Good Paper}, booktitle={ICSE}, doi={10.1/good}}",
+    "@inproceedings{fake, author={Ada Lovelace}, title={Fake Paper}, booktitle={ICSE}, doi={10.1/fake}}",
+    "@inproceedings{drift, author={Ada Lovelace}, title={Drift Paper}, booktitle={ICSE}}",
+    "@inproceedings{flaky, author={Ada Lovelace}, title={Flaky Paper}, booktitle={ICSE}}",
+    "@misc{pre, author={Ada Lovelace}, title={A Preprint}, journal={arXiv preprint}}",
+  ].join("\n");
+  const r = await settle(onlineReferences(bib));
+  assert.equal(r.kind, "checked");
+  const byKey = Object.fromEntries(
+    (r.kind === "checked" ? r.entries : []).map((e) => [
+      e.key,
+      [e.exists, e.authors, e.why ?? ""],
+    ]),
+  );
+  assert.deepEqual(
+    {
+      good: byKey["good"]!.slice(0, 2),
+      fake: byKey["fake"]!.slice(0, 2),
+      drift: byKey["drift"]!.slice(0, 2),
+      driftWhy: String(byKey["drift"]![2]).split("; ").pop(),
+      flaky: [byKey["flaky"]![1], String(byKey["flaky"]![2]).split("; ").pop()],
+      pre: byKey["pre"]!.slice(0, 2),
+    },
+    {
+      good: ["true", "match"],
+      fake: ["false", "skipped"],
+      drift: ["unresolvable", "mismatch"],
+      driftWhy: "missing turing (DBLP: ICSE 2024)",
+      flaky: ["unchecked", "DBLP lookup failed: DBLP 500"],
+      pre: ["unresolvable", "skipped"],
+    },
+  );
+});
+
+test("an author finding names what is extra and what is out of order; an entry only the author check read is unresolvable", async () => {
+  vi.useFakeTimers();
+  fakeFetch({
+    "https://dblp.org/search/publ/api/?q=Extra%20Paper": () =>
+      dblp("Extra Paper", ["Ada Lovelace"]),
+    "https://dblp.org/search/publ/api/?q=Swapped%20Paper": () =>
+      dblp("Swapped Paper", ["Ada Lovelace", "Alan Turing"]),
+    "https://dblp.org/search/publ/api/?q=Comment%20Paper": () =>
+      dblp("Comment Paper", ["Grace Hopper"]),
+    "https://dblp.org/": () => json(200, {}),
+    "https://api.crossref.org/": () => json(200),
+  });
+  const bib = [
+    "@inproceedings{extra, author={Ada Lovelace and Grace Hopper}, title={Extra Paper}, booktitle={ICSE}}",
+    "@inproceedings{swapped, author={Alan Turing and Ada Lovelace}, title={Swapped Paper}, booktitle={ICSE}}",
+    // verify-cites skips @comment blocks; bib-authors reads them — so only one checker sees this key.
+    "@comment{commented, author={Ada Lovelace}, title={Comment Paper}, booktitle={ICSE}}",
+  ].join("\n");
+  const r = await settle(onlineReferences(bib));
+  const why = Object.fromEntries(
+    (r.kind === "checked" ? r.entries : []).map((e) => [
+      e.key,
+      [e.exists, e.why],
+    ]),
+  );
+  assert.deepEqual(why, {
+    extra: [
+      "unresolvable",
+      "not found by title in any database (no resolvable id to disprove) — could be a legit unindexed/regional/pre-digital work, NOT fabrication; extra hopper (DBLP: ICSE 2024)",
+    ],
+    swapped: [
+      "unresolvable",
+      "not found by title in any database (no resolvable id to disprove) — could be a legit unindexed/regional/pre-digital work, NOT fabrication; order differs (DBLP: ICSE 2024)",
+    ],
+    commented: [
+      "unresolvable",
+      "missing hopper; extra lovelace (DBLP: ICSE 2024)",
+    ],
+  });
+});
+
+// ── #107: the per-citation lookups run concurrently ──────────────────────────────────────
+
+/** A fetch where every request takes `ms` of real time; `down` DOIs fail at the transport. */
+function slowFetch(ms: number, down: readonly string[] = []) {
+  vi.stubGlobal("fetch", async (url: string) => {
+    await new Promise((r) => {
+      setTimeout(r, ms);
+    });
+    if (down.some((d) => url.includes(encodeURIComponent(d))))
+      throw Object.assign(new Error("aborted"), { name: "AbortError" });
+    if (url.startsWith("https://api.crossref.org/works/"))
+      return json(200, {
+        message: { title: ["Paper"], issued: { "date-parts": [[2024]] } },
+      });
+    if (url.startsWith("https://doi.org/api/handles/"))
+      return json(200, { responseCode: 1 });
+    return url === "https://api.crossref.org/" ? json(200) : json(404);
+  });
+}
+/** N preprint-style entries (no venue, so the DBLP pass and its pauses do not apply). */
+const manyBib = (n: number) =>
+  Array.from(
+    { length: n },
+    (_, i) =>
+      `@misc{k${String(i)}, author={Ada Lovelace}, title={Paper}, doi={10.1234/p${String(i)}}}`,
+  ).join("\n");
+
+// The serial baseline alone takes ~4 s, past vitest's 5 s default with the concurrent run added.
+test(
+  "#107: twenty lookups of ~200 ms each finish in far less than twenty times 200 ms, with the serial verdicts",
+  { timeout: 20_000 },
+  async () => {
+    // Five requests per citation (four resolvers and doi.org), 40 ms each: ~200 ms per lookup.
+    slowFetch(40);
+    const bib = manyBib(20);
+    const serial: string[] = [];
+    for (const c of cites.parseBib(bib) as { id: string }[])
+      serial.push(
+        (
+          (await cites.verifyCitationLive(c, { cache: {} })) as {
+            verdict: string;
+          }
+        ).verdict,
+      );
+    const t0 = Date.now();
+    const r = await onlineReferences(bib);
+    const ms = Date.now() - t0;
+    assert.equal(r.kind, "checked");
+    assert.deepEqual(
+      r.kind === "checked" ? r.entries.map((e) => e.exists) : [],
+      serial,
+    );
+    assert.ok(ms < (20 * 200) / 2, `${String(ms)} ms for 20 lookups`);
+  },
+);
+
+test("#107: a lookup whose requests time out is that entry's `unresolvable`, not the others' failure", async () => {
+  slowFetch(5, ["10.1234/p3"]);
+  const r = await onlineReferences(manyBib(8));
+  assert.deepEqual(
+    r.kind === "checked" ? r.entries.map((e) => [e.key, e.exists]) : [],
+    Array.from({ length: 8 }, (_, i) => [
+      `k${String(i)}`,
+      i === 3 ? "unresolvable" : "true",
+    ]),
+  );
+});

package/src/adapters/references/index.ts

@@ -12,7 +12,11 @@ import type {
   CheckReferences,
   EntryVerdict,
 } from "../../ports/check-references.ts";
+import { mapLimit } from "./pool.ts";
 import { unreachable } from "./reach.io.ts";
+
+/** Citations looked up at once (#107). */
+const LOOKUPS_IN_FLIGHT = 6;
 // @ts-expect-error — a skill script in .mjs, it has no types
 import * as cites from "../../../skills/verify-citations/scripts/verify-cites.mjs";
 // @ts-expect-error — a skill script in .mjs, it has no types
@@ -86,13 +90,24 @@ function entryVerdict(
 export const onlineReferences: CheckReferences = async (bib) => {
   const why = await unreachable();
   if (why !== null) return { kind: "not-checked", why };
+  // 🔴 #107: the lookups were one at a time — 27 references took 217 s. They now run
+  // LOOKUPS_IN_FLIGHT at a time, and the DBLP author pass runs beside them (it keeps its own
+  // serial pace and 900 ms pause, which DBLP asks for). Each citation's own requests go to four
+  // different services one after another, so a pool of 6 is at most 6 requests to any one of
+  // them. The cache is shared: two citations with the same identifier may both miss it and ask
+  // twice — an extra request, never a different answer.
   const cache = {};
-  const found: CiteResult[] = [];
-  for (const c of cites.parseBib(bib) as { id?: string }[])
-    if (c.id) found.push(await cites.verifyCitationLive(c, { cache }));
-  const a = (await authors.checkAuthors(
-    authors.parseBib(bib),
-  )) as AuthorBuckets;
+  const citations = (cites.parseBib(bib) as { id?: string }[]).filter(
+    (c) => c.id,
+  );
+  const [found, a] = await Promise.all([
+    mapLimit(
+      citations,
+      LOOKUPS_IN_FLIGHT,
+      (c) => cites.verifyCitationLive(c, { cache }) as Promise<CiteResult>,
+    ),
+    authors.checkAuthors(authors.parseBib(bib)) as Promise<AuthorBuckets>,
+  ]);
   const keys = new Set([
     ...found.map((c) => c.id),
     ...a.findings.map((f) => f.key),

package/src/adapters/references/pool.test.ts

@@ -0,0 +1,31 @@
+/** mapLimit: results in input order, never more than `limit` in flight, an empty list at once. */
+import assert from "node:assert/strict";
+import { test } from "vitest";
+import { mapLimit } from "./pool.ts";
+
+test("results keep the input order although they finish out of order", async () => {
+  const out = await mapLimit([30, 5, 20, 1], 2, async (ms) => {
+    await new Promise((r) => {
+      setTimeout(r, ms);
+    });
+    return ms * 10;
+  });
+  assert.deepEqual(out, [300, 50, 200, 10]);
+});
+
+test("never more than `limit` calls in flight; fewer items than the limit start fewer workers", async () => {
+  let now = 0;
+  let peak = 0;
+  const track = async () => {
+    peak = Math.max(peak, ++now);
+    await new Promise((r) => {
+      setTimeout(r, 2);
+    });
+    now--;
+  };
+  await mapLimit(Array.from({ length: 10 }), 3, track);
+  const three = peak;
+  peak = 0;
+  await mapLimit([1, 2], 6, track);
+  assert.deepEqual([three, peak, await mapLimit([], 6, track)], [3, 2, []]);
+});

package/src/adapters/references/pool.ts

@@ -0,0 +1,24 @@
+/**
+ * `fn` over `items` with at most `limit` calls in flight, results in `items`' order — a promise
+ * pool, the one piece of concurrency this adapter needs, so it is written here rather than
+ * depended on. A rejection rejects the whole map, as `Promise.all` would: callers that must not
+ * lose the other results catch inside `fn`.
+ */
+export async function mapLimit<T, R>(
+  items: readonly T[],
+  limit: number,
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const out: R[] = new Array<R>(items.length);
+  let next = 0;
+  const worker = async (): Promise<void> => {
+    while (next < items.length) {
+      const i = next++;
+      out[i] = await fn(items[i] as T);
+    }
+  };
+  await Promise.all(
+    Array.from({ length: Math.min(limit, items.length) }, worker),
+  );
+  return out;
+}

package/src/build-engine.harness.mjs

@@ -8,7 +8,6 @@
  *   2. prepareEngine: use the cache · use the system · refuse without a human · ask, decline,
  *      accept and install · dry-run never asks or installs.
  */
-import assert from "node:assert/strict";
 import {
   chmodSync,
   mkdirSync,
@@ -20,15 +19,12 @@ import {
 import { tmpdir } from "node:os";
 import { basename, delimiter, dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { createChecker } from "../lib/check.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const E = await import(join(HERE, "build-engine.ts"));
 
-let n = 0;
-const check = (label, cond, detail = "") => {
-  assert.ok(cond, `${label}${detail ? ` — ${detail}` : ""}`);
-  n++;
-};
+const check = createChecker();
 
 // ── 1. texts ────────────────────────────────────────────────────────────────────────────
 const noCache = {
@@ -311,4 +307,4 @@ const x_run = { found: {} };
 }
 
 rmSync(work, { recursive: true, force: true });
-console.log(`build-engine: ${n} checks passed`);
+console.log(`build-engine: ${check.count} checks passed`);