paperlint 3.1.0 → 3.1.2

package/skills/verify-citations/scripts/verify-cites.test.mjs

@@ -15,7 +15,6 @@ import { test } from "vitest";
 import {
   levenshtein,
   titleSimilarity,
-  titlesMatch,
   yearMatch,
   titleRelation,
   bestTitleRelation,
@@ -30,6 +29,9 @@ import {
   normalizeDoi,
   normalizeArxiv,
   normalizeIdentifiers,
+  normalizeTitle,
+  titleIdentity,
+  isValidCveId,
 } from "./verify-cites.mjs";
 
 /** One vitest case per check, named by it. */
@@ -52,17 +54,17 @@ ok(
   "case/normalize identical titles",
 );
 ok(
-  titlesMatch(
+  titleSimilarity(
     "Deep Residual Learning for Image Recognition",
     "Deep Residual Learning for Image Recognitionn",
-  ),
+  ) >= 0.7,
   "one typo still ≥0.70",
 );
 ok(
-  !titlesMatch(
+  titleSimilarity(
     "Attention Is All You Need",
     "A Survey of Reinforcement Learning",
-  ),
+  ) < 0.7,
   "unrelated titles < 0.70",
 );
 ok(yearMatch(2020, 2021), "year ±1 ok");
@@ -733,3 +735,147 @@ console.log("\n[S4] .bib parser edge cases:");
     "bibitem DOI trailing period not swallowed",
   );
 }
+
+// ── Edges of the pure layer: the defensive arms, each with the input that reaches it ──────────
+
+test("levenshtein, normalizeTitle and titleIdentity take a missing string as empty", () => {
+  assert.deepEqual(
+    [
+      levenshtein(null, "ab"),
+      levenshtein("abc", undefined),
+      levenshtein("", ""),
+      normalizeTitle(undefined),
+      titleIdentity(undefined),
+    ],
+    [2, 3, 0, "", ""],
+  );
+});
+
+test("normalizeDoi / normalizeArxiv pass a missing value through untouched", () => {
+  assert.deepEqual(
+    [
+      normalizeDoi(undefined),
+      normalizeDoi(""),
+      normalizeArxiv(null),
+      normalizeArxiv(""),
+    ],
+    [undefined, "", null, ""],
+  );
+});
+
+test("titleRelation: an acronym of one letter is not an acronym; scripts are told apart", () => {
+  assert.deepEqual(
+    [
+      // "x" is one letter: it cannot spell the initials of anything.
+      titleRelation("x", "extensible markup language"),
+      titleRelation("Нейронные сети", "Neural networks"),
+      titleRelation("ニューラルネットワーク", "Neural networks"),
+      titleRelation("Νευρωνικά δίκτυα", "Neural networks"),
+      titleRelation("الشبكات العصبية", "Neural networks"),
+      titleRelation("!!!", "Neural networks"),
+    ],
+    [
+      "different",
+      "incomparable",
+      "incomparable",
+      "incomparable",
+      "incomparable",
+      "incomparable",
+    ],
+  );
+});
+
+test("yearMatch reads a year out of a string and ignores one that is not a year", () => {
+  assert.deepEqual(
+    [
+      yearMatch("2017-06", 2018),
+      yearMatch("2017", 2020),
+      yearMatch("n.d.", 2020),
+    ],
+    [true, false, true],
+  );
+});
+
+test("classifyResolver: a title search with no `records` field is a miss", () => {
+  assert.deepEqual(
+    classifyResolver(
+      { title: "A Paper" },
+      { db: "crossref", transport: "ok", query: "title" },
+    ),
+    { db: "crossref", status: "title_miss" },
+  );
+});
+
+test("checkNvd with no CVE on the citation says nothing; isValidCveId of nothing is false", () => {
+  assert.deepEqual(
+    [checkNvd({}, null), isValidCveId(undefined)],
+    [null, false],
+  );
+});
+
+test("reduceVerdict: an arXiv id missed only by a non-authoritative source is unresolvable", () => {
+  const r = reduceVerdict({ id: "a", arxiv: "1706.03762" }, [
+    {
+      db: "semantic_scholar",
+      status: "id_unmatched",
+      query: "arxiv",
+      authoritative: false,
+    },
+  ]);
+  assert.deepEqual(r, {
+    id: "a",
+    verdict: "unresolvable",
+    reason:
+      "arXiv id 1706.03762 missed only by non-authoritative sources (the reachable arXiv API did not definitively reject it) — cannot assert fabrication → unresolvable",
+    matched_db: null,
+  });
+});
+
+test("parseBib: @comment/@string/@preamble are skipped, a bare value is read, an empty one is absent", () => {
+  const bib = [
+    "@comment{c, not an entry}",
+    "@string{s, x = {y}}",
+    '@article{k1, title = "Quoted Title", year = 2017, eprint = {1706.03762}, archiveprefix = {arXiv}}',
+    "@misc{k2, title = {T}, year = ,}",
+  ].join("\n");
+  assert.deepEqual(parseBib(bib), [
+    { id: "k1", title: "Quoted Title", year: "2017", arxiv: "1706.03762" },
+    { id: "k2", title: "T" },
+  ]);
+});
+
+test("parseBib's \\bibitem fallback reads an arXiv id and an \\emph title", () => {
+  const text = [
+    "\\begin{thebibliography}{9}",
+    "\\bibitem{a} A. Author. \\emph{A Long Enough Title}. arXiv:1706.03762, 2017.",
+    "\\end{thebibliography}",
+  ].join("\n");
+  assert.deepEqual(parseBib(text), [
+    {
+      id: "a",
+      arxiv: "1706.03762",
+      year: "2017",
+      title: "A Long Enough Title",
+    },
+  ]);
+});
+
+test("parseArxivFeed: an entry without a title is skipped, one without a date has no year", () => {
+  assert.deepEqual(
+    parseArxivFeed("<entry><id>x</id></entry><entry><title>T</title></entry>"),
+    [{ title: "T", year: null }],
+  );
+});
+
+test("a two-letter token ending in s is depluralised to one letter, which spells no acronym", () => {
+  assert.equal(
+    titleRelation("TS models", "totally separate models"),
+    "different",
+  );
+});
+
+test("parseArxivFeed keeps an entry titled Error when it has no <id> to prove it is the sentinel", () => {
+  assert.deepEqual(parseArxivFeed("<entry><title>Error</title></entry>"), [
+    { title: "Error", year: null },
+  ]);
+});

package/src/adapters/banal/failure.test.ts

@@ -4,7 +4,12 @@
  */
 import assert from "node:assert/strict";
 import { test } from "vitest";
-import { describe, describeLine, PERL_MISSING } from "./failure.ts";
+import {
+  type BanalFailure,
+  describe,
+  describeLine,
+  PERL_MISSING,
+} from "./failure.ts";
 import { acceptProbe } from "./probe.ts";
 
 test("describe: perl missing carries the fix", () => {
@@ -61,3 +66,67 @@ test("acceptProbe: one page and a body size is accepted; anything else is `probe
   assert.equal(!r.ok && r.error.kind, "probe-rejected");
   assert.equal(acceptProbe({ pages: [], bodyfontsize: 10 }).ok, false);
 });
+
+test("describe: every other kind is one exact sentence", () => {
+  const cases: readonly [BanalFailure, string][] = [
+    [
+      { kind: "process-failed", status: 2, stderrHead: "boom" },
+      "banal failed (exit 2): boom",
+    ],
+    [
+      { kind: "signalled", signal: "SIGKILL", stderrHead: "" },
+      "banal failed (SIGKILL): no output",
+    ],
+    [
+      { kind: "signalled", signal: "SIGTERM", stderrHead: "late" },
+      "banal failed (SIGTERM): late",
+    ],
+    [{ kind: "spawn-failed", message: "EACCES" }, "banal failed: EACCES"],
+    [
+      { kind: "timed-out", afterMs: 500 },
+      "banal failed: no answer after 500 ms",
+    ],
+    [{ kind: "no-json", head: "" }, "banal printed no JSON: nothing"],
+    [{ kind: "no-json", head: "Usage:" }, "banal printed no JSON: Usage:"],
+    [
+      { kind: "banal-error", stderrHead: "bad xml" },
+      "banal could not read the banal input XML: bad xml",
+    ],
+    [
+      {
+        kind: "unexpected-shape",
+        issues: ["pages missing", "bodyfontsize not a number"],
+      },
+      "banal printed JSON that is not a measurement: pages missing; bodyfontsize not a number",
+    ],
+    [
+      { kind: "probe-rejected", got: { pages: [] } },
+      'banal ran on a one-page probe but measured nothing: {"pages":[]}',
+    ],
+    [
+      { kind: "download-failed", url: "https://x/banal", detail: "HTTP 404" },
+      "could not download banal from https://x/banal: HTTP 404",
+    ],
+    [
+      { kind: "not-pinned", path: "/c/banal" },
+      "banal in /c/banal is not the pinned one (sha256 differs)",
+    ],
+    [
+      {
+        kind: "does-not-run",
+        path: "/c/banal",
+        why: { kind: "timed-out", afterMs: 9 },
+      },
+      "banal in /c/banal does not run: banal failed: no answer after 9 ms",
+    ],
+  ];
+  for (const [failure, sentence] of cases)
+    assert.deepEqual(describe(failure), [sentence]);
+});
+
+test("describeLine: a sha256 mismatch joins its three lines", () => {
+  assert.equal(
+    describeLine({ kind: "sha-mismatch", url: "u", expected: "e", got: "g" }),
+    "banal from u does not have the pinned sha256 — refusing to install it; expected e; got      g",
+  );
+});

package/src/adapters/banal/index.test.ts

@@ -103,6 +103,22 @@ test("installer: ensure reports the download, and Ready says where and what was
   assert.equal(i.check().ok, true);
 });
 
+test("installer with no source: the pin is downloaded, and bytes that are not the pin are refused", () => {
+  const download = fixedDownload(BODY);
+  const io = memoryPorts({
+    run: scriptedProcess(() => exitedWith("")),
+    download,
+  });
+  const r = banalInstaller(io, parseBanalSettings({}, dirs)).ensure(() => {});
+  assert.ok(!r.ok);
+  assert.match(
+    r.error[0],
+    /does not have the pinned sha256 — refusing to install it/,
+  );
+  assert.equal(download.urls.length, 1);
+  assert.notEqual(download.urls[0], source.url);
+});
+
 test("installer: a failure is lines of text, the first saying what happened", () => {
   const io = memoryPorts({
     run: scriptedProcess(() => ({ kind: "not-found", file: "perl" })),

package/src/adapters/banal/output.test.ts

@@ -60,6 +60,11 @@ const ROWS: readonly (readonly [
     exited({ stdout: "Usage: banal" }),
     { kind: "no-json", head: "Usage: banal" },
   ],
+  [
+    "nothing on stdout: the head is stderr's first line",
+    exited({ stdout: "", stderr: "\nCan't locate XML/Parser.pm\n" }),
+    { kind: "no-json", head: "Can't locate XML/Parser.pm" },
+  ],
   [
     // Guards: banal's own failure object exits 0 — it must not read as zero pages.
     'banal\'s `"error": true` with exit 0',

package/src/adapters/curl/download.test.ts

@@ -16,7 +16,8 @@ import { join } from "node:path";
 import { pathToFileURL } from "node:url";
 import { afterAll as after, test } from "vitest";
 import type { AbsolutePath } from "../../domain/paths.ts";
-import { curlDownload } from "./download.io.ts";
+import { curlDownload, curlFailure } from "./download.io.ts";
+import type { ProcessExit } from "../../ports/process.ts";
 import { spawnProcess } from "../node/index.ts";
 
 const root = realpathSync(
@@ -55,3 +56,44 @@ test("🔴 Download: no curl on PATH is named as that, not as a failed URL", ()
   // Guards: a missing curl is its own diagnosis — once, it read "spawnSync curl ENOENT".
   assert.deepEqual(!r.ok && r.error, { detail: "curl is not installed" });
 });
+
+test("curlFailure: every way curl can end, in its own words", () => {
+  const cases: readonly [ProcessExit, string | null][] = [
+    [{ kind: "exited", status: 0, stdout: "", stderr: "" }, null],
+    [{ kind: "spawn-failed", message: "EACCES" }, "EACCES"],
+    [
+      { kind: "timed-out", afterMs: 40_000, stdout: "", stderr: "" },
+      "no answer after 40000 ms",
+    ],
+    [{ kind: "exited", status: 22, stdout: "", stderr: "" }, "curl exited 22"],
+    [
+      {
+        kind: "exited",
+        status: 22,
+        stdout: "",
+        stderr: "curl: (22) 404\nmore",
+      },
+      "curl: (22) 404",
+    ],
+    [
+      { kind: "signalled", signal: "SIGTERM", stdout: "", stderr: "" },
+      "SIGTERM",
+    ],
+  ];
+  for (const [exit, words] of cases) assert.equal(curlFailure(exit), words);
+});
+
+test("Download: curl exits 0 and writes nothing — a failure that says so, not empty bytes", () => {
+  const tmp = at("dl-silent");
+  mkdirSync(tmp, { recursive: true });
+  const d = curlDownload({
+    run: { run: () => ({ kind: "exited", status: 0, stdout: "", stderr: "" }) },
+    env: {},
+    tmpDir: tmp,
+  });
+  assert.deepEqual(d.fetch("https://example.test/banal", 1_000), {
+    ok: false,
+    error: { detail: "curl wrote no file" },
+  });
+  assert.deepEqual(readdirSync(tmp), []);
+});

package/src/adapters/memory/index.test.ts

@@ -0,0 +1,25 @@
+/** The in-memory test doubles keep the contracts of the ports they stand in for. */
+import assert from "node:assert/strict";
+import { test } from "vitest";
+import { fixedDownload, memoryWorkspace } from "./index.ts";
+
+test("memoryWorkspace: a `within` that throws is recorded as `threw`, and the error goes on", () => {
+  const ws = memoryWorkspace();
+  assert.throws(
+    () =>
+      ws.within("x-", (s) => {
+        s.write("f", "body", "read");
+        throw new Error("inside");
+      }),
+    /inside/,
+  );
+  assert.deepEqual(ws.ended, ["threw"]);
+  assert.deepEqual(ws.written, [{ name: "f", content: "body", mode: "read" }]);
+});
+
+test("fixedDownload: bytes are handed back as they are, and every URL is recorded", () => {
+  const bytes = new Uint8Array([1, 2, 3]);
+  const d = fixedDownload(bytes);
+  assert.deepEqual(d.fetch("https://a", 1), { ok: true, value: bytes });
+  assert.deepEqual(d.urls, ["https://a"]);
+});

package/src/adapters/node/files.io.ts

@@ -11,7 +11,8 @@ import { dirname } from "node:path";
 import type { AbsolutePath } from "../../domain/paths.ts";
 import type { Files } from "../../ports/files.ts";
 
-const code = (e: unknown): unknown =>
+/** An error's errno code; anything thrown that is not an Error has none. */
+export const errorCode = (e: unknown): unknown =>
   e instanceof Error ? (e as NodeJS.ErrnoException).code : undefined;
 
 export const nodeFiles: Files = {
@@ -22,7 +23,7 @@ export const nodeFiles: Files = {
     try {
       return readFileSync(p);
     } catch (e) {
-      if (code(e) === "ENOENT") return null;
+      if (errorCode(e) === "ENOENT") return null;
       throw e;
     }
   },

package/src/adapters/node/files.test.ts

@@ -5,7 +5,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll as after, test } from "vitest";
 import type { AbsolutePath } from "../../domain/paths.ts";
-import { nodeFiles } from "./files.io.ts";
+import { errorCode, nodeFiles } from "./files.io.ts";
 
 const root = realpathSync(mkdtempSync(join(tmpdir(), "paperlint-files-test-")));
 after(() => rmSync(root, { recursive: true, force: true }));
@@ -26,3 +26,15 @@ test("Files: writeAtomic creates the directory and leaves no .part behind", () =
   );
   assert.equal(existsSync(`${p}.part`), false);
 });
+
+test("Files: a read that fails for another reason than absence throws — a directory is not an empty file", () => {
+  assert.throws(() => nodeFiles.readBytes(at()), { code: "EISDIR" });
+});
+
+test("errorCode: an Error's errno code; a thrown non-Error has none", () => {
+  assert.equal(
+    errorCode(Object.assign(new Error("x"), { code: "ENOENT" })),
+    "ENOENT",
+  );
+  assert.equal(errorCode("ENOENT"), undefined);
+});

package/src/adapters/node/process.test.ts

@@ -5,6 +5,7 @@
  */
 import assert from "node:assert/strict";
 import { test } from "vitest";
+import type { AbsolutePath } from "../../domain/paths.ts";
 import type { Command } from "../../ports/process.ts";
 import { spawnProcess } from "./process.io.ts";
 
@@ -44,3 +45,40 @@ test("the environment is the Command's, whole: nothing leaks in from the parent"
   // Guards: `Command.env` is the child's entire environment — the adapter merges nothing.
   assert.deepEqual(r.kind === "exited" && r.stdout, "[][1]\n");
 });
+
+test("a program killed by a signal is `signalled`, with what it printed first", () => {
+  assert.deepEqual(run.run(sh("echo before; kill -TERM $$")), {
+    kind: "signalled",
+    signal: "SIGTERM",
+    stdout: "before\n",
+    stderr: "",
+  });
+});
+
+test("output past maxOutputBytes is a spawn failure, named by Node's own message", () => {
+  const r = run.run({ ...sh("head -c 100000 /dev/zero"), maxOutputBytes: 10 });
+  assert.equal(r.kind, "spawn-failed");
+  assert.match(r.kind === "spawn-failed" ? r.message : "", /ENOBUFS/);
+});
+
+test("cwd is the Command's when it names one", () => {
+  const r = run.run({ ...sh("pwd"), cwd: "/" as AbsolutePath });
+  assert.deepEqual(r.kind === "exited" && r.stdout, "/\n");
+});
+
+test("a spawn that reports no status, no signal and no error reads as exit 1, not success", () => {
+  const silent = spawnProcess((() => ({
+    pid: 0,
+    output: [],
+    stdout: "",
+    stderr: "",
+    status: null,
+    signal: null,
+  })) as never);
+  assert.deepEqual(silent.run(sh("true")), {
+    kind: "exited",
+    status: 1,
+    stdout: "",
+    stderr: "",
+  });
+});

package/src/adapters/pdfjs/fill.test.ts

@@ -109,3 +109,37 @@ check(
   "fillsOf: an empty item is `unknown`",
   L.fillsOf(OPS, ops(), [" "])[0]?.kind === "unknown",
 );
+
+// ── argument shapes pdf.js versions differ on ────────────────────────────────────────────
+test("isUpright on a short transform; hexOf on a non-array and on non-numbers", () => {
+  assert.deepEqual(
+    [L.isUpright([]), L.hexOf(null), L.hexOf([255.4, "x", undefined])],
+    [true, "#000000", "#ff0000"],
+  );
+});
+
+test("the walk reads odd show-text arguments as no characters, and odd state operators as defaults", () => {
+  const drawn = L.drawnCharacters(
+    OPS,
+    ops(
+      [OPS.restore], // a restore with nothing saved: the default state
+      [OPS.setTextRenderingMode], // a mode with no argument: 0
+      [99], // an operator the walk does not read
+      [OPS.showText, "not an array"],
+      [OPS.showText, ["not glyphs"]],
+      [OPS.showText, [[7, null, { unicode: undefined }, { unicode: "A" }]]],
+    ),
+  );
+  assert.deepEqual(drawn, [{ ch: "A", fill: { kind: "unknown" } }]);
+});
+
+test("an item that matches only up to the end of the drawn text is not found", () => {
+  const list = ops(
+    [OPS.setFillRGBColor, ["#112233"]],
+    [OPS.showText, glyphs("ab")],
+  );
+  assert.deepEqual(L.fillsOf(OPS, list, ["bc", "b"]), [
+    { kind: "unknown" },
+    { kind: "rgb", hex: "#112233" },
+  ]);
+});

package/src/adapters/pdfjs/fill.ts

@@ -81,40 +81,45 @@ function applyState(
   return true;
 }
 
+/** One drawn character and the fill it was drawn in. */
+interface Drawn {
+  readonly ch: string;
+  readonly fill: Fill;
+}
+
 /** Every drawn character, in content order, with the fill it was drawn in. */
-export function drawnCharacters(
-  ops: ColourOps,
-  list: OperatorList,
-): { chars: string[]; fills: Fill[] } {
+export function drawnCharacters(ops: ColourOps, list: OperatorList): Drawn[] {
   const s = { cur: { fill: { kind: "unknown" } as Fill, mode: 0 }, stack: [] };
-  const chars: string[] = [];
-  const fills: Fill[] = [];
+  const drawn: Drawn[] = [];
   list.fnArray.forEach((fn, i) => {
     const args = list.argsArray[i];
     if (applyState(ops, fn, args, s)) return;
     if (fn !== ops.showText && fn !== ops.showSpacedText) return;
     const fill: Fill = s.cur.mode === 3 ? { kind: "invisible" } : s.cur.fill;
-    for (const ch of squeeze(glyphText(args))) {
-      chars.push(ch);
-      fills.push(fill);
-    }
+    for (const ch of squeeze(glyphText(args))) drawn.push({ ch, fill });
   });
-  return { chars, fills };
+  return drawn;
 }
 
 /** How far ahead the walk looks for an item it cannot find where it expects it. */
 export const LOOKAHEAD = 4000;
 
-/** The position at or after `from` where `want` starts in `chars`, or -1 within the look-ahead. */
+/**
+ * Where `want` starts in `drawn`, at or after `from` within the look-ahead, and the fill it was
+ * drawn in — or null. The fill comes from the matched element itself, not a second lookup.
+ */
 function findRun(
-  chars: readonly string[],
+  drawn: readonly Drawn[],
   want: readonly string[],
   from: number,
-): number {
-  const limit = Math.min(chars.length, from + LOOKAHEAD);
-  for (let q = from; q < limit; q++)
-    if (want.every((ch, k) => chars[q + k] === ch)) return q;
-  return -1;
+): { readonly at: number; readonly fill: Fill } | null {
+  const window = drawn.slice(from, from + LOOKAHEAD);
+  for (const [i, d] of window.entries()) {
+    const at = from + i;
+    if (want.every((ch, k) => drawn[at + k]?.ch === ch))
+      return { at, fill: d.fill };
+  }
+  return null;
 }
 
 /**
@@ -128,14 +133,28 @@ export function fillsOf(
   list: OperatorList,
   items: readonly string[],
 ): Fill[] {
-  const { chars, fills } = drawnCharacters(ops, list);
+  return fillsFor(ops, list, items, (text) => text).map((p) => p.fill);
+}
+
+/**
+ * Each item paired with its fill — `fillsOf` for items that are not bare strings, so a caller never
+ * matches the two lists up by index.
+ */
+export function fillsFor<T>(
+  ops: ColourOps,
+  list: OperatorList,
+  items: readonly T[],
+  textOf: (item: T) => string,
+): { readonly item: T; readonly fill: Fill }[] {
+  const drawn = drawnCharacters(ops, list);
   let at = 0;
-  return items.map((text): Fill => {
+  const fillOf = (text: string): Fill => {
     const want = [...squeeze(text)];
     if (want.length === 0) return { kind: "unknown" };
-    const q = findRun(chars, want, at);
-    if (q < 0) return { kind: "unknown" };
-    at = q + want.length;
-    return fills[q] ?? { kind: "unknown" };
-  });
+    const hit = findRun(drawn, want, at);
+    if (hit === null) return { kind: "unknown" };
+    at = hit.at + want.length;
+    return hit.fill;
+  };
+  return items.map((item) => ({ item, fill: fillOf(textOf(item)) }));
 }

package/src/adapters/references/clock.io.ts

@@ -0,0 +1,8 @@
+/** The clock, for the lookup cache: the day an answer was fetched, in UTC. */
+export const todayUtc = (): string => new Date().toISOString().slice(0, 10);
+
+/** Wait `ms` — DBLP's pace between requests. */
+export const sleep = (ms: number): Promise<void> =>
+  new Promise((r) => {
+    setTimeout(r, ms);
+  });