paperlint 7.2.0 → 9.0.0

package/skills/paper-pipeline/scripts/extract-ref-facts.mjs

@@ -3,34 +3,32 @@
  * extract-ref-facts.mjs — parse a paper's bibliography, ask the registries and write FACTS to JSON.
  * It judges nothing.
  *
- * Usage: node extract-ref-facts.mjs <paper dir|paper.md|refs.bib> [--offline] [--refresh]
+ * Usage: node extract-ref-facts.mjs <paper dir|paper.tex|refs.bib> [--offline] [--refresh]
  *                                   [--cache=PATH] [--out=PATH] [--quiet]
- * Exit: 0 — facts written · 1 — nothing to write (no source, no entries).
+ * Exit: 0 — facts written · 1 — nothing to write (no source, a source that is neither a .bib nor
+ * a .tex, no entries).
  *
  * WHY THE SPLIT. Before 2026-08-26 measurement and judgement lived in one script
  * (`verify-refs.mjs`, 20 emit sites, 8 kinds of findings) — that is, on the fifth rung of the
  * ladder in `the consumer's papers CLAUDE.md`. The judgement moved into ESLint rules
  * (`eslint-rules/ref-facts.mjs`): a registry of rules, severity from config, `eslint-disable` with a
- * reason, `file:line:col` positions. What stayed here is the plumbing: parsing markup, parsing
- * `.bib`, the network, parsing the registries' responses.
+ * reason, `file:line:col` positions. What stayed here is the plumbing: parsing `.bib`, the network,
+ * parsing the registries' responses.
  *
  * The precedent in this same knowledge base is `render-paper/extract-pdf-facts.mjs` +
  * `eslint-rules/pdf-facts.mjs`. The boundary is exactly the same: the script goes out into the
  * world, the rule passes the judgement.
  *
- * 🔴 WHY THE RULE CANNOT LIVE OVER THE PAPER ITSELF. Of the three papers in the knowledge base one
- * has a `paper.md` (`<paper-a>`); in `<paper-b>` and `<paper-c>` the bibliography is `refs.bib`, and
- * no ESLint language plugin parses `.bib`. The common input that can do both is the facts JSON.
- * The price is known and written down: a finding is addressed into the facts, not into a line of
- * `paper.md`; that is why every entry in the facts carries the `line` of its source, and the rule
- * prints it in the message.
+ * 🔴 WHY THE RULE CANNOT LIVE OVER THE BIBLIOGRAPHY ITSELF. No ESLint language plugin parses
+ * `.bib`, so the rule's input is the facts JSON. The price is known and written down: a finding is
+ * addressed into the facts, not into a line of the `.bib`; that is why every entry in the facts
+ * carries the `line` of its source, and the rule prints it in the message.
  *
  * 🔴 WHAT THIS MOVE FIXED (both legs were DEAD, measured 2026-08-26):
  *
- *  1. `.bib` WAS NEVER OPENED. `resolveSource` walked `paper.md` → `draft.md` →
- *     `build/custom.bib`; `refs.bib` was not on the list, although the error text promised
- *     "or .bib". The run: `verify-refs.mjs <papers-root>/<paper-b>` →
- *     "no paper.md, draft.md or .bib … nothing to check".
+ *  1. `.bib` WAS NEVER OPENED. `resolveSource` did not list `refs.bib`, although the error text
+ *     promised "or .bib". The run: `verify-refs.mjs <papers-root>/<paper-b>` →
+ *     "… or .bib … nothing to check".
  *  2. THE `.bib` PARSER DID NOT TAKE THE REAL FILES. The regex `/@\w+\{([^,]+),([\s\S]*?)\n\}/g`
  *     required a `}` on a line of its own, while both of our `refs.bib` close an entry on the line
  *     of the last field (`note={arXiv:2310.06770}}`). A run straight at the file: "parsed 0
@@ -62,23 +60,17 @@
  * registry's response, not by a comparison. If the cache kept parsed entries, those readers would
  * stay uncovered while looking covered.
  */
-import {
-  readFileSync,
-  writeFileSync,
-  existsSync,
-  mkdirSync,
-  statSync,
-} from "node:fs";
+import { readFileSync, writeFileSync, existsSync, mkdirSync } from "node:fs";
 import { createHash } from "node:crypto";
 import { join, resolve, dirname, basename } from "node:path";
-import { headings as mdHeadings, requireMarkdown } from "#lib/markdown";
+import { latexReader } from "#src/adapters/latex/index";
+import { nodeFiles } from "#src/adapters/node/index";
+import { sourcesCodec } from "#src/adapters/sources-record/index";
+import { absolutePath } from "#src/domain/paths";
+import { bibReader } from "#src/adapters/bibtex/index";
+import { bibliographyAt, bibliographyUnreadWhy } from "#src/paper-sources";
 import { isMain } from "./consumer.mjs";
 
-// Markup is parsed with a parser (`CLAUDE.md`, 2026-08-11). We fail rather than degrade: without
-// the parser the reference list would not be found at all, and the facts would come out empty —
-// that is, a clean verdict about a paper that in fact has forty references.
-requireMarkdown();
-
 const ROOT = process.env.CLAUDE_PROJECT_DIR || process.cwd();
 const CROSSREF = "https://api.crossref.org/works/";
 const ARXIV = "https://export.arxiv.org/api/query?id_list=";
@@ -89,135 +81,12 @@ const UA =
 
 export const SCHEMA = 1;
 
-// ── parsing: the reference list in markdown ──────────────────────────────────
-//
-// The same shape that `repro/md2submission.py` reads, deliberately: that script turns the list into
-// the .bib actually uploaded to the venue, so whatever it counts as an entry is an entry.
-// `^\d{1,2}. ` opens an entry, an indent continues it, a blank line closes it.
-
-const REF_OPEN = /^(\d{1,2})\.\s+(.*)$/;
-/** Working notes that never reach the PDF; md2submission strips them, so we do too. */
-const BRACKET_NOTE = /\[(VERIFY|ANONYMIZ)[^\]]*\]/g;
-
-export function parseMarkdownRefs(text) {
-  // The section is cut by the parser. The former `text.split(/^## References\s*$/m)[1]` opened the
-  // reference list on a `## References` quoted inside a ```-block — and in this repo papers quote
-  // their own markup in chunks.
-  const refsHs = mdHeadings(text).filter(
-    (h) => h.depth === 2 && /^References\s*$/u.test(h.text),
-  );
-  if (refsHs.length === 0) return [];
-  const nl = text.indexOf("\n", refsHs[0].offset);
-  const body = text.slice(
-    nl === -1 ? text.length : nl,
-    refsHs[1] ? refsHs[1].offset : text.length,
-  );
-  // The line where the body starts in the source file — so that an entry has an ADDRESS in
-  // `paper.md` and not just a number in a list. The rule prints it in the message: the facts file
-  // lives in `_build/`, and without the source line a finding would have to be hunted for by eye.
-  const bodyLine = text
-    .slice(0, nl === -1 ? text.length : nl)
-    .split("\n").length;
-  // A heading closes the list — appendices come AFTER the bibliography, and without this their
-  // numbered prose would be parsed as references. The line numbers come from the parser:
-  // `/^#{1,6}\s/` also counted a hash inside a ```-block as a heading.
-  const headingLines = new Set(mdHeadings(body).map((h) => h.line));
-  const entries = [];
-  let cur = null;
-  let buf = [];
-  let curLine = 0;
-  const flush = () => {
-    if (cur !== null)
-      entries.push({
-        n: cur,
-        line: curLine,
-        raw: buf.join(" ").replace(BRACKET_NOTE, "").trim(),
-      });
-    cur = null;
-    buf = [];
-  };
-  const lns = body.split("\n");
-  for (let i = 0; i < lns.length; i++) {
-    const ln = lns[i];
-    if (headingLines.has(i)) break;
-    const m = REF_OPEN.exec(ln);
-    if (m) {
-      flush();
-      cur = Number(m[1]);
-      curLine = bodyLine + i;
-      buf = [m[2].trim()];
-    } else if (cur !== null && ln.trim()) buf.push(ln.trim());
-    else if (cur !== null) flush();
-  }
-  flush();
-  return entries.map(splitEntry).filter(Boolean);
-}
-
-/** `A, B, C. *Title.* rest` → authors / title / remainder — the shape md2submission produces. */
-function splitEntry(e) {
-  const base = { n: e.n, line: e.line, key: null, raw: e.raw };
-  const mt = /\*(.+?)\*/.exec(e.raw);
-  if (!mt)
-    return {
-      ...base,
-      authors: [],
-      truncated: false,
-      title: null,
-      year: yearIn(e.raw),
-      venue_text: e.raw,
-    };
-  const title = mt[1].trim().replace(/\.$/, "");
-  const rawAuthors = splitAuthors(e.raw.slice(0, mt.index));
-  const note = e.raw
-    .slice(mt.index + mt[0].length)
-    .trim()
-    .replace(/^,\s*/, "");
-  const { authors, truncated } = dropEtAl(rawAuthors);
-  return {
-    ...base,
-    authors,
-    truncated,
-    title,
-    year: yearIn(note) ?? yearIn(e.raw),
-    venue_text: note,
-  };
-}
-
-function splitAuthors(a) {
-  const s = a
-    .trim()
-    .replace(/\.\s*$/, "")
-    .trim();
-  if (!s) return [];
-  return s
-    .split(",")
-    .map((p) => p.trim())
-    .filter(Boolean);
-}
-
-/**
- * `Z. Xiang et al.` — the entry deliberately gives a PREFIX of the author list. That is a fact about
- * the entry, not a judgement, so the truncation is removed here and the rule receives an honest
- * prefix plus a flag.
- */
-const ET_AL = /\bet\s+al\.?$/i;
-function dropEtAl(authors) {
-  if (!authors.length || !ET_AL.test(authors[authors.length - 1].trim()))
-    return { authors, truncated: false };
-  const out = authors.slice();
-  out[out.length - 1] = out[out.length - 1].replace(ET_AL, "").trim();
-  return { authors: out.filter(Boolean), truncated: true };
-}
-
-const yearIn = (s) => (/\b(19|20)\d{2}\b/.exec(s) ?? [null])[0];
-
 // ── parsing: .bib WITH A REAL PARSER ─────────────────────────────────────────
 
 /**
- * The name order is normalised to "First name Surname" — the same shape in which authors are written
- * in markdown. Then the rule has ONE `surname()` function (the last alphabetic token) for both
- * sources instead of two branches. Normalising the format is the parser's job; comparing is the
- * rule's job.
+ * The name order is normalised to "First name Surname". Then the rule has ONE `surname()` function
+ * (the last alphabetic token). Normalising the format is the parser's job; comparing is the rule's
+ * job.
  *
  * 🔴 TWO SHAPES, AND BOTH ARE LISTED EXPLICITLY. The parser returns an institution
  * (`author={{Adversa AI}}` — the double brace means "one name as a whole, do not split") as
@@ -241,47 +110,15 @@ const joinName = (a) =>
     .trim();
 
 /**
- * 🔴 THE .bib PARSER IS AN OPTIONAL DEPENDENCY, AND THE FAILURE MUST BE LOUD AND CARRY THE CURE.
- * It weighs 15 MB out of the consumer's 56 MB install (the package itself 9 MB plus the English
- * model `wink-eng-lite-web-model` 4 MB and `unicode2latex` 2 MB) — 27% of the weight for a single
- * call that only whoever extracts bibliography facts needs. The import here is dynamic anyway, so
- * the laziness was already there; the manifest merely stopped lying about it being required.
- *
- * ⚠️ A silent skip here would be the worst of the options: a missing checker and a passing one look
- * the same, and "the bibliography was not checked" reads as "the bibliography is fine". That is why
- * the message names the command, not the fact.
+ * The facts' entries of one database, as the bibtex reader read them (src/ports/bib-reader.ts): each
+ * with its line in the file that holds it, so a finding points where the author edits (the block's
+ * lines in paper.tex, or the `.bib`'s).
  */
-export async function parseBib(
-  text,
-  { load = () => import("@retorquere/bibtex-parser") } = {},
-) {
-  let parse;
-  try {
-    ({ parse } = await load());
-  } catch (e) {
-    if (e?.code !== "ERR_MODULE_NOT_FOUND") throw e;
-    throw new Error(
-      "parsing .bib requires @retorquere/bibtex-parser — it is declared OPTIONAL because it " +
-        "weighs 15 MB and is needed only for bibliography facts.\n" +
-        "   Install:  npm i -D @retorquere/bibtex-parser\n" +
-        "   Why not our own regex: measured 26.08 — the regex gave 0 entries on both real " +
-        "files, four libraries gave the correct 27 and 51.",
-    );
-  }
-  // `sentenceCase: false` — the title is needed as written. Our normalisation folds the case
-  // anyway, but a fact has to be a fact: a paraphrased title cannot be shown to a human.
-  const res = parse(text, { sentenceCase: false, verbatimFields: [] });
-  // The entry's line in the file — found by its key. The parser gives no positions, and a finding
-  // needs an address; a key in `.bib` is unique by definition of the format, so the search is
-  // unambiguous.
-  const lines = text.split("\n");
-  const lineOfKey = (key) => {
-    const i = lines.findIndex((l) => l.includes(`{${key},`));
-    return i === -1 ? 0 : i + 1;
-  };
-  return res.entries.map((e, i) => {
-    const f = e.fields ?? {};
-    const names = Array.isArray(f.author) ? f.author : [];
+export function factEntries(bib) {
+  const lineOf = (at) => bib.text.slice(0, at).split("\n").length;
+  return bib.entries.map((e, i) => {
+    const f = e.fields;
+    const names = e.names.author ?? [];
     // `and others` is BibTeX's `et al.`. The parser returns it as an author with no first name.
     const isOthers = (a) => !a.firstName && /^others$/i.test(a.lastName ?? "");
     const truncated = names.some(isOthers);
@@ -289,29 +126,20 @@ export async function parseBib(
       .filter((a) => !isOthers(a))
       .map(joinName)
       .filter(Boolean);
-    // A field is a string in today's parser output (list fields such as `keywords` are arrays,
-    // and none of those is read); an array or a number from another parser version is joined or
-    // printed rather than dropped.
-    const str = (v) =>
-      Array.isArray(v)
-        ? v.join(" ")
-        : typeof v === "string"
-          ? v
-          : v == null
-            ? ""
-            : String(v);
+    // Every field the reader gives is text; a field the entry lacks is "".
+    const str = (v) => v ?? "";
     const venueText = [f.booktitle, f.journal, f.note, f.howpublished]
       .map(str)
       .filter(Boolean)
       .join(" ");
     return {
       n: i + 1,
-      line: lineOfKey(e.key),
+      line: lineOf(e.span.start),
       key: e.key,
       // `raw` is where identifiers are looked for: they hide in `note`, in `journal`
       // (`arXiv preprint arXiv:2107.03374`), and in a separate `doi` field.
       raw: [
-        str(f.author && authors.join(" and ")),
+        authors.join(" and "),
         str(f.title),
         venueText,
         str(f.doi),
@@ -527,27 +355,40 @@ export function recordFrom(key, cached) {
 
 // ── assembling the facts ─────────────────────────────────────────────────────
 
-/** `paper.md` → `draft.md` → `refs.bib` → `build/custom.bib`. `refs.bib` ADDED 26.08 (defect #1). */
-export const SOURCE_ORDER = [
-  "paper.md",
-  "draft.md",
-  "refs.bib",
-  "build/custom.bib",
-];
+/**
+ * WHICH BIBLIOGRAPHY: the one the last build's bibtex opened, as `paperlint build` recorded it
+ * (`_build/sources.json`, docs/design/paper-sources.md §1) and `bibliographyAt` reads it
+ * (src/paper-sources.ts) — never an order of file names of our own, and never read out of TeX source.
+ *
+ *   a directory   the databases bibtex opened for its paper.tex
+ *   a .tex        the same, for that file's folder
+ *   a .bib        that file, as named
+ *
+ * A paper with no current record is refused: run `npx paperlint build` first. A database TeX wrote from
+ * a `filecontents` block is read from the block, where the author edits it.
+ */
+const DEPS = {
+  files: nodeFiles,
+  codec: sourcesCodec,
+  latex: latexReader,
+  bib: bibReader,
+};
 
-export function resolveSource(target) {
-  const t = resolve(target);
-  if (existsSync(t) && statSync(t).isFile()) return t;
-  for (const c of SOURCE_ORDER) {
-    const p = join(t, c);
-    if (existsSync(p)) return p;
-  }
-  return null;
+/**
+ * The bibliography of `target`: the texts to read, or why there are none — `bibliographyAt`
+ * (src/paper-sources.ts) answers, in its words. `paperDir` is where the cache and the facts go.
+ */
+export function bibliographyFrom(target) {
+  const r = bibliographyAt(absolutePath(resolve(target)), DEPS);
+  if (!r.ok) return { ok: false, why: bibliographyUnreadWhy(r.error) };
+  const { paperDir, texts } = r.value;
+  return { ok: true, paperDir, texts };
 }
 
-export async function loadEntries(path) {
-  const text = readFileSync(path, "utf8");
-  return path.endsWith(".bib") ? parseBib(text) : parseMarkdownRefs(text);
+/** The entries of one text TeX reads, each with its file. */
+export function loadEntries(bib) {
+  const file = rel(bib.path);
+  return factEntries(bib).map((e) => ({ ...e, file }));
 }
 
 /** Every registry key a list of entries needs. Computed once so the requests go in one batch. */
@@ -559,7 +400,11 @@ export function keysFor(entries) {
 
 const rel = (p) => (p.startsWith(ROOT) ? p.slice(ROOT.length + 1) : p);
 
-export function buildFacts({ source, text, entries, cache, cachePath }) {
+/**
+ * The facts of a run. `source` is the file the first database was read from (the only one, for a
+ * paper with one), `sources` every one; `source_sha256` hashes those files' bytes, in order.
+ */
+export function buildFacts({ texts, entries, cache, cachePath }) {
   const withIds = entries.map((e) => {
     const ids = extractIds(e);
     return { ...e, ids, primary: primaryOf(ids), keys: allKeys(ids) };
@@ -570,9 +415,12 @@ export function buildFacts({ source, text, entries, cache, cachePath }) {
       if (!(k in records)) records[k] = recordFrom(k, cache[k]);
   return {
     schema: SCHEMA,
-    source: rel(source),
-    source_kind: source.endsWith(".bib") ? "bibtex" : "markdown",
-    source_sha256: createHash("sha256").update(text).digest("hex"),
+    source: rel(texts[0].path),
+    sources: [...new Set(texts.map((b) => rel(b.path)))],
+    source_kind: "bibtex",
+    source_sha256: [...new Set(texts.map((b) => b.text))]
+      .reduce((h, t) => h.update(t), createHash("sha256"))
+      .digest("hex"),
     cache: cachePath ? rel(cachePath) : null,
     generated: new Date().toISOString().slice(0, 10),
     entries: withIds,
@@ -594,16 +442,12 @@ async function main(argv) {
       .slice(1)
       .join("=");
 
-  const src = resolveSource(target);
-  if (!src) {
-    console.error(
-      `🛑 no ${SOURCE_ORDER.join(", ")} under ${resolve(target)} — nowhere to take a bibliography from.`,
-    );
+  const found = bibliographyFrom(target);
+  if (!found.ok) {
+    console.error(`🛑 ${found.why}.`);
     return 1;
   }
-  const paperDir = statSync(resolve(target)).isFile()
-    ? dirname(src)
-    : resolve(target);
+  const { paperDir, texts } = found;
   const cachePath = opt("cache")
     ? resolve(opt("cache"))
     : join(paperDir, "repro", "refs-cache.json");
@@ -611,13 +455,13 @@ async function main(argv) {
     ? resolve(opt("out"))
     : join(paperDir, "_build", "refs.facts.json");
 
-  const text = readFileSync(src, "utf8");
-  const entries = await loadEntries(src);
+  const entries = texts.flatMap(loadEntries);
+  const names = [...new Set(texts.map((b) => basename(b.path)))].join(", ");
   if (!entries.length) {
     // 🔴 Zero entries is a suspect, not a success. Facts with an empty list would look like a
     // clean bibliography, so we do not write them at all and exit with a non-zero code.
     console.error(
-      `🛑 parsed 0 entries out of ${rel(src)}. Silence here would look like a clean bibliography.`,
+      `🛑 parsed 0 entries out of ${[...new Set(texts.map((b) => rel(b.path)))].join(", ")}. Silence here would look like a clean bibliography.`,
     );
     return 1;
   }
@@ -644,12 +488,12 @@ async function main(argv) {
     if (fetched) saveCache(cachePath, cache);
   }
 
-  const facts = buildFacts({ source: src, text, entries, cache, cachePath });
+  const facts = buildFacts({ texts, entries, cache, cachePath });
   mkdirSync(dirname(out), { recursive: true });
   writeFileSync(out, `${JSON.stringify(facts, null, 2)}\n`);
   const resolved = Object.values(facts.records).filter((r) => r.found).length;
   console.log(
-    `📚 ${basename(src)} → ${rel(out)} (${entries.length} entries, ${Object.keys(facts.records).length} identifiers, ` +
+    `📚 ${names} → ${rel(out)} (${entries.length} entries, ${Object.keys(facts.records).length} identifiers, ` +
       `${resolved} resolved${offline ? ", offline" : fetched ? `, +${fetched} fetched` : ", all from cache"})`,
   );
   return 0;