paperlint 6.0.0 → 6.1.0

package/src/adapters/latex/prose.ts

@@ -14,10 +14,18 @@
  * own passage; one that cites or links leaves a citation mark in the sentence it hangs from.
  */
 import type * as Ast from "@unified-latex/unified-latex-types";
-import { hasAtLeast, sortBy } from "remeda";
-import type { Owner, Passage, ProsePiece } from "../../domain/tex-document.ts";
+import { hasAtLeast, mapWithFeedback, only, sortBy, zip } from "remeda";
+import type {
+  Emphasis,
+  EmphasisStyle,
+  Owner,
+  Passage,
+  Place,
+  ProsePiece,
+} from "../../domain/tex-document.ts";
 import {
   argumentsOf,
+  collapse,
   inPlace,
   macroPlace,
   mandatory,
@@ -28,12 +36,16 @@ import {
 } from "./nodes.ts";
 import { CITATION_MACROS, DEFINITION_MACROS, type ParsedTex } from "./parse.ts";
 
-/** What the walk yields: a piece, the end of a passage, the start of the back matter, a footnote. */
+/**
+ * What the walk yields: a piece, the end of a passage, the start of the back matter, a footnote, and
+ * an emphasised phrase — which stands beside the pieces it typesets, and makes no passage of its own.
+ */
 type Event =
   | { readonly kind: "piece"; readonly piece: ProsePiece }
   | { readonly kind: "break" }
   | { readonly kind: "end" }
-  | { readonly kind: "aside"; readonly events: readonly Event[] };
+  | { readonly kind: "aside"; readonly events: readonly Event[] }
+  | { readonly kind: "emphasis"; readonly emphasis: Emphasis };
 
 const BREAK: Event = { kind: "break" };
 const END: Event = { kind: "end" };
@@ -251,6 +263,75 @@ const OWN_EVENTS: ReadonlyMap<
   ],
 ]);
 
+/** Formatting macros that set their argument apart from the running text, and how. */
+const EMPHASIS_MACROS: ReadonlyMap<string, EmphasisStyle> = new Map([
+  ["textbf", "bold"],
+  ["emph", "italic"],
+  ["textit", "italic"],
+  ["textsl", "italic"],
+  ["underline", "underline"],
+  ["uline", "underline"],
+]);
+
+/**
+ * Declarations that set the rest of their group apart — `{\bfseries …}`, `{\em …}` — and how. The
+ * group is the phrase.
+ */
+const EMPHASIS_DECLARATIONS: ReadonlyMap<string, EmphasisStyle> = new Map([
+  ["bfseries", "bold"],
+  ["bf", "bold"],
+  ["em", "italic"],
+  ["itshape", "italic"],
+  ["it", "italic"],
+  ["slshape", "italic"],
+]);
+
+/** An emphasised phrase over `place`, typesetting what `inner` typesets; nothing when it has no place or no text. */
+const emphasisOf = (
+  style: EmphasisStyle,
+  place: Place,
+  inner: readonly Event[],
+): readonly Event[] =>
+  inPlace(place, (span) => {
+    const text = collapse(typeset(inner));
+    return text === ""
+      ? []
+      : [
+          {
+            kind: "emphasis",
+            emphasis: { style, place: "inline", text, span },
+          },
+        ];
+  });
+
+/** A formatting macro's events: what it typesets, and — for an emphasising one — the phrase. */
+function formattingEvents(t: ParsedTex, m: Macro): readonly Event[] {
+  const inner = walkList(t, lastArgument(m));
+  const style = EMPHASIS_MACROS.get(m.content);
+  return style === undefined
+    ? inner
+    : [...emphasisOf(style, macroPlace(m), inner), ...inner];
+}
+
+/** The style a group's opening declaration sets, when it opens with one (`{\bfseries …}`). */
+function declaredStyle(list: readonly Node[]): EmphasisStyle | undefined {
+  const first = list.find(
+    (n) => n.type !== "whitespace" && n.type !== "comment",
+  );
+  return first?.type === "macro"
+    ? EMPHASIS_DECLARATIONS.get(first.content)
+    : undefined;
+}
+
+/** A group's events: what it typesets, and — when it opens with an emphasis declaration — the phrase. */
+function groupEvents(t: ParsedTex, g: NodeOf<"group">): readonly Event[] {
+  const inner = walkList(t, g.content);
+  const style = declaredStyle(g.content);
+  return style === undefined
+    ? inner
+    : [...emphasisOf(style, placeOf(g), inner), ...inner];
+}
+
 function macroEvents(t: ParsedTex, m: Macro): readonly Event[] {
   const name = m.content;
   if (DEFINITION_MACROS.has(name)) return [];
@@ -260,7 +341,7 @@ function macroEvents(t: ParsedTex, m: Macro): readonly Event[] {
   const own = OWN_EVENTS.get(name);
   if (own !== undefined) return own(t, m);
   if (BREAK_MACROS.has(name)) return [BREAK];
-  if (INLINE_MACROS.has(name)) return walkList(t, lastArgument(m));
+  if (INLINE_MACROS.has(name)) return formattingEvents(t, m);
   const ch = CHARACTER_MACROS.get(name);
   return ch === undefined ? [] : characterAt(m, ch);
 }
@@ -298,7 +379,7 @@ function nodeEvents(t: ParsedTex, n: Node): readonly Event[] {
     case "whitespace":
       return textAt(n, " ");
     case "group":
-      return walkList(t, n.content);
+      return groupEvents(t, n);
     case "environment":
     case "mathenv":
       return environmentEvents(t, n);
@@ -346,20 +427,12 @@ function step(
  * macro the parser gave no signature — `\institution{…}`, `\ccsdesc[500]{…}`, `\tool{}`,
  * `\if\conference1` — which are its arguments, not text. A formatting macro's group (`\enquote{…}` without csquotes' signature) stays prose.
  */
-const proseNodes = (list: readonly Node[]): readonly Node[] =>
-  list.reduce<{
-    readonly kept: readonly Node[];
-    readonly state: ArgumentState;
-  }>(
-    (acc, n) => {
-      const next = step(acc.state, n);
-      return {
-        kept: next.skip ? acc.kept : [...acc.kept, n],
-        state: next.state,
-      };
-    },
-    { kept: [], state: "prose" },
-  ).kept;
+const proseNodes = (list: readonly Node[]): readonly Node[] => {
+  // Each node's step, fed the state the one before left: a scan, linear in the list.
+  const start: ReturnType<typeof step> = { skip: false, state: "prose" };
+  const steps = mapWithFeedback(list, (prev, n) => step(prev.state, n), start);
+  return zip(list, steps).flatMap(([n, st]) => (st.skip ? [] : [n]));
+};
 
 /** Formatting macros a run-in heading is set in: `\textbf{Correctness gate.} Each task…`. */
 const RUN_IN_MACROS: ReadonlySet<string> = new Set([
@@ -378,11 +451,33 @@ const typeset = (events: readonly Event[]): string =>
     )
     .join("");
 
-/** A formatting macro whose phrase ends in `.` or `:` — a run-in heading, when it opens a paragraph. */
-const isRunInHeading = (t: ParsedTex, n: Node): boolean =>
-  n.type === "macro" &&
-  RUN_IN_MACROS.has(n.content) &&
-  /[.:]$/u.test(typeset(walkList(t, lastArgument(n))).trim());
+/** The one node a group holds besides spaces and comments — `{\textbf{…}}` → the `\textbf` — or null. */
+const soleChild = (n: Node): Node | null => {
+  if (n.type !== "group") return null;
+  const kept = n.content.filter(
+    (c) => c.type !== "whitespace" && c.type !== "comment",
+  );
+  return only(kept) ?? null;
+};
+
+/** A node with the groups around it taken off: `{{\textbf{…}}}` → the `\textbf`. */
+const unwrapped = (n: Node): Node => {
+  const inner = soleChild(n);
+  return inner === null ? n : unwrapped(inner);
+};
+
+/**
+ * A formatting macro whose phrase ends in `.` or `:` — a run-in heading, when it opens a paragraph.
+ * Braces around it change nothing on the page, so `{\textbf{Threats:}}` is one too.
+ */
+const isRunInHeading = (t: ParsedTex, node: Node): boolean => {
+  const n = unwrapped(node);
+  return (
+    n.type === "macro" &&
+    RUN_IN_MACROS.has(n.content) &&
+    /[.:]$/u.test(typeset(walkList(t, lastArgument(n))).trim())
+  );
+};
 
 /** Macros that typeset no text, and so leave a paragraph's opening where it was. */
 const SILENT_AT_OPENING: ReadonlySet<string> = new Set([
@@ -405,28 +500,47 @@ const keepsOpening = (n: Node): boolean =>
 const opensAfter = (n: Node): boolean =>
   n.type === "parbreak" || (n.type === "macro" && BREAK_MACROS.has(n.content));
 
+/**
+ * An emphasising node's events when it opens a paragraph: its own phrase — the first event — is a
+ * label, not inline; braces around it change nothing on the page (`{\textbf{…}}.`). Any other node
+ * (an environment, a footnote) is left as it is: what it holds opens nothing here.
+ */
+const asOpening = (node: Node, events: readonly Event[]): readonly Event[] => {
+  const [own, ...rest] = events;
+  const n = unwrapped(node);
+  const emphasises =
+    (n.type === "macro" && EMPHASIS_MACROS.has(n.content)) ||
+    (n.type === "group" && declaredStyle(n.content) !== undefined);
+  return emphasises && own?.kind === "emphasis"
+    ? [
+        { kind: "emphasis", emphasis: { ...own.emphasis, place: "opening" } },
+        ...rest,
+      ]
+    : events;
+};
+
 /**
  * A list's events. `opens` says the list starts a paragraph (the body, an environment's content);
- * a run-in heading that opens a paragraph ends the passage before it and is left out, like a heading.
+ * a run-in heading that opens a paragraph ends the passage before it and is left out, like a heading,
+ * and a phrase emphasised there is a label (`opening`).
  */
 function walkList(
   t: ParsedTex,
   list: readonly Node[],
   opens = false,
 ): readonly Event[] {
-  return proseNodes(list).reduce<{
-    readonly events: readonly Event[];
-    readonly opening: boolean;
-  }>(
-    (acc, n) => ({
-      events: [
-        ...acc.events,
-        ...(acc.opening && isRunInHeading(t, n) ? [BREAK] : nodeEvents(t, n)),
-      ],
-      opening: opensAfter(n) || (acc.opening && keepsOpening(n)),
-    }),
-    { events: [], opening: opens },
-  ).events;
+  const nodes = proseNodes(list);
+  // Whether a paragraph is open AFTER each node: a scan, so the walk stays linear in the list.
+  const after = mapWithFeedback(
+    nodes,
+    (opening, n) => opensAfter(n) || (opening && keepsOpening(n)),
+    opens,
+  );
+  return nodes.flatMap((n, i) => {
+    const opening = i === 0 ? opens : after[i - 1] === true;
+    if (!opening) return nodeEvents(t, n);
+    return isRunInHeading(t, n) ? [BREAK] : asOpening(n, nodeEvents(t, n));
+  });
 }
 
 /** The events before the back matter starts. */
@@ -437,27 +551,35 @@ const untilEnd = (events: readonly Event[]): readonly Event[] => {
 
 /** Events → passages: pieces between breaks, each footnote a passage of its own. */
 function passagesOf(events: readonly Event[]): readonly Passage[] {
-  const { done, current } = events.reduce<{
-    readonly done: readonly (readonly ProsePiece[])[];
-    readonly current: readonly ProsePiece[];
-  }>(
-    (acc, e) =>
-      e.kind === "piece"
-        ? { done: acc.done, current: [...acc.current, e.piece] }
-        : e.kind === "break"
-          ? { done: [...acc.done, acc.current], current: [] }
-          : acc,
-    { done: [], current: [] },
+  // Where each passage starts and ends: between breaks. Slicing between them keeps this linear in
+  // the events — a paragraph of thousands of words is one slice, not a copy per word.
+  const breaks = events.flatMap((e, i) => (e.kind === "break" ? [i] : []));
+  const bounds = zip(
+    [0, ...breaks.map((b) => b + 1)],
+    [...breaks, events.length],
   );
+  const own = bounds.flatMap(([from, to]) => {
+    const pieces = events
+      .slice(from, to)
+      .flatMap((e) => (e.kind === "piece" ? [e.piece] : []));
+    return hasAtLeast(pieces, 1) ? [{ pieces }] : [];
+  });
   const asides = events.flatMap((e) =>
     e.kind === "aside" ? passagesOf(untilEnd(e.events)) : [],
   );
-  const own = [...done, current].flatMap((pieces) =>
-    hasAtLeast(pieces, 1) ? [{ pieces }] : [],
-  );
   return [...own, ...asides];
 }
 
+/** Every emphasised phrase among events, a footnote's included, up to the back matter. */
+const emphasesOf = (events: readonly Event[]): readonly Emphasis[] =>
+  untilEnd(events).flatMap((e) =>
+    e.kind === "emphasis"
+      ? [e.emphasis]
+      : e.kind === "aside"
+        ? emphasesOf(e.events)
+        : [],
+  );
+
 const isDocument = (n: Node): n is Readonly<Ast.Environment> =>
   n.type === "environment" && n.env === "document";
 
@@ -476,9 +598,17 @@ const startOf = (p: Passage): number => {
  * source when there is none), up to the first back-matter node.
  */
 export function bodyProse(t: ParsedTex): readonly Passage[] {
-  const body = t.root.content.find(isDocument)?.content ?? t.root.content;
-  return sortBy(
-    passagesOf(untilEnd(walkList(t, body, true))).filter(hasProse),
-    startOf,
-  );
+  return sortBy(passagesOf(untilEnd(bodyEvents(t))).filter(hasProse), startOf);
+}
+
+/** The walk of the body: the content of the `document` environment, or the whole source when there is none. */
+const bodyEvents = (t: ParsedTex): readonly Event[] =>
+  walkList(t, t.root.content.find(isDocument)?.content ?? t.root.content, true);
+
+/**
+ * The phrases set in bold, italics or underline inside the body's prose, in document order: what
+ * `bodyProse` reads, so never a heading, a run-in heading, a caption, a float or a table.
+ */
+export function bodyEmphasis(t: ParsedTex): readonly Emphasis[] {
+  return sortBy(emphasesOf(bodyEvents(t)), (e) => e.span.start);
 }

package/src/cli.ts

@@ -65,6 +65,10 @@ import {
 } from "./tex-venue-rules.ts";
 import { claimProvenanceRule } from "./claim-provenance.ts";
 import { registerRule } from "./register.ts";
+import {
+  REGISTER_BAND_RULE_LEVELS,
+  registerBandRules,
+} from "./register-bands.ts";
 import {
   paperRules,
   identityField,
@@ -236,6 +240,13 @@ settings — paperlint.json, at two levels, one schema. Both are optional.
   file, is an error.
 `;
 
+/** What the rules over a paper.tex read with: the disk, the shipped presets, the LaTeX reader. */
+const TEX_RULE_DEPS = {
+  files: nodeFiles,
+  venuesDir: presetsDir(),
+  latex: latexReader,
+};
+
 /** The config the user would otherwise write by hand. The data comes from `opts`, the mechanism is here. */
 export function buildConfig(
   opts: PaperlintConfig = {},
@@ -325,30 +336,17 @@ export function buildConfig(
           languages: { latex: texLanguage },
           rules: {
             ...texBuild,
-            ...texVenueRules({
-              files: nodeFiles,
-              venuesDir: presetsDir(),
-              latex: latexReader,
-            }),
-            "claim-provenance": claimProvenanceRule({
-              files: nodeFiles,
-              latex: latexReader,
-            }),
-            register: registerRule({
-              files: nodeFiles,
-              latex: latexReader,
-            }),
+            ...texVenueRules(TEX_RULE_DEPS),
+            "claim-provenance": claimProvenanceRule(TEX_RULE_DEPS),
+            register: registerRule(TEX_RULE_DEPS),
+            ...registerBandRules(TEX_RULE_DEPS),
           },
         },
         paper: { rules: texPaperRules },
         bib: bibReachable,
         format: {
           rules: {
-            ...formatRules({
-              files: nodeFiles,
-              venuesDir: presetsDir(),
-              latex: latexReader,
-            }),
+            ...formatRules(TEX_RULE_DEPS),
             ...pageLimitRules({ files: nodeFiles, venuesDir: presetsDir() }),
           },
         },
@@ -364,6 +362,9 @@ export function buildConfig(
         "tex/claim-provenance": "warn",
         // Sentence-initial conjunctions over the whole body: register, not vocabulary.
         "tex/register": "warn",
+        // Contrast frames, claims in bold, relation markers: each against the band the venue
+        // preset's anchors set, silent for a preset without anchors (src/register-bands.ts).
+        ...REGISTER_BAND_RULE_LEVELS,
         "tex/acm-frontmatter-override": "error",
         // Silent for a paper whose paperlint.json names no venue (src/venue-rules.ts).
         ...VENUE_RULE_LEVELS,

package/src/domain/register.test.ts

@@ -0,0 +1,153 @@
+/**
+ * The register measures on real parses: what counts as a contrast frame, a relation marker and a
+ * claim in bold, and what does not; the band the anchors set; the excerpt a finding quotes.
+ * Every "does not count" case below is a sentence shape found in an accepted ACSAC paper.
+ */
+import { describe, expect, it } from "vitest";
+import { latexReader } from "../adapters/latex/index.ts";
+import {
+  bandOf,
+  bodySentences,
+  claimEmphasis,
+  contrastFrames,
+  excerpt,
+  rateOf,
+  relationMarkers,
+  type RegisterAnchor,
+} from "./register.ts";
+
+const doc = (body: string): string =>
+  `\\documentclass{article}\n\\begin{document}\n${body}\n\\end{document}\n`;
+const sentencesOf = (body: string) =>
+  bodySentences(latexReader.bodyProse(doc(body)));
+const frames = (body: string): number =>
+  contrastFrames(sentencesOf(body)).length;
+const markers = (body: string): number =>
+  relationMarkers(sentencesOf(body)).length;
+const claims = (body: string): readonly string[] =>
+  claimEmphasis(latexReader.bodyEmphasis(doc(body))).map((o) => o.match);
+
+describe("contrastFrames — a claim set against a rejected alternative", () => {
+  it.each([
+    [
+      "X, not Y",
+      "They react to the dangerous word, not the dangerous operation.",
+    ],
+    ["X --- not Y", "It guarantees encryption --- not authenticity."],
+    ["X; not Y", "We score the operation; not the attacker."],
+    ["X, and not Y", "It reads the field, and not the raw input."],
+    ["not X but Y", "Over-blocking is not a side metric but a precondition."],
+    ["rather than", "It fires on the mention rather than the act."],
+    ["instead of", "It reads the whole event instead of the command."],
+    ["as opposed to", "It matches words as opposed to operations."],
+  ])("🔴 %s is one frame", (_form, sentence) => {
+    expect(frames(sentence)).toBe(1);
+  });
+
+  it.each([
+    ["not only … but also", "It is not only fast but also exact."],
+    [
+      "a negated clause",
+      "The nature of the audit is not reported, but it did not assess quality.",
+    ],
+    ["«, not yet»", "The fix is merged, not yet released."],
+    [
+      "«, not already in the corpus»",
+      "We kept every guard published since, not already in the corpus.",
+    ],
+    ["«without»", "The attack works without any access to the device."],
+    ["«unlike»", "Unlike prior work, we measure the installed guards."],
+    ["a bare negation", "The guard does not block it."],
+  ])("%s is no frame", (_form, sentence) => {
+    expect(frames(sentence)).toBe(0);
+  });
+
+  it("two frames in one sentence are two; one phrase matching two forms is one", () => {
+    expect(
+      frames("It reads the word, not the act, rather than the operation."),
+    ).toBe(2);
+    expect(frames("It is not a word but rather than a verb.")).toBe(2);
+  });
+
+  it("a frame in a table, a caption or a heading is not prose, so not counted", () => {
+    expect(
+      frames(
+        "\\section{Words, not operations}\n\\begin{table}\\caption{Words, not operations.}\\begin{tabular}{l}a, not b\\end{tabular}\\end{table}\nPlain text here.",
+      ),
+    ).toBe(0);
+  });
+});
+
+describe("relationMarkers — words that name how a sentence relates to the one before", () => {
+  it("🔴 each marker counts where it stands, sentence-initial or not", () => {
+    expect(
+      markers(
+        "However, the rate is low. It fails because the rule matches words. For example, echo is blocked. It is therefore unsafe.",
+      ),
+    ).toBe(4);
+  });
+
+  it("a marker inside a longer word is not one: «thusly» is no «thus», «sincere» no «since»", () => {
+    expect(
+      markers("The sincere author wrote thusly. Eventually it held."),
+    ).toBe(0);
+  });
+});
+
+describe("claimEmphasis — a claim set in bold inside a sentence", () => {
+  it("🔴 a bold phrase of six words or more, or holding a number, inside a sentence", () => {
+    expect(
+      claims(
+        "We find that \\textbf{they react to the dangerous word}. The \\textbf{median coverage is 2.5/10} here.",
+      ),
+    ).toEqual([
+      "they react to the dangerous word",
+      "median coverage is 2.5/10",
+    ]);
+  });
+
+  it("a term in bold, a label opening a paragraph or item, italics, and a run-in heading are not claims", () => {
+    expect(
+      claims(
+        "BaroVox offers a \\textbf{Pressure-Acoustic Transformation (PAT)} here.\n\n\\textbf{Varying the orientation of the sound source from the device}. We vary it.\n\\begin{itemize}\n\\item \\textbf{A corpus of user security questions with sources}: we release it.\n\\end{itemize}\nWe \\emph{find that the guards react to the word}.\n\n\\textbf{Threats to validity of the 46 guards.} None.",
+      ),
+    ).toEqual([]);
+  });
+});
+
+describe("bandOf — the anchors' range, widened by two Poisson standard deviations", () => {
+  const anchor = (words: number, contrast_frames: number): RegisterAnchor => ({
+    paper: "p",
+    words,
+    counts: { contrast_frames, claim_emphasis: 0, relation_markers: 0 },
+  });
+
+  it("🔴 the edges stand two standard deviations of the edge anchor's count beyond it", () => {
+    // 4 in 10,000 words: rate 4, sd 2 → max 8. 1 in 10,000: rate 1, sd 1 → min 1 - 2 < 0 → 0.
+    expect(
+      bandOf([anchor(10_000, 4), anchor(10_000, 1)], "contrast_frames"),
+    ).toEqual({ min: 0, max: 8, anchorMin: 1, anchorMax: 4, anchors: 2 });
+  });
+
+  it("an anchor with no occurrence is widened as if it had one; a floor above zero stays", () => {
+    expect(bandOf([anchor(10_000, 0)], "contrast_frames")?.max).toBe(2);
+    expect(bandOf([anchor(10_000, 100)], "contrast_frames")?.min).toBe(80);
+  });
+
+  it("no anchors, no band; a body of no words has a rate of zero", () => {
+    expect(bandOf([], "contrast_frames")).toBeNull();
+    expect(rateOf(3, 0)).toBe(0);
+  });
+});
+
+describe("excerpt — the words around an occurrence, as a finding quotes them", () => {
+  it("the match with its neighbours, cut at a word, an ellipsis where the sentence goes on", () => {
+    const s =
+      "We score 46 gates and find that they react to the dangerous word, not the dangerous operation, in every one of the ten cases we built.";
+    const at = s.indexOf(", not");
+    expect(excerpt(s, at, at + 6)).toBe(
+      "…that they react to the dangerous word, not the dangerous operation, in every one…",
+    );
+    expect(excerpt("a, not b", 1, 7)).toBe("a, not b");
+  });
+});