paperlint 13.2.0 → 13.3.1

package/skills/assess-novelty/scripts/check_novelty_coverage.test.ts

@@ -0,0 +1,176 @@
+/**
+ * check_novelty.py, coverage: a neighbour marked read "full" proves it with one verbatim sentence per chunk of
+ * its saved text. Execution tier: the novelty fixture on disk, no model, no network.
+ * Run: npx vitest run skills/assess-novelty/scripts/check_novelty_coverage.test.ts
+ *
+ * Why: one Read returns at most 25,000 tokens — a 1,790-line text came back as lines 1-627 — yet
+ * the neighbour was marked read "full", and a 'no' cell (the one that grants novelty) rests on that mark.
+ * Each case copies the fixture, makes ONE change; every failing class has a clean twin that passes.
+ */
+import {
+  cpSync,
+  mkdtempSync,
+  readFileSync,
+  realpathSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { spawnSync } from "node:child_process";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import assert from "node:assert/strict";
+import { test } from "vitest";
+
+test("check_novelty_coverage", () => {
+  const here = dirname(fileURLToPath(import.meta.url));
+  const FIXTURE = join(here, "fixtures", "novelty");
+  const CHECKER = join(here, "check_novelty.py");
+  const TEXT = join("siblings", "data", "doe-preprint.txt");
+
+  // Copy the fixture, let `change` edit the novelty table and the saved text, run the checker.
+  const run = (change: (f: { json: any; text: string }) => void) => {
+    const dir = realpathSync(mkdtempSync(join(tmpdir(), "novelty-cov-")));
+    try {
+      cpSync(FIXTURE, dir, { recursive: true });
+      const jsonPath = join(dir, "siblings", "novelty.json");
+      const files = {
+        json: JSON.parse(readFileSync(jsonPath, "utf8")),
+        text: readFileSync(join(dir, TEXT), "utf8"),
+      };
+      change(files);
+      writeFileSync(jsonPath, JSON.stringify(files.json, null, 2));
+      writeFileSync(join(dir, TEXT), files.text);
+      const r = spawnSync("python3", ["-I", CHECKER, dir], {
+        encoding: "utf8",
+      });
+      return { code: r.status, out: `${r.stdout}\n${r.stderr}` };
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  };
+  const passes = (change: Parameters<typeof run>[0], what: string) => {
+    const r = run(change);
+    assert.equal(r.code, 0, `must pass: ${what}\n${r.out}`);
+  };
+  const fails = (
+    change: Parameters<typeof run>[0],
+    re: RegExp,
+    what: string,
+  ) => {
+    const r = run(change);
+    assert.equal(r.code, 1, `must fail: ${what}\n${r.out}`);
+    assert.match(r.out, re, `wrong reason for: ${what}\n${r.out}`);
+  };
+  const doe = (j: any) => j.neighbours.find((n: any) => n.id === "doe26");
+  const lines = (t: string) =>
+    t.split("\n").length - (t.endsWith("\n") ? 1 : 0);
+  // A sentence that sits on line 9 of the fixture text, and one from its last lines.
+  const EARLY = "instruction that the model interprets. The agent’s block is";
+  const LATE = "the denominator of the main estimate; it means the estimate";
+
+  // ── clean: the committed fixture covers its one chunk ───────────────────────────
+  passes(
+    () => {},
+    "fixture as committed: one chunk, lines 1-63, quote inside it",
+  );
+
+  // ── 1. read "full" with no coverage at all ─────────────────────────────────────
+  fails(
+    (f) => {
+      delete doe(f.json).coverage;
+    },
+    /doe26: read 'full' not proven: no coverage/,
+    "full without coverage",
+  );
+  {
+    // An abstract read turns P5-1 ⚪ against its 🟢 ledger row, so this twin cannot exit 0; it must only be silent
+    // about coverage.
+    const r = run((f) => {
+      delete doe(f.json).coverage;
+      doe(f.json).read = "abstract";
+      for (const c of f.json.claims)
+        for (const cell of Object.values<any>(c.cells?.doe26 ?? {}))
+          if (cell.has === "no") cell.has = "unknown";
+    });
+    assert.doesNotMatch(
+      r.out,
+      /coverage|not proven/,
+      `an abstract-only neighbour needs no coverage\n${r.out}`,
+    );
+  }
+
+  // ── 2. the chunks must tile the text: start at 1, no gap, end at the last line ──
+  fails(
+    (f) => {
+      doe(f.json).coverage = [
+        { lines: [1, 30], quote: EARLY },
+        { lines: [40, lines(f.text)], quote: LATE },
+      ];
+    },
+    /doe26: read 'full' not proven: lines 31-39 have no coverage quote/,
+    "a gap between chunks",
+  );
+  fails(
+    (f) => {
+      doe(f.json).coverage = [{ lines: [1, 50], quote: EARLY }];
+    },
+    /doe26: read 'full' not proven: lines 51-63 have no coverage quote/,
+    "stops before the last line",
+  );
+  passes((f) => {
+    doe(f.json).coverage = [
+      { lines: [1, 30], quote: EARLY },
+      { lines: [31, lines(f.text)], quote: LATE },
+    ];
+  }, "two chunks that tile the text");
+
+  // ── 3. a chunk longer than one careful read ────────────────────────────────────
+  fails(
+    (f) => {
+      f.text =
+        f.text +
+        Array.from(
+          { length: 400 },
+          (_, i) => `padding line ${i} so the text runs past one chunk`,
+        ).join("\n") +
+        "\n";
+      doe(f.json).coverage = [{ lines: [1, lines(f.text)], quote: EARLY }];
+    },
+    /doe26: coverage chunk 1-463 is 463 lines; at most 400/,
+    "one chunk over 400 lines",
+  );
+
+  // ── 4. the quote must be inside ITS chunk, not anywhere in the text ────────────
+  fails(
+    (f) => {
+      doe(f.json).coverage = [
+        { lines: [1, 30], quote: LATE },
+        { lines: [31, lines(f.text)], quote: EARLY },
+      ];
+    },
+    /doe26: coverage quote for lines 1-30 not found in those lines/,
+    "quotes swapped between chunks",
+  );
+
+  // ── 5. a coverage quote is a sentence, not a word ──────────────────────────────
+  fails(
+    (f) => {
+      doe(f.json).coverage = [
+        { lines: [1, lines(f.text)], quote: "the model" },
+      ];
+    },
+    /doe26: coverage quote for lines 1-63 has 2 words; at least 8/,
+    "a two-word quote",
+  );
+
+  // ── 6. lines are counted the way Read and wc -l count them: by \n only ─────────
+  // pdftotext writes a form feed at every page break; Python's splitlines() splits on it, so a 1,825-line text
+  // counted 1,850 and readers (sibling-read) had to stretch their last range past the file's real end.
+  passes((f) => {
+    f.text = f.text
+      .replace("\n\nAbstract", "\n\f\nAbstract")
+      .replace("Pat Doe", "Pat\fDoe");
+    doe(f.json).coverage = [{ lines: [1, lines(f.text)], quote: EARLY }];
+  }, "a text with form feeds is covered by its \\n line count");
+}, 300_000);

package/skills/assess-novelty/scripts/crop_quote.py

@@ -0,0 +1,104 @@
+"""Find a verbatim passage in a PDF, render its page, highlight the passage and crop to its column.
+
+Usage: python3 -I crop_quote.py <pdf> <out.png> "<passage>" [--page N]
+Prints one JSON line: {"page": N, "pages": [every page the passage is on], "words": "the matched words"}.
+Exit 1: the passage is not in the PDF (nothing written). Exit 3: it is on more than one page and --page
+does not pick one (an abstract repeats its body; the first hit is the wrong one as often as the right one).
+
+PyMuPDF does the reading and the drawing (it is what CI installs for the PDF harnesses; the runner has no
+poppler, so pdftotext and pdftoppm are not there, measured on run 40927062797).
+
+Matching ignores case, punctuation, spacing and line-end hyphenation: the PDF's words and the passage are
+both reduced to letters and digits (NFKC first, so a ligature reads as its letters) and compared as one
+stream. A passage matches whole or not at all; there is no shorter fallback, because a fallback that finds
+"a" place is how a crop lands on the wrong sentence.
+"""
+
+import argparse
+import json
+from bisect import bisect_right
+from itertools import accumulate
+import re
+import sys
+import unicodedata
+
+import pymupdf
+
+DPI = 150
+CONTEXT_PT = 60  # points of page shown above and below the passage
+
+
+def norm(text):
+    return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKC", text).lower())
+
+
+def words(pdf):
+    """Every word of the PDF as (page, x0, y0, x1, y1, text), pages counted from 1."""
+    with pymupdf.open(pdf) as doc:
+        return [(number, x0, y0, x1, y1, text)
+                for number, page in enumerate(doc, start=1)
+                for x0, y0, x1, y1, text, *_ in page.get_text("words")]
+
+
+def occurrences(ws, passage):
+    """Each match of the passage as the list of word indices it covers."""
+    keys = [norm(w[5]) for w in ws]
+    starts = list(accumulate((len(k) for k in keys), initial=0))
+    stream, needle = "".join(keys), norm(passage)
+    if not needle:
+        return []
+    hits = [m.start() for m in re.finditer(f"(?={re.escape(needle)})", stream)]
+    # A match covers the words from the one its first letter falls in to the one its last letter falls in.
+    span = [(bisect_right(starts, at) - 1, bisect_right(starts, at + len(needle) - 1) - 1) for at in hits]
+    return [[i for i in range(lo, hi + 1) if keys[i]] for lo, hi in span]
+
+
+def render(pdf, page, sel, ws, out):
+    # The column the passage sits in: every word of the page within 40 pt of the passage's left and right edge.
+    xs0, xs1 = min(w[1] for w in sel), max(w[3] for w in sel)
+    ys0, ys1 = min(w[2] for w in sel), max(w[4] for w in sel)
+    col = [w for w in ws if w[0] == page and w[1] >= xs0 - 40 and w[3] <= xs1 + 40]
+    cx0 = min([w[1] for w in col] + [xs0]) - 8
+    cx1 = max([w[3] for w in col] + [xs1]) + 8
+    with pymupdf.open(pdf) as doc:
+        pg = doc[page - 1]
+        # Drawn on the page in memory only; the PDF on disk is never saved.
+        for _, x0, y0, x1, y1, _ in sel:
+            pg.draw_rect(pymupdf.Rect(x0 - 0.5, y0 - 0.5, x1 + 0.5, y1 + 0.5),
+                         color=None, fill=(1, 0.84, 0), fill_opacity=0.43, overlay=True)
+        clip = pymupdf.Rect(cx0, ys0 - CONTEXT_PT, cx1, ys1 + CONTEXT_PT) & pg.rect
+        pg.get_pixmap(dpi=DPI, clip=clip).save(out)
+
+
+def main(argv):
+    p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    p.add_argument("pdf")
+    p.add_argument("out")
+    p.add_argument("passage")
+    p.add_argument("--page", type=int)
+    a = p.parse_args(argv)
+    ws = words(a.pdf)
+    found = occurrences(ws, a.passage)
+    pages = sorted({ws[m[0]][0] for m in found})
+    if not found:
+        print(f"NOT FOUND: {a.passage[:80]}", file=sys.stderr)
+        return 1
+    picked = [m for m in found if a.page is None or ws[m[0]][0] == a.page]
+    if not picked:
+        print(f"NOT ON PAGE {a.page}: found on pages {pages}", file=sys.stderr)
+        return 1
+    if len({ws[m[0]][0] for m in picked}) > 1:
+        print(f"ON SEVERAL PAGES {pages}: pass --page to pick one", file=sys.stderr)
+        return 3
+    match = picked[0]
+    page = ws[match[0]][0]
+    sel = [ws[i] for i in match if ws[i][0] == page]
+    if len(sel) < len(match):
+        print(f"passage runs past page {page}: highlighted the part on it", file=sys.stderr)
+    render(a.pdf, page, sel, ws, a.out)
+    print(json.dumps({"page": page, "pages": pages, "words": " ".join(w[5] for w in sel)}, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/skills/assess-novelty/scripts/crop_quote.test.ts

@@ -0,0 +1,130 @@
+/**
+ * crop_quote.py: a verbatim passage is found in a PDF by its words, highlighted on its page and cropped.
+ * Execution tier: the committed fixtures/crop-quote/sample.pdf (made by make_sample.py beside it), no network.
+ * Run: npx vitest run skills/assess-novelty/scripts/crop_quote.test.ts
+ *
+ * Why: the scratch version matched the first 6 or 3 words when the whole passage did not match, and
+ * took the first hit, so a crop of a paper landed on the abstract instead of the method section. Every
+ * passage shown to the author as "the paper says this" rests on this match, so a wrong place must be a refusal.
+ */
+import { existsSync, mkdtempSync, realpathSync, rmSync } from "node:fs";
+import { spawnSync } from "node:child_process";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import assert from "node:assert/strict";
+import { test } from "vitest";
+
+test("crop_quote", () => {
+  const here = dirname(fileURLToPath(import.meta.url));
+  const TOOL = join(here, "crop_quote.py");
+  const PDF = join(here, "fixtures", "crop-quote", "sample.pdf");
+  // -E -P is -I without -s. On the CI runner pip installs PyMuPDF into the user's site-packages (it is not
+  // root), which -I hides: run 40937989243 failed with neither pymupdf nor fitz importable. -E and -P still keep
+  // PYTHON* variables out and the current and script directories off sys.path.
+  const PY = ["-E", "-P"];
+
+  const run = (passage: string, ...extra: string[]) => {
+    const dir = realpathSync(mkdtempSync(join(tmpdir(), "crop-quote-")));
+    try {
+      const out = join(dir, "crop.png");
+      const r = spawnSync(
+        "python3",
+        [...PY, TOOL, PDF, out, passage, ...extra],
+        {
+          encoding: "utf8",
+        },
+      );
+      // Yellow highlight pixels in the crop: red and green high, blue low.
+      const yellow = existsSync(out)
+        ? Number(
+            spawnSync(
+              "python3",
+              [
+                ...PY,
+                "-c",
+                "import sys\nimport pymupdf\npx = pymupdf.Pixmap(sys.argv[1]); s, n = px.samples, px.n; " +
+                  "print(sum(1 for i in range(0, len(s), n) if s[i] > 200 and s[i + 1] > 180 and s[i + 2] < 160))",
+                out,
+              ],
+              { encoding: "utf8" },
+            ).stdout.trim(),
+          )
+        : 0;
+      return {
+        code: r.status,
+        out: r.stdout,
+        err: r.stderr,
+        written: existsSync(out),
+        yellow,
+      };
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  };
+  const json = (s: string) => JSON.parse(s.trim());
+
+  // ── fires: a passage on one page is found, highlighted and reported with its page ──
+  {
+    const r = run(
+      "We ran Checkstyle before and after applying the initial patch",
+    );
+    assert.equal(r.code, 0, r.err);
+    assert.deepEqual(json(r.out), {
+      page: 2,
+      pages: [2],
+      words: "We ran Checkstyle before and after applying the initial patch",
+    });
+    assert.ok(
+      r.yellow > 500,
+      `the crop carries the highlight (yellow pixels: ${r.yellow})`,
+    );
+  }
+
+  // ── a word split by a line-end hyphen still matches, and case and punctuation do not matter ──
+  {
+    const r = run("THE CONFIGURATIONS, of every linter");
+    assert.equal(r.code, 0, r.err);
+    assert.equal(json(r.out).words, "the configu- rations of every linter");
+  }
+
+  // ── two columns: a sentence over two lines of the left column is read in column order, not across the page ──
+  {
+    const r = run(
+      "Developers can configure most linters through a file that sits next to the code",
+    );
+    assert.equal(r.code, 0, r.err);
+    assert.equal(json(r.out).page, 3);
+  }
+
+  // ── quiet: a passage the PDF does not hold writes nothing ──
+  {
+    const r = run("Rules written only in prose are broken more often than");
+    assert.equal(r.code, 1, "a near miss is NOT FOUND, never a shorter match");
+    assert.match(r.err, /NOT FOUND/);
+    assert.equal(r.written, false);
+  }
+
+  // ── refuses: the same passage on two pages (abstract and body) without --page ──
+  {
+    const r = run("Rules written only in prose are broken as often");
+    assert.equal(r.code, 3);
+    assert.match(r.err, /ON SEVERAL PAGES \[1, 2\]: pass --page/);
+    assert.equal(r.written, false);
+  }
+
+  // ── --page picks one, and a page the passage is not on is NOT FOUND ──
+  {
+    const r = run(
+      "Rules written only in prose are broken as often",
+      "--page",
+      "2",
+    );
+    assert.equal(r.code, 0, r.err);
+    assert.deepEqual(json(r.out).pages, [1, 2]);
+    assert.equal(json(r.out).page, 2);
+    const wrong = run("We ran Checkstyle before and after", "--page", "1");
+    assert.equal(wrong.code, 1);
+    assert.match(wrong.err, /NOT ON PAGE 1: found on pages \[2\]/);
+  }
+}, 300_000);

package/skills/assess-novelty/scripts/fixtures/crop-quote/make_sample.py

@@ -0,0 +1,42 @@
+"""Writes sample.pdf, the crop_quote.harness.ts fixture. Run once; the PDF is committed so the test needs no reportlab.
+
+Page 1 states a sentence the way an abstract does; page 2 repeats it in the body (the case where the first hit
+is the wrong one), holds one sentence found nowhere else, and splits a word across a line with a hyphen.
+Page 3 has two columns at the same heights, as a conference paper does: a sentence that runs over two lines
+of the left column reads whole only if words keep column order, not top-to-bottom order across the page.
+"""
+import sys
+
+from reportlab.lib.pagesizes import letter
+from reportlab.pdfgen import canvas
+
+PAGES = [
+    ["Abstract. Rules written only in prose are broken as often as rules a linter checks.",
+     "This sentence fills the first page and is not quoted by any test."],
+    ["Introduction. Rules written only in prose are broken as often as rules a linter checks.",
+     "We ran Checkstyle before and after applying the initial patch and the final patch.",
+     "Most repositories never turn on the configu-",
+     "rations of every linter they install, and nobody notices."],
+]
+
+COLUMNS = [
+    (72, ["Developers can configure most", "linters through a file that", "sits next to the code."]),
+    (320, ["The right column talks", "about something else", "entirely, line by line."]),
+]
+
+c = canvas.Canvas(sys.argv[1], pagesize=letter, invariant=1)
+for lines in PAGES:
+    text = c.beginText(72, 720)
+    text.setFont("Helvetica", 11)
+    for line in lines:
+        text.textLine(line)
+    c.drawText(text)
+    c.showPage()
+for x, lines in COLUMNS:
+    text = c.beginText(x, 720)
+    text.setFont("Helvetica", 11)
+    for line in lines:
+        text.textLine(line)
+    c.drawText(text)
+c.showPage()
+c.save()

package/skills/assess-novelty/scripts/fixtures/crop-quote/sample.pdf

@@ -0,0 +1,106 @@
+%PDF-1.3
+%���� ReportLab Generated PDF document (opensource)
+1 0 obj
+<<
+/F1 2 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+4 0 obj
+<<
+/Contents 10 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+5 0 obj
+<<
+/Contents 11 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+6 0 obj
+<<
+/PageMode /UseNone /Pages 8 0 R /Type /Catalog
+>>
+endobj
+7 0 obj
+<<
+/Author (anonymous) /CreationDate (D:20000101000000+00'00') /Creator (anonymous) /Keywords () /ModDate (D:20000101000000+00'00') /Producer (ReportLab PDF Library - \(opensource\)) 
+  /Subject (unspecified) /Title (untitled) /Trapped /False
+>>
+endobj
+8 0 obj
+<<
+/Count 3 /Kids [ 3 0 R 4 0 R 5 0 R ] /Type /Pages
+>>
+endobj
+9 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 229
+>>
+stream
+GappXYn"W)$q,<Ri^R<(U`_CE";;Hk&#@%<JM_r1#kdX@"X.+$anL1G%@110SA#5O(5mfrWF9"uFgZre@W*X*2a;C_CG0,n053mr<Ri13\ntQ,o2=Q>N5Tt++@6Ap`;q.@?8<1!+Z(M&e8UB%b#aht`$'j\<8]k/2m-bs=4fO1[L3Kjoo^]0+\%3.+&uE^G8C]`B.0"Mi:5#XB/G5n#7.U4POoqjhK*^0?i~>endstream
+endobj
+10 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 316
+>>
+stream
+Gaq3!d7sce&:EqJb[WMVQEuG:)GNot`K9jg3Tso9mKN;@p\+`s2@u/_1UFD9Cs#ZCZq'%Oj6f?%p!=apgMNYAn>o[p=+A_T^?c=$n\eCdH/J:[\UGhr4Nph-;hpDo\r<6q6l^b,/dR+ZEHX%X@@-ac*_Zcb3F,OH(rS]!'("iK+a7#7?dRD,Il\iBG\XH6Z$87S`QHO-j/^*l;d9Ck+epRTJcl`A)6mZr<R_77@i+HTc@Y+(c9XFDP(MJ5nWPu`lqM/]I+ZcSs.X&hUbt@dGrb%?j>q&l'B;gQ9>':n.7Ji2Q\ilB@eC*KBm_K~>endstream
+endobj
+11 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 250
+>>
+stream
+Garo:bAP0N&4Q>@`EcAp66)*>,@JL=5N%2d@djKQPH+HU%Y<&OJNU4q^;o9/:s)\bhKWG+YrXA5$G(rF'H?6L%XKJrlA9C/W=moL82ZOqU:QQ[4aiKO2bFsI$#!d]D)W_lO-r5*0@1*NSIKs:\5.,.H5O$?OgO8;ad+^aI5*]R68Ik#`D?D=Be;O;S[:LS[d.'c\A8ESES(-baOG3;jQMGI)`j)(oEm%O:#s;?`0jG_e+m&IZZ"#d>/U~>endstream
+endobj
+xref
+0 12
+0000000000 65535 f 
+0000000061 00000 n 
+0000000092 00000 n 
+0000000199 00000 n 
+0000000392 00000 n 
+0000000586 00000 n 
+0000000780 00000 n 
+0000000848 00000 n 
+0000001109 00000 n 
+0000001180 00000 n 
+0000001499 00000 n 
+0000001906 00000 n 
+trailer
+<<
+/ID 
+[<1c178198fbdfa51b25995d89d4102043><1c178198fbdfa51b25995d89d4102043>]
+% ReportLab generated PDF document -- digest (opensource)
+
+/Info 7 0 R
+/Root 6 0 R
+/Size 12
+>>
+startxref
+2247
+%%EOF

package/skills/assess-novelty/scripts/fixtures/novelty/CLAIMS.md

@@ -0,0 +1,11 @@
+---
+title: "CLAIMS — fixture for check_novelty.harness.ts"
+tags: [paper, fixture, novelty]
+---
+
+# CLAIMS — fixture for check_novelty.harness.ts
+
+| # | Claim | Status | File |
+|---|---|---|---|
+| R1 | The first measurement of a rule extractor's recall in this literature | 🔴 taken (Doe 2026, recall 66.3 %) | `siblings/novelty.json` |
+| P5-1 | The repository's linter does not make 58.3 % of lint-checkable requirements an error | 🟢 ours | `siblings/novelty.json` |

package/skills/assess-novelty/scripts/fixtures/novelty/paper.tex

@@ -0,0 +1,10 @@
+% Fixture for check_novelty.harness.ts: sentences shaped like a measurement paper's,
+% with no priority claim on the rule finder's recall.
+\documentclass{article}
+\begin{document}
+We also report the recall of our rule finder: it finds 41.2\% of the directive-like rules in a public
+human-segmented gold set~\cite{gold}.
+In 50 repositories, the repository's own linter does not make 58.3\% of lint-checkable requirements an error.
+The first row of Table~1 lists the rules a linter can check.
+None of the 900 prose blocks we examined names an enforcement mechanism.
+\end{document}

package/skills/assess-novelty/scripts/fixtures/novelty/siblings/2026-10-27-novelty-refutation.md

@@ -0,0 +1,8 @@
+---
+title: "Refutation pass — fixture for check_novelty"
+tags: [paper, fixture, novelty]
+---
+
+# Refutation pass — fixture
+
+Stands in for the independent refutation pass a 🟢 claim must point at.

package/skills/assess-novelty/scripts/fixtures/novelty/siblings/data/doe-preprint.txt

@@ -0,0 +1,63 @@
+When “Avoid” Is Not block: Safety Rules in
+AGENTS.md vs. Built-In Guards
+
+Example preprint, 05 Jan 2026
+
+Pat Doe
+
+Abstract—In AGENTS.md, “avoid” is a natural-language
+instruction that the model interprets. The agent’s block is
+a built-in guard that stops an action before the agent can take
+it. Both can express the same safety goal, but they control
+the agent in different ways. We measure this gap in 481 public
+AGENTS.md files. A model matched the extracted candidate rules
+against the documented guards, and two safety reviewers
+independently checked a sample without seeing the model’s
+answers or each other’s labels. Depending on how closely a guard
+had to match the written rule, only about 4–16%
+of the retrieved safety rules had a matching built-in guard.
+Under the strictest standard the estimate was 4.4% (95% CI:
+2.6–6.7%), and the two reviewers agreed closely on which rules
+had a match. A manual review of complete files found that our
+extraction method captured 66.3% of eligible safety rules; the
+reported rates therefore apply to the rules it captured. This is
+a usable safety problem: a rules file is a write-only channel. A
+developer writes a safety rule but gets no feedback on whether
+a guard will enforce it. The same plain-text form hides two
+kinds of rule: those a guard or sandbox can
+enforce, and those left to the model to interpret.
+
+I. INTRODUCTION
+Language models have lowered the barrier to building
+software. Many people now build applications and coding
+agents with little safety experience, and they increasingly
+direct those agents in natural language. Agent instruction files
+such as AGENTS.md extend this to safety:
+a developer writes a rule like “never print secrets,”
+“ask before deleting data,” or “only push through
+the review branch,” in plain prose, and the agent reads it. But
+this is a write-only channel. In ordinary software, stating a
+requirement comes with feedback that closes the loop: code
+
+numbers use only this rule-word pass. Our released code also
+includes an optional model step that can catch more rules, but
+we did not use it here.
+We measure rule-word-retrieval recall with a separate
+whole-file audit. The reviewer, a safety practitioner, reads
+a simple-random sample of 50 included files in full, blind
+to the candidate list and classifier outputs, and records
+every prospective safety rule that an agent could violate
+through its own action. A second scope pass over the recorded
+rows excludes text that only describes the current architecture
+or mechanism. This leaves 95 in-scope reference
+
+high pre-adjudication agreement as showing the labels can be
+reproduced under that guide, not as proof that the guide itself
+draws the boundary correctly.
+Extraction recall. Rule-word retrieval catches about two-
+thirds of the safety rules in a file. In the 50-file audit it
+finds 63 of 95 in-scope reference segments, a recall of 66.3%
+(file-clustered bootstrap 95% confidence interval [48.8, 82.0]).
+The 32 it misses are rules written as flat statements rather than
+commands, or rules using words outside the rule-word list. This does not change
+the denominator of the main estimate; it means the estimate