paperlint 13.2.0 → 13.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/.github/workflows/ci.yml +10 -0
  2. package/dist/doctor.d.ts.map +1 -1
  3. package/dist/doctor.js +2 -2
  4. package/dist/doctor.js.map +1 -1
  5. package/dist/skills/assess-novelty/scripts/candidate_page.d.ts +1 -0
  6. package/dist/skills/assess-novelty/scripts/novelty_reader_brief.d.ts +25 -0
  7. package/docs/toolchain.md +1 -1
  8. package/eslint.config.mjs +52 -0
  9. package/package.json +2 -1
  10. package/presets/agentverify.md +15 -4
  11. package/skills/assess-novelty/scripts/candidate_page.css +31 -0
  12. package/skills/assess-novelty/scripts/candidate_page.html +934 -0
  13. package/skills/assess-novelty/scripts/candidate_page.js +165 -0
  14. package/skills/assess-novelty/scripts/candidate_page.test.ts +401 -0
  15. package/skills/assess-novelty/scripts/candidate_page.ts +194 -0
  16. package/skills/assess-novelty/scripts/check_novelty.py +230 -0
  17. package/skills/assess-novelty/scripts/check_novelty.test.ts +490 -0
  18. package/skills/assess-novelty/scripts/check_novelty_coverage.test.ts +176 -0
  19. package/skills/assess-novelty/scripts/crop_quote.py +104 -0
  20. package/skills/assess-novelty/scripts/crop_quote.test.ts +130 -0
  21. package/skills/assess-novelty/scripts/fixtures/crop-quote/make_sample.py +42 -0
  22. package/skills/assess-novelty/scripts/fixtures/crop-quote/sample.pdf +106 -0
  23. package/skills/assess-novelty/scripts/fixtures/novelty/CLAIMS.md +11 -0
  24. package/skills/assess-novelty/scripts/fixtures/novelty/paper.tex +10 -0
  25. package/skills/assess-novelty/scripts/fixtures/novelty/siblings/2026-10-27-novelty-refutation.md +8 -0
  26. package/skills/assess-novelty/scripts/fixtures/novelty/siblings/data/doe-preprint.txt +63 -0
  27. package/skills/assess-novelty/scripts/fixtures/novelty/siblings/novelty.json +54 -0
  28. package/skills/assess-novelty/scripts/novelty_candidates.py +136 -0
  29. package/skills/assess-novelty/scripts/novelty_candidates.test.ts +222 -0
  30. package/skills/assess-novelty/scripts/novelty_cells.py +88 -0
  31. package/skills/assess-novelty/scripts/novelty_cells.test.ts +159 -0
  32. package/skills/assess-novelty/scripts/novelty_coverage.py +55 -0
  33. package/skills/assess-novelty/scripts/novelty_reader_brief.js +165 -0
  34. package/skills/assess-novelty/scripts/novelty_reader_brief.test.ts +251 -0
  35. package/skills/assess-novelty/scripts/novelty_reader_brief.ts +215 -0
  36. package/skills/assess-novelty/scripts/quote_match.py +34 -0
  37. package/skills/assess-novelty/scripts/script_edges.test.ts +174 -0
  38. package/src/doctor.ts +4 -2
@@ -0,0 +1,104 @@
1
+ """Find a verbatim passage in a PDF, render its page, highlight the passage and crop to its column.
2
+
3
+ Usage: python3 -I crop_quote.py <pdf> <out.png> "<passage>" [--page N]
4
+ Prints one JSON line: {"page": N, "pages": [every page the passage is on], "words": "the matched words"}.
5
+ Exit 1: the passage is not in the PDF (nothing written). Exit 3: it is on more than one page and --page
6
+ does not pick one (an abstract repeats its body; the first hit is the wrong one as often as the right one).
7
+
8
+ PyMuPDF does the reading and the drawing (it is what CI installs for the PDF harnesses; the runner has no
9
+ poppler, so pdftotext and pdftoppm are not there, measured on run 40927062797).
10
+
11
+ Matching ignores case, punctuation, spacing and line-end hyphenation: the PDF's words and the passage are
12
+ both reduced to letters and digits (NFKC first, so a ligature reads as its letters) and compared as one
13
+ stream. A passage matches whole or not at all; there is no shorter fallback, because a fallback that finds
14
+ "a" place is how a crop lands on the wrong sentence.
15
+ """
16
+
17
+ import argparse
18
+ import json
19
+ from bisect import bisect_right
20
+ from itertools import accumulate
21
+ import re
22
+ import sys
23
+ import unicodedata
24
+
25
+ import pymupdf
26
+
27
+ DPI = 150
28
+ CONTEXT_PT = 60 # points of page shown above and below the passage
29
+
30
+
31
+ def norm(text):
32
+ return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKC", text).lower())
33
+
34
+
35
+ def words(pdf):
36
+ """Every word of the PDF as (page, x0, y0, x1, y1, text), pages counted from 1."""
37
+ with pymupdf.open(pdf) as doc:
38
+ return [(number, x0, y0, x1, y1, text)
39
+ for number, page in enumerate(doc, start=1)
40
+ for x0, y0, x1, y1, text, *_ in page.get_text("words")]
41
+
42
+
43
+ def occurrences(ws, passage):
44
+ """Each match of the passage as the list of word indices it covers."""
45
+ keys = [norm(w[5]) for w in ws]
46
+ starts = list(accumulate((len(k) for k in keys), initial=0))
47
+ stream, needle = "".join(keys), norm(passage)
48
+ if not needle:
49
+ return []
50
+ hits = [m.start() for m in re.finditer(f"(?={re.escape(needle)})", stream)]
51
+ # A match covers the words from the one its first letter falls in to the one its last letter falls in.
52
+ span = [(bisect_right(starts, at) - 1, bisect_right(starts, at + len(needle) - 1) - 1) for at in hits]
53
+ return [[i for i in range(lo, hi + 1) if keys[i]] for lo, hi in span]
54
+
55
+
56
+ def render(pdf, page, sel, ws, out):
57
+ # The column the passage sits in: every word of the page within 40 pt of the passage's left and right edge.
58
+ xs0, xs1 = min(w[1] for w in sel), max(w[3] for w in sel)
59
+ ys0, ys1 = min(w[2] for w in sel), max(w[4] for w in sel)
60
+ col = [w for w in ws if w[0] == page and w[1] >= xs0 - 40 and w[3] <= xs1 + 40]
61
+ cx0 = min([w[1] for w in col] + [xs0]) - 8
62
+ cx1 = max([w[3] for w in col] + [xs1]) + 8
63
+ with pymupdf.open(pdf) as doc:
64
+ pg = doc[page - 1]
65
+ # Drawn on the page in memory only; the PDF on disk is never saved.
66
+ for _, x0, y0, x1, y1, _ in sel:
67
+ pg.draw_rect(pymupdf.Rect(x0 - 0.5, y0 - 0.5, x1 + 0.5, y1 + 0.5),
68
+ color=None, fill=(1, 0.84, 0), fill_opacity=0.43, overlay=True)
69
+ clip = pymupdf.Rect(cx0, ys0 - CONTEXT_PT, cx1, ys1 + CONTEXT_PT) & pg.rect
70
+ pg.get_pixmap(dpi=DPI, clip=clip).save(out)
71
+
72
+
73
+ def main(argv):
74
+ p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
75
+ p.add_argument("pdf")
76
+ p.add_argument("out")
77
+ p.add_argument("passage")
78
+ p.add_argument("--page", type=int)
79
+ a = p.parse_args(argv)
80
+ ws = words(a.pdf)
81
+ found = occurrences(ws, a.passage)
82
+ pages = sorted({ws[m[0]][0] for m in found})
83
+ if not found:
84
+ print(f"NOT FOUND: {a.passage[:80]}", file=sys.stderr)
85
+ return 1
86
+ picked = [m for m in found if a.page is None or ws[m[0]][0] == a.page]
87
+ if not picked:
88
+ print(f"NOT ON PAGE {a.page}: found on pages {pages}", file=sys.stderr)
89
+ return 1
90
+ if len({ws[m[0]][0] for m in picked}) > 1:
91
+ print(f"ON SEVERAL PAGES {pages}: pass --page to pick one", file=sys.stderr)
92
+ return 3
93
+ match = picked[0]
94
+ page = ws[match[0]][0]
95
+ sel = [ws[i] for i in match if ws[i][0] == page]
96
+ if len(sel) < len(match):
97
+ print(f"passage runs past page {page}: highlighted the part on it", file=sys.stderr)
98
+ render(a.pdf, page, sel, ws, a.out)
99
+ print(json.dumps({"page": page, "pages": pages, "words": " ".join(w[5] for w in sel)}, ensure_ascii=False))
100
+ return 0
101
+
102
+
103
+ if __name__ == "__main__":
104
+ sys.exit(main(sys.argv[1:]))
@@ -0,0 +1,130 @@
1
+ /**
2
+ * crop_quote.py: a verbatim passage is found in a PDF by its words, highlighted on its page and cropped.
3
+ * Execution tier: the committed fixtures/crop-quote/sample.pdf (made by make_sample.py beside it), no network.
4
+ * Run: npx vitest run skills/assess-novelty/scripts/crop_quote.test.ts
5
+ *
6
+ * Why: the scratch version matched the first 6 or 3 words when the whole passage did not match, and
7
+ * took the first hit, so a crop of a paper landed on the abstract instead of the method section. Every
8
+ * passage shown to the author as "the paper says this" rests on this match, so a wrong place must be a refusal.
9
+ */
10
+ import { existsSync, mkdtempSync, realpathSync, rmSync } from "node:fs";
11
+ import { spawnSync } from "node:child_process";
12
+ import { tmpdir } from "node:os";
13
+ import { dirname, join } from "node:path";
14
+ import { fileURLToPath } from "node:url";
15
+ import assert from "node:assert/strict";
16
+ import { test } from "vitest";
17
+
18
+ test("crop_quote", () => {
19
+ const here = dirname(fileURLToPath(import.meta.url));
20
+ const TOOL = join(here, "crop_quote.py");
21
+ const PDF = join(here, "fixtures", "crop-quote", "sample.pdf");
22
+ // -E -P is -I without -s. On the CI runner pip installs PyMuPDF into the user's site-packages (it is not
23
+ // root), which -I hides: run 40937989243 failed with neither pymupdf nor fitz importable. -E and -P still keep
24
+ // PYTHON* variables out and the current and script directories off sys.path.
25
+ const PY = ["-E", "-P"];
26
+
27
+ const run = (passage: string, ...extra: string[]) => {
28
+ const dir = realpathSync(mkdtempSync(join(tmpdir(), "crop-quote-")));
29
+ try {
30
+ const out = join(dir, "crop.png");
31
+ const r = spawnSync(
32
+ "python3",
33
+ [...PY, TOOL, PDF, out, passage, ...extra],
34
+ {
35
+ encoding: "utf8",
36
+ },
37
+ );
38
+ // Yellow highlight pixels in the crop: red and green high, blue low.
39
+ const yellow = existsSync(out)
40
+ ? Number(
41
+ spawnSync(
42
+ "python3",
43
+ [
44
+ ...PY,
45
+ "-c",
46
+ "import sys\nimport pymupdf\npx = pymupdf.Pixmap(sys.argv[1]); s, n = px.samples, px.n; " +
47
+ "print(sum(1 for i in range(0, len(s), n) if s[i] > 200 and s[i + 1] > 180 and s[i + 2] < 160))",
48
+ out,
49
+ ],
50
+ { encoding: "utf8" },
51
+ ).stdout.trim(),
52
+ )
53
+ : 0;
54
+ return {
55
+ code: r.status,
56
+ out: r.stdout,
57
+ err: r.stderr,
58
+ written: existsSync(out),
59
+ yellow,
60
+ };
61
+ } finally {
62
+ rmSync(dir, { recursive: true, force: true });
63
+ }
64
+ };
65
+ const json = (s: string) => JSON.parse(s.trim());
66
+
67
+ // ── fires: a passage on one page is found, highlighted and reported with its page ──
68
+ {
69
+ const r = run(
70
+ "We ran Checkstyle before and after applying the initial patch",
71
+ );
72
+ assert.equal(r.code, 0, r.err);
73
+ assert.deepEqual(json(r.out), {
74
+ page: 2,
75
+ pages: [2],
76
+ words: "We ran Checkstyle before and after applying the initial patch",
77
+ });
78
+ assert.ok(
79
+ r.yellow > 500,
80
+ `the crop carries the highlight (yellow pixels: ${r.yellow})`,
81
+ );
82
+ }
83
+
84
+ // ── a word split by a line-end hyphen still matches, and case and punctuation do not matter ──
85
+ {
86
+ const r = run("THE CONFIGURATIONS, of every linter");
87
+ assert.equal(r.code, 0, r.err);
88
+ assert.equal(json(r.out).words, "the configu- rations of every linter");
89
+ }
90
+
91
+ // ── two columns: a sentence over two lines of the left column is read in column order, not across the page ──
92
+ {
93
+ const r = run(
94
+ "Developers can configure most linters through a file that sits next to the code",
95
+ );
96
+ assert.equal(r.code, 0, r.err);
97
+ assert.equal(json(r.out).page, 3);
98
+ }
99
+
100
+ // ── quiet: a passage the PDF does not hold writes nothing ──
101
+ {
102
+ const r = run("Rules written only in prose are broken more often than");
103
+ assert.equal(r.code, 1, "a near miss is NOT FOUND, never a shorter match");
104
+ assert.match(r.err, /NOT FOUND/);
105
+ assert.equal(r.written, false);
106
+ }
107
+
108
+ // ── refuses: the same passage on two pages (abstract and body) without --page ──
109
+ {
110
+ const r = run("Rules written only in prose are broken as often");
111
+ assert.equal(r.code, 3);
112
+ assert.match(r.err, /ON SEVERAL PAGES \[1, 2\]: pass --page/);
113
+ assert.equal(r.written, false);
114
+ }
115
+
116
+ // ── --page picks one, and a page the passage is not on is NOT FOUND ──
117
+ {
118
+ const r = run(
119
+ "Rules written only in prose are broken as often",
120
+ "--page",
121
+ "2",
122
+ );
123
+ assert.equal(r.code, 0, r.err);
124
+ assert.deepEqual(json(r.out).pages, [1, 2]);
125
+ assert.equal(json(r.out).page, 2);
126
+ const wrong = run("We ran Checkstyle before and after", "--page", "1");
127
+ assert.equal(wrong.code, 1);
128
+ assert.match(wrong.err, /NOT ON PAGE 1: found on pages \[2\]/);
129
+ }
130
+ }, 300_000);
@@ -0,0 +1,42 @@
1
+ """Writes sample.pdf, the crop_quote.harness.ts fixture. Run once; the PDF is committed so the test needs no reportlab.
2
+
3
+ Page 1 states a sentence the way an abstract does; page 2 repeats it in the body (the case where the first hit
4
+ is the wrong one), holds one sentence found nowhere else, and splits a word across a line with a hyphen.
5
+ Page 3 has two columns at the same heights, as a conference paper does: a sentence that runs over two lines
6
+ of the left column reads whole only if words keep column order, not top-to-bottom order across the page.
7
+ """
8
+ import sys
9
+
10
+ from reportlab.lib.pagesizes import letter
11
+ from reportlab.pdfgen import canvas
12
+
13
+ PAGES = [
14
+ ["Abstract. Rules written only in prose are broken as often as rules a linter checks.",
15
+ "This sentence fills the first page and is not quoted by any test."],
16
+ ["Introduction. Rules written only in prose are broken as often as rules a linter checks.",
17
+ "We ran Checkstyle before and after applying the initial patch and the final patch.",
18
+ "Most repositories never turn on the configu-",
19
+ "rations of every linter they install, and nobody notices."],
20
+ ]
21
+
22
+ COLUMNS = [
23
+ (72, ["Developers can configure most", "linters through a file that", "sits next to the code."]),
24
+ (320, ["The right column talks", "about something else", "entirely, line by line."]),
25
+ ]
26
+
27
+ c = canvas.Canvas(sys.argv[1], pagesize=letter, invariant=1)
28
+ for lines in PAGES:
29
+ text = c.beginText(72, 720)
30
+ text.setFont("Helvetica", 11)
31
+ for line in lines:
32
+ text.textLine(line)
33
+ c.drawText(text)
34
+ c.showPage()
35
+ for x, lines in COLUMNS:
36
+ text = c.beginText(x, 720)
37
+ text.setFont("Helvetica", 11)
38
+ for line in lines:
39
+ text.textLine(line)
40
+ c.drawText(text)
41
+ c.showPage()
42
+ c.save()
@@ -0,0 +1,106 @@
1
+ %PDF-1.3
2
+ %���� ReportLab Generated PDF document (opensource)
3
+ 1 0 obj
4
+ <<
5
+ /F1 2 0 R
6
+ >>
7
+ endobj
8
+ 2 0 obj
9
+ <<
10
+ /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
11
+ >>
12
+ endobj
13
+ 3 0 obj
14
+ <<
15
+ /Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
16
+ /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
17
+ >> /Rotate 0 /Trans <<
18
+
19
+ >>
20
+ /Type /Page
21
+ >>
22
+ endobj
23
+ 4 0 obj
24
+ <<
25
+ /Contents 10 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
26
+ /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
27
+ >> /Rotate 0 /Trans <<
28
+
29
+ >>
30
+ /Type /Page
31
+ >>
32
+ endobj
33
+ 5 0 obj
34
+ <<
35
+ /Contents 11 0 R /MediaBox [ 0 0 612 792 ] /Parent 8 0 R /Resources <<
36
+ /Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
37
+ >> /Rotate 0 /Trans <<
38
+
39
+ >>
40
+ /Type /Page
41
+ >>
42
+ endobj
43
+ 6 0 obj
44
+ <<
45
+ /PageMode /UseNone /Pages 8 0 R /Type /Catalog
46
+ >>
47
+ endobj
48
+ 7 0 obj
49
+ <<
50
+ /Author (anonymous) /CreationDate (D:20000101000000+00'00') /Creator (anonymous) /Keywords () /ModDate (D:20000101000000+00'00') /Producer (ReportLab PDF Library - \(opensource\))
51
+ /Subject (unspecified) /Title (untitled) /Trapped /False
52
+ >>
53
+ endobj
54
+ 8 0 obj
55
+ <<
56
+ /Count 3 /Kids [ 3 0 R 4 0 R 5 0 R ] /Type /Pages
57
+ >>
58
+ endobj
59
+ 9 0 obj
60
+ <<
61
+ /Filter [ /ASCII85Decode /FlateDecode ] /Length 229
62
+ >>
63
+ stream
64
+ GappXYn"W)$q,<Ri^R<(U`_CE";;Hk&#@%<JM_r1#kdX@"X.+$anL1G%@110SA#5O(5mfrWF9"uFgZre@W*X*2a;C_CG0,n053mr<Ri13\ntQ,o2=Q>N5Tt++@6Ap`;q.@?8<1!+Z(M&e8UB%b#aht`$'j\<8]k/2m-bs=4fO1[L3Kjoo^]0+\%3.+&uE^G8C]`B.0"Mi:5#XB/G5n#7.U4POoqjhK*^0?i~>endstream
65
+ endobj
66
+ 10 0 obj
67
+ <<
68
+ /Filter [ /ASCII85Decode /FlateDecode ] /Length 316
69
+ >>
70
+ stream
71
+ Gaq3!d7sce&:EqJb[WMVQEuG:)GNot`K9jg3Tso9mKN;@p\+`s2@u/_1UFD9Cs#ZCZq'%Oj6f?%p!=apgMNYAn>o[p=+A_T^?c=$n\eCdH/J:[\UGhr4Nph-;hpDo\r<6q6l^b,/dR+ZEHX%X@@-ac*_Zcb3F,OH(rS]!'("iK+a7#7?dRD,Il\iBG\XH6Z$87S`QHO-j/^*l;d9Ck+epRTJcl`A)6mZr<R_77@i+HTc@Y+(c9XFDP(MJ5nWPu`lqM/]I+ZcSs.X&hUbt@dGrb%?j>q&l'B;gQ9>':n.7Ji2Q\ilB@eC*KBm_K~>endstream
72
+ endobj
73
+ 11 0 obj
74
+ <<
75
+ /Filter [ /ASCII85Decode /FlateDecode ] /Length 250
76
+ >>
77
+ stream
78
+ Garo:bAP0N&4Q>@`EcAp66)*>,@JL=5N%2d@djKQPH+HU%Y<&OJNU4q^;o9/:s)\bhKWG+YrXA5$G(rF'H?6L%XKJrlA9C/W=moL82ZOqU:QQ[4aiKO2bFsI$#!d]D)W_lO-r5*0@1*NSIKs:\5.,.H5O$?OgO8;ad+^aI5*]R68Ik#`D?D=Be;O;S[:LS[d.'c\A8ESES(-baOG3;jQMGI)`j)(oEm%O:#s;?`0jG_e+m&IZZ"#d>/U~>endstream
79
+ endobj
80
+ xref
81
+ 0 12
82
+ 0000000000 65535 f
83
+ 0000000061 00000 n
84
+ 0000000092 00000 n
85
+ 0000000199 00000 n
86
+ 0000000392 00000 n
87
+ 0000000586 00000 n
88
+ 0000000780 00000 n
89
+ 0000000848 00000 n
90
+ 0000001109 00000 n
91
+ 0000001180 00000 n
92
+ 0000001499 00000 n
93
+ 0000001906 00000 n
94
+ trailer
95
+ <<
96
+ /ID
97
+ [<1c178198fbdfa51b25995d89d4102043><1c178198fbdfa51b25995d89d4102043>]
98
+ % ReportLab generated PDF document -- digest (opensource)
99
+
100
+ /Info 7 0 R
101
+ /Root 6 0 R
102
+ /Size 12
103
+ >>
104
+ startxref
105
+ 2247
106
+ %%EOF
@@ -0,0 +1,11 @@
1
+ ---
2
+ title: "CLAIMS — fixture for check_novelty.harness.ts"
3
+ tags: [paper, fixture, novelty]
4
+ ---
5
+
6
+ # CLAIMS — fixture for check_novelty.harness.ts
7
+
8
+ | # | Claim | Status | File |
9
+ |---|---|---|---|
10
+ | R1 | The first measurement of a rule extractor's recall in this literature | 🔴 taken (Doe 2026, recall 66.3 %) | `siblings/novelty.json` |
11
+ | P5-1 | The repository's linter does not make 58.3 % of lint-checkable requirements an error | 🟢 ours | `siblings/novelty.json` |
@@ -0,0 +1,10 @@
1
+ % Fixture for check_novelty.harness.ts: sentences shaped like a measurement paper's,
2
+ % with no priority claim on the rule finder's recall.
3
+ \documentclass{article}
4
+ \begin{document}
5
+ We also report the recall of our rule finder: it finds 41.2\% of the directive-like rules in a public
6
+ human-segmented gold set~\cite{gold}.
7
+ In 50 repositories, the repository's own linter does not make 58.3\% of lint-checkable requirements an error.
8
+ The first row of Table~1 lists the rules a linter can check.
9
+ None of the 900 prose blocks we examined names an enforcement mechanism.
10
+ \end{document}
@@ -0,0 +1,8 @@
1
+ ---
2
+ title: "Refutation pass — fixture for check_novelty"
3
+ tags: [paper, fixture, novelty]
4
+ ---
5
+
6
+ # Refutation pass — fixture
7
+
8
+ Stands in for the independent refutation pass a 🟢 claim must point at.
@@ -0,0 +1,63 @@
1
+ When “Avoid” Is Not block: Safety Rules in
2
+ AGENTS.md vs. Built-In Guards
3
+
4
+ Example preprint, 05 Jan 2026
5
+
6
+ Pat Doe
7
+
8
+ Abstract—In AGENTS.md, “avoid” is a natural-language
9
+ instruction that the model interprets. The agent’s block is
10
+ a built-in guard that stops an action before the agent can take
11
+ it. Both can express the same safety goal, but they control
12
+ the agent in different ways. We measure this gap in 481 public
13
+ AGENTS.md files. A model matched the extracted candidate rules
14
+ against the documented guards, and two safety reviewers
15
+ independently checked a sample without seeing the model’s
16
+ answers or each other’s labels. Depending on how closely a guard
17
+ had to match the written rule, only about 4–16%
18
+ of the retrieved safety rules had a matching built-in guard.
19
+ Under the strictest standard the estimate was 4.4% (95% CI:
20
+ 2.6–6.7%), and the two reviewers agreed closely on which rules
21
+ had a match. A manual review of complete files found that our
22
+ extraction method captured 66.3% of eligible safety rules; the
23
+ reported rates therefore apply to the rules it captured. This is
24
+ a usable safety problem: a rules file is a write-only channel. A
25
+ developer writes a safety rule but gets no feedback on whether
26
+ a guard will enforce it. The same plain-text form hides two
27
+ kinds of rule: those a guard or sandbox can
28
+ enforce, and those left to the model to interpret.
29
+
30
+ I. INTRODUCTION
31
+ Language models have lowered the barrier to building
32
+ software. Many people now build applications and coding
33
+ agents with little safety experience, and they increasingly
34
+ direct those agents in natural language. Agent instruction files
35
+ such as AGENTS.md extend this to safety:
36
+ a developer writes a rule like “never print secrets,”
37
+ “ask before deleting data,” or “only push through
38
+ the review branch,” in plain prose, and the agent reads it. But
39
+ this is a write-only channel. In ordinary software, stating a
40
+ requirement comes with feedback that closes the loop: code
41
+
42
+ numbers use only this rule-word pass. Our released code also
43
+ includes an optional model step that can catch more rules, but
44
+ we did not use it here.
45
+ We measure rule-word-retrieval recall with a separate
46
+ whole-file audit. The reviewer, a safety practitioner, reads
47
+ a simple-random sample of 50 included files in full, blind
48
+ to the candidate list and classifier outputs, and records
49
+ every prospective safety rule that an agent could violate
50
+ through its own action. A second scope pass over the recorded
51
+ rows excludes text that only describes the current architecture
52
+ or mechanism. This leaves 95 in-scope reference
53
+
54
+ high pre-adjudication agreement as showing the labels can be
55
+ reproduced under that guide, not as proof that the guide itself
56
+ draws the boundary correctly.
57
+ Extraction recall. Rule-word retrieval catches about two-
58
+ thirds of the safety rules in a file. In the 50-file audit it
59
+ finds 63 of 95 in-scope reference segments, a recall of 66.3%
60
+ (file-clustered bootstrap 95% confidence interval [48.8, 82.0]).
61
+ The 32 it misses are rules written as flat statements rather than
62
+ commands, or rules using words outside the rule-word list. This does not change
63
+ the denominator of the main estimate; it means the estimate
@@ -0,0 +1,54 @@
1
+ {
2
+ "paper": "paper.tex",
3
+ "searched": [
4
+ {"source": "venue-pc", "how": "venue PC list, DBLP 2024-2026 of each member", "date": "2026-10-08"},
5
+ {"source": "prev-edition", "how": "previous edition accepted papers, titles and abstracts", "date": "2026-10-08"},
6
+ {"source": "arxiv-recent", "how": "arXiv cs.SE since 2026-01: AGENTS.md, rules files", "date": "2026-10-08"}
7
+ ],
8
+ "neighbours": [
9
+ {"id": "doe26", "cite": "doe2026rules", "source": "arxiv-recent", "read": "full",
10
+ "text": "siblings/data/doe-preprint.txt", "dates": {"first_public": "2026-01-05"},
11
+ "title": "When \"Avoid\" Is Not block: Safety Rules in AGENTS.md vs. Built-In Guards",
12
+ "url": "https://example.org/preprint/doe-2026",
13
+ "coverage": [{"lines": [1, 63], "quote": "We measure this gap in 481 public AGENTS.md files."}]}
14
+ ],
15
+ "claims": [
16
+ {"id": "R1", "ledger": "R1",
17
+ "sentence": "We report the recall of our rule finder on a human-segmented gold set.",
18
+ "type": "instrument",
19
+ "pieces": {
20
+ "population": "public agent rules files",
21
+ "object": "recall of the rule extractor",
22
+ "measure": "recall against a blind whole-file human audit"
23
+ },
24
+ "cells": {
25
+ "doe26": {
26
+ "population": {"has": "yes", "quote": "We measure this gap in 481 public AGENTS.md files."},
27
+ "object": {"has": "yes", "quote": "our extraction method captured 66.3% of eligible safety rules"},
28
+ "measure": {"has": "yes", "quote": "The reviewer, a safety practitioner, reads a simple-random sample of 50 included files in full"}
29
+ }
30
+ },
31
+ "objection_answer": "The finder's recall is reported as a property of our instrument, not as a first: Doe measured extraction recall (66.3%) before us."
32
+ },
33
+ {"id": "P5-1", "ledger": "P5-1",
34
+ "sentence": "In 50 repositories, the repository's own linter does not make 58.3% of lint-checkable requirements an error.",
35
+ "type": "installed-base",
36
+ "pieces": {
37
+ "population": "public agent rules files",
38
+ "object": "the repository's own committed linter configuration",
39
+ "measure": "share of lint-checkable requirements the linter does not make an error"
40
+ },
41
+ "cells": {
42
+ "doe26": {
43
+ "population": {"has": "yes", "quote": "We measure this gap in 481 public AGENTS.md files."},
44
+ "object": {"has": "no", "quote": null},
45
+ "measure": {"has": "no", "quote": null}
46
+ }
47
+ },
48
+ "objection_answer": "That rules are often unenforced is folklore; this is the first count against each repository's own linter configuration, at a fixed revision.",
49
+ "refutation": {"by": "fable", "date": "2026-10-27", "file": "siblings/2026-10-27-novelty-refutation.md", "verdict": "holds"}
50
+ }
51
+ ],
52
+ "paper_sentences": [],
53
+ "one_new_result": {"claim": "P5-1", "diluted_by": []}
54
+ }