paperlint 7.2.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build.d.ts.map +1 -1
- package/dist/build.js +1 -6
- package/dist/build.js.map +1 -1
- package/dist/cli.d.ts +3 -4
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +13 -42
- package/dist/cli.js.map +1 -1
- package/dist/eslint-rules/latex-language.d.ts.map +1 -1
- package/dist/eslint-rules/latex-language.js +10 -13
- package/dist/eslint-rules/latex-language.js.map +1 -1
- package/dist/eslint-rules/paper-context.js +2 -2
- package/dist/eslint-rules/paper-typography.d.ts.map +1 -1
- package/dist/eslint-rules/paper-typography.js +22 -91
- package/dist/eslint-rules/paper-typography.js.map +1 -1
- package/dist/eslint-rules/papers.d.ts +1 -3
- package/dist/eslint-rules/papers.d.ts.map +1 -1
- package/dist/eslint-rules/papers.js +2 -4
- package/dist/eslint-rules/papers.js.map +1 -1
- package/dist/eslint-rules/tex-build.d.ts.map +1 -1
- package/dist/eslint-rules/tex-build.js +8 -25
- package/dist/eslint-rules/tex-build.js.map +1 -1
- package/dist/init.d.ts +1 -4
- package/dist/init.d.ts.map +1 -1
- package/dist/init.js +3 -9
- package/dist/init.js.map +1 -1
- package/dist/lib/markdown.d.ts +5 -81
- package/dist/lib/markdown.d.ts.map +1 -1
- package/dist/lib/markdown.js +5 -116
- package/dist/lib/markdown.js.map +1 -1
- package/dist/lib/paper-config.d.ts.map +1 -1
- package/dist/lib/paper-config.js +0 -2
- package/dist/lib/paper-config.js.map +1 -1
- package/dist/lib/skill-eval-fixture.d.ts +6 -4
- package/dist/lib/skill-eval-fixture.d.ts.map +1 -1
- package/dist/lib/skill-eval-fixture.js +25 -18
- package/dist/lib/skill-eval-fixture.js.map +1 -1
- package/dist/new-paper.d.ts +5 -11
- package/dist/new-paper.d.ts.map +1 -1
- package/dist/new-paper.js +12 -23
- package/dist/new-paper.js.map +1 -1
- package/dist/paper-files.d.ts +4 -4
- package/dist/paper-prose.d.ts +17 -0
- package/dist/paper-prose.d.ts.map +1 -0
- package/dist/paper-prose.js +46 -0
- package/dist/paper-prose.js.map +1 -0
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +4 -3
- package/dist/structure.js.map +1 -1
- package/dist/types.d.ts +0 -2
- package/dist/types.d.ts.map +1 -1
- package/docs/configuration.md +7 -9
- package/docs/e2e.md +4 -6
- package/docs/prior-art/paper-folder-scaffolding.md +24 -29
- package/docs/rules/tex/claim-provenance.md +0 -1
- package/docs/rules.md +2 -6
- package/eslint-rules/latex-language.harness.mjs +3 -3
- package/eslint-rules/latex-language.ts +10 -13
- package/eslint-rules/paper-context.ts +2 -2
- package/eslint-rules/paper-typography.test.mjs +17 -85
- package/eslint-rules/paper-typography.ts +22 -98
- package/eslint-rules/papers.harness.mjs +1 -1
- package/eslint-rules/papers.ts +2 -4
- package/eslint-rules/tex-build.harness.mjs +2 -2
- package/eslint-rules/tex-build.ts +8 -25
- package/eslint.config.mjs +0 -1
- package/hooks/hooks.harness.ts +15 -15
- package/hooks/paper-edit-guard.hook.mjs +5 -8
- package/hooks/paper-edit-guard.test.ts +12 -3
- package/hooks/paper-skills-nudge.hook.mjs +2 -3
- package/hooks/paper-skills-nudge.test.ts +13 -3
- package/hooks/paper-status-gates.hook.mjs +2 -2
- package/hooks/paper-status-gates.test.ts +14 -2
- package/lib/markdown.test.mjs +5 -60
- package/lib/markdown.ts +9 -136
- package/lib/paper-config.ts +0 -2
- package/lib/skill-checks.test.mjs +4 -1
- package/lib/skill-eval-fixture.ts +25 -18
- package/package.json +1 -3
- package/presets/realm.jsonc +1 -1
- package/scripts/harness-api.frozen.json +0 -5
- package/skills/cold-read-diff/SKILL.md +3 -3
- package/skills/cold-read-diff/SKILL.md.spec.ts +2 -2
- package/skills/grade-paper-writing/SKILL.md +14 -22
- package/skills/grade-paper-writing/SKILL.md.spec.ts +13 -21
- package/skills/grade-paper-writing/__snapshots__/prose-lint.test.ts.snap +40 -26
- package/skills/grade-paper-writing/prose-lint.mjs +108 -278
- package/skills/grade-paper-writing/prose-lint.test.ts +184 -81
- package/skills/paper-adversarial-review/SKILL.md +2 -2
- package/skills/paper-adversarial-review/SKILL.md.spec.ts +1 -1
- package/skills/paper-pipeline/SKILL.md +9 -12
- package/skills/paper-pipeline/SKILL.md.spec.ts +8 -11
- package/skills/paper-pipeline/description-language.eval.mjs +25 -15
- package/skills/paper-pipeline/framing-vs-vocabulary.eval.mjs +25 -15
- package/skills/paper-pipeline/grade-paper-writing-ablation.eval.mjs +25 -15
- package/skills/paper-pipeline/pipeline-firing.eval.mjs +27 -16
- package/skills/paper-pipeline/pipeline-language.eval.mjs +25 -15
- package/skills/paper-pipeline/references/acceptance-gate.md +16 -20
- package/skills/paper-pipeline/references/artifact-checklist.md +4 -5
- package/skills/paper-pipeline/references/pipeline-status-template.md +1 -1
- package/skills/paper-pipeline/scripts/README.md +0 -4
- package/skills/paper-pipeline/scripts/extract-ref-facts.harness.mjs +8 -72
- package/skills/paper-pipeline/scripts/extract-ref-facts.mjs +48 -156
- package/skills/paper-pipeline/scripts/extract-ref-facts.test.mjs +81 -75
- package/skills/paper-pipeline/scripts/generated-code.harness.mjs +3 -3
- package/skills/paper-status/SKILL.md +2 -2
- package/skills/paper-status/SKILL.md.spec.ts +1 -1
- package/skills/render-paper/SKILL.md +2 -2
- package/skills/render-paper/SKILL.md.spec.ts +1 -1
- package/skills/render-paper/extract-pdf-facts.mjs +2 -2
- package/skills/tighten-paper/SKILL.md +29 -42
- package/skills/tighten-paper/SKILL.md.spec.ts +28 -41
- package/skills/verify-citations/scripts/bib-authors.check.test.mjs +0 -13
- package/skills/verify-citations/scripts/bib-authors.cli.test.ts +23 -11
- package/skills/verify-citations/scripts/bib-authors.mjs +8 -93
- package/skills/verify-citations/scripts/bib-authors.test.mjs +1 -44
- package/src/build.harness.mjs +2 -2
- package/src/build.test.ts +1 -1
- package/src/build.ts +1 -6
- package/src/cli.harness.mjs +26 -46
- package/src/cli.test.ts +32 -40
- package/src/cli.ts +14 -54
- package/src/folder-venue-rule.test.ts +1 -1
- package/src/init.test.ts +6 -7
- package/src/init.ts +3 -22
- package/src/lint-fix.test.ts +10 -4
- package/src/new-paper.harness.mjs +9 -28
- package/src/new-paper.test.ts +6 -6
- package/src/new-paper.ts +11 -30
- package/src/paper-files.test.ts +5 -4
- package/src/paper-files.ts +4 -4
- package/src/paper-prose.test.ts +49 -0
- package/src/paper-prose.ts +67 -0
- package/src/parse-args.test.ts +0 -1
- package/src/structure.harness.ts +5 -11
- package/src/structure.ts +4 -3
- package/src/types.ts +0 -2
- package/fixtures/real-markdown-paper/baseline.json +0 -22
- package/fixtures/real-markdown-paper/baseline.ts +0 -70
- package/skills/paper-pipeline/PIPELINE-MAP.md +0 -381
- package/skills/paper-pipeline/pipeline-firing.baseline.json +0 -270
- package/skills/paper-pipeline/references/sweep-2026-08-09-ideal-pipeline.md +0 -585
- package/skills/paper-pipeline/scripts/artifact-coverage.harness.mjs +0 -492
- package/skills/paper-pipeline/scripts/artifact-coverage.mjs +0 -397
- package/skills/paper-pipeline/scripts/artifact-coverage.test.ts +0 -86
- package/skills/paper-pipeline/scripts/check-provenance.mjs +0 -184
- package/skills/paper-pipeline/scripts/population-map.harness.mjs +0 -135
- package/skills/paper-pipeline/scripts/population-map.mjs +0 -188
- package/skills/paper-pipeline/scripts/population-map.test.mjs +0 -154
- package/skills/paper-pipeline/scripts/provenance.harness.mjs +0 -266
- package/skills/paper-pipeline/scripts/round-diff.harness.ts +0 -907
- package/skills/paper-pipeline/scripts/round-diff.mjs +0 -583
- package/skills/paper-pipeline/scripts/round-diff.test.mjs +0 -159
- package/skills/tighten-paper/__snapshots__/structure.test.ts.snap +0 -133
- package/skills/tighten-paper/structure.mjs +0 -364
- package/skills/tighten-paper/structure.test.ts +0 -117
- package/templates/paper/paper.md +0 -4
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* grade-paper-writing-ablation.eval.mjs — WHICH PART OF A DESCRIPTION KILLS A SKILL?
|
|
3
3
|
*
|
|
4
|
+
* ⚠️ THE FIXTURE CHANGED AFTER THE RECORDED RUNS. The paper the model is shown is now `paper.tex`;
|
|
5
|
+
* every result recorded for this eval (here and in `repro/`) was measured on the previous fixture,
|
|
6
|
+
* the same paper in a source form paperlint no longer reads. The next run is a fresh, deliberate
|
|
7
|
+
* measurement — record it, do not compare it against those numbers.
|
|
8
|
+
*
|
|
4
9
|
* Run: node .claude/skills/paper-pipeline/grade-paper-writing-ablation.eval.mjs [--trials N] [--mode main|oracle|preflight|setupdiff] [--arms A0,A6]
|
|
5
10
|
*
|
|
6
11
|
* --mode preflight every guard + the arm-by-arm word-set deltas; spends NO tokens <- run this FIRST
|
|
@@ -415,42 +420,47 @@ if (!["main", "oracle", "preflight", "setupdiff"].includes(MODE))
|
|
|
415
420
|
// Byte-identical to the parent's (and therefore to both of ITS siblings). If it drifts, a difference
|
|
416
421
|
// between this run and the 12% it exists to explain stops being attributable to the descriptions.
|
|
417
422
|
const FIXTURE = {
|
|
418
|
-
"paper/paper.
|
|
419
|
-
"
|
|
423
|
+
"paper/paper.tex": [
|
|
424
|
+
"\\documentclass{article}",
|
|
425
|
+
"\\title{Prose Isn't Policy: Measuring Whether Agent-Config Rules Are Enforceable}",
|
|
426
|
+
"\\begin{document}",
|
|
427
|
+
"\\maketitle",
|
|
420
428
|
"",
|
|
421
|
-
"
|
|
429
|
+
"\\begin{abstract}",
|
|
422
430
|
"Agent configuration files state rules in prose and assume the model obeys them. We compile a",
|
|
423
|
-
"corpus of real rules and measure what fraction can be mechanically enforced. We find that 84
|
|
431
|
+
"corpus of real rules and measure what fraction can be mechanically enforced. We find that 84\\%",
|
|
424
432
|
"of rules in our corpus are enforceable, and that LLM-authored checkers for the remainder leak",
|
|
425
|
-
"silently in 84
|
|
433
|
+
"silently in 84--96\\% of adversarial cases~\\cite{greshake2023}.",
|
|
434
|
+
"\\end{abstract}",
|
|
426
435
|
"",
|
|
427
|
-
"
|
|
436
|
+
"\\section{Introduction}",
|
|
428
437
|
"Every agent harness ships a natural-language rulebook. Nothing checks it. This is the same",
|
|
429
|
-
"mistake as a code comment that claims an invariant no test enforces
|
|
438
|
+
"mistake as a code comment that claims an invariant no test enforces~\\cite{thompson1984}.",
|
|
430
439
|
"",
|
|
431
|
-
"
|
|
440
|
+
"\\section{Method}",
|
|
432
441
|
"We gather rules from public repositories, classify each by enforceability, and build a",
|
|
433
442
|
"two-stage adversarial gate that validates a synthesized rule against a blind gold set.",
|
|
434
443
|
"",
|
|
435
|
-
"
|
|
436
|
-
"See Table
|
|
444
|
+
"\\section{Results}",
|
|
445
|
+
"See Table~1. The headline number is 84\\%.",
|
|
437
446
|
"",
|
|
438
|
-
"
|
|
447
|
+
"\\section{Discussion}",
|
|
439
448
|
"The result generalizes beyond our corpus in the sense that the mechanism is not corpus-specific,",
|
|
440
449
|
"though of course the specific percentages are, and it is important to note in this context that",
|
|
441
450
|
"the framing itself may be what carries, rather than the measurement.",
|
|
442
451
|
"",
|
|
443
|
-
"
|
|
452
|
+
"\\section{Threats to Validity}",
|
|
444
453
|
"Our corpus is drawn from public repositories and may not represent private configurations.",
|
|
445
454
|
"",
|
|
446
|
-
"
|
|
455
|
+
"\\section{Related Work}",
|
|
447
456
|
"TODO",
|
|
448
457
|
"",
|
|
449
|
-
"
|
|
458
|
+
"\\section{Conclusion}",
|
|
450
459
|
"Prose is not policy.",
|
|
460
|
+
"\\end{document}",
|
|
451
461
|
].join("\n"),
|
|
452
462
|
"paper/repro/README.md":
|
|
453
|
-
"# Reproduction artifact\n\n`python3 paper_numbers.py` recomputes every bolded figure in paper.
|
|
463
|
+
"# Reproduction artifact\n\n`python3 paper_numbers.py` recomputes every bolded figure in paper.tex.\n",
|
|
454
464
|
"paper/PIPELINE-STATUS.md":
|
|
455
465
|
"# Pipeline status\n\n| gate | state |\n|---|---|\n| numbers | pass |\n| structure | not run |\n| citations | not run |\n",
|
|
456
466
|
};
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* pipeline-firing.eval.mjs — do the paper-pipeline skills actually FIRE when they should?
|
|
3
3
|
*
|
|
4
|
+
* ⚠️ THE FIXTURE CHANGED AFTER THE RECORDED RUNS. The paper the model is shown is now `paper.tex`;
|
|
5
|
+
* every result recorded for this eval (here and in `repro/`) was measured on the previous fixture,
|
|
6
|
+
* the same paper in a source form paperlint no longer reads. The next run is a fresh, deliberate
|
|
7
|
+
* measurement — record it, do not compare it against those numbers.
|
|
8
|
+
*
|
|
4
9
|
* Run: node .claude/skills/paper-pipeline/pipeline-firing.eval.mjs [flags]
|
|
5
10
|
* --only <skill> run one case (repeatable: --only tighten-paper --only argument-arc)
|
|
6
11
|
* --trials N trials per prompt (default 1)
|
|
@@ -117,7 +122,8 @@
|
|
|
117
122
|
*
|
|
118
123
|
* ─────────────────────────────────────────────────────────────────────────────
|
|
119
124
|
* FIRST RUN — 2026-08-07, sonnet, 64 runs, 1 trial/prompt, 36 competitors, ~$9.55 API-equivalent
|
|
120
|
-
* (billed to a Claude subscription, $0 metered).
|
|
125
|
+
* (billed to a Claude subscription, $0 metered). Its baseline file was dropped when the fixture
|
|
126
|
+
* changed; `--update-baseline` records the next one.
|
|
121
127
|
*
|
|
122
128
|
* skill recall FP-rate precision n
|
|
123
129
|
* tighten-paper 75% 0% 100% 4
|
|
@@ -235,42 +241,47 @@ const GATE = !flag("no-gate");
|
|
|
235
241
|
// therefore a LOWER bound, and a low number for that case is partly an artifact of this gap, not
|
|
236
242
|
// necessarily a description defect. Do not report it as one.
|
|
237
243
|
const FIXTURE = {
|
|
238
|
-
"paper/paper.
|
|
239
|
-
"
|
|
244
|
+
"paper/paper.tex": [
|
|
245
|
+
"\\documentclass{article}",
|
|
246
|
+
"\\title{Prose Isn't Policy: Measuring Whether Agent-Config Rules Are Enforceable}",
|
|
247
|
+
"\\begin{document}",
|
|
248
|
+
"\\maketitle",
|
|
240
249
|
"",
|
|
241
|
-
"
|
|
250
|
+
"\\begin{abstract}",
|
|
242
251
|
"Agent configuration files state rules in prose and assume the model obeys them. We compile a",
|
|
243
|
-
"corpus of real rules and measure what fraction can be mechanically enforced. We find that 84
|
|
252
|
+
"corpus of real rules and measure what fraction can be mechanically enforced. We find that 84\\%",
|
|
244
253
|
"of rules in our corpus are enforceable, and that LLM-authored checkers for the remainder leak",
|
|
245
|
-
"silently in 84
|
|
254
|
+
"silently in 84--96\\% of adversarial cases~\\cite{greshake2023}.",
|
|
255
|
+
"\\end{abstract}",
|
|
246
256
|
"",
|
|
247
|
-
"
|
|
257
|
+
"\\section{Introduction}",
|
|
248
258
|
"Every agent harness ships a natural-language rulebook. Nothing checks it. This is the same",
|
|
249
|
-
"mistake as a code comment that claims an invariant no test enforces
|
|
259
|
+
"mistake as a code comment that claims an invariant no test enforces~\\cite{thompson1984}.",
|
|
250
260
|
"",
|
|
251
|
-
"
|
|
261
|
+
"\\section{Method}",
|
|
252
262
|
"We gather rules from public repositories, classify each by enforceability, and build a",
|
|
253
263
|
"two-stage adversarial gate that validates a synthesized rule against a blind gold set.",
|
|
254
264
|
"",
|
|
255
|
-
"
|
|
256
|
-
"See Table
|
|
265
|
+
"\\section{Results}",
|
|
266
|
+
"See Table~1. The headline number is 84\\%.",
|
|
257
267
|
"",
|
|
258
|
-
"
|
|
268
|
+
"\\section{Discussion}",
|
|
259
269
|
"The result generalizes beyond our corpus in the sense that the mechanism is not corpus-specific,",
|
|
260
270
|
"though of course the specific percentages are, and it is important to note in this context that",
|
|
261
271
|
"the framing itself may be what carries, rather than the measurement.",
|
|
262
272
|
"",
|
|
263
|
-
"
|
|
273
|
+
"\\section{Threats to Validity}",
|
|
264
274
|
"Our corpus is drawn from public repositories and may not represent private configurations.",
|
|
265
275
|
"",
|
|
266
|
-
"
|
|
276
|
+
"\\section{Related Work}",
|
|
267
277
|
"TODO",
|
|
268
278
|
"",
|
|
269
|
-
"
|
|
279
|
+
"\\section{Conclusion}",
|
|
270
280
|
"Prose is not policy.",
|
|
281
|
+
"\\end{document}",
|
|
271
282
|
].join("\n"),
|
|
272
283
|
"paper/repro/README.md":
|
|
273
|
-
"# Reproduction artifact\n\n`python3 paper_numbers.py` recomputes every bolded figure in paper.
|
|
284
|
+
"# Reproduction artifact\n\n`python3 paper_numbers.py` recomputes every bolded figure in paper.tex.\n",
|
|
274
285
|
"paper/PIPELINE-STATUS.md":
|
|
275
286
|
"# Pipeline status\n\n| gate | state |\n|---|---|\n| numbers | pass |\n| structure | not run |\n| citations | not run |\n",
|
|
276
287
|
};
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* pipeline-language.eval.mjs — does prompt LANGUAGE change whether a skill fires?
|
|
3
3
|
*
|
|
4
|
+
* ⚠️ THE FIXTURE CHANGED AFTER THE RECORDED RUNS. The paper the model is shown is now `paper.tex`;
|
|
5
|
+
* every result recorded for this eval (here and in `repro/`) was measured on the previous fixture,
|
|
6
|
+
* the same paper in a source form paperlint no longer reads. The next run is a fresh, deliberate
|
|
7
|
+
* measurement — record it, do not compare it against those numbers.
|
|
8
|
+
*
|
|
4
9
|
* Run: node .claude/skills/paper-pipeline/pipeline-language.eval.mjs [flags]
|
|
5
10
|
* --trials N trials per prompt (default 3)
|
|
6
11
|
* --concurrency N parallel runs (default 3)
|
|
@@ -184,42 +189,47 @@ const CONCURRENCY = Number(val("concurrency", "3"));
|
|
|
184
189
|
// Byte-identical to the sibling eval's FIXTURE on purpose. If the two files drift, a difference
|
|
185
190
|
// between this run and that one stops being attributable to language.
|
|
186
191
|
const FIXTURE = {
|
|
187
|
-
"paper/paper.
|
|
188
|
-
"
|
|
192
|
+
"paper/paper.tex": [
|
|
193
|
+
"\\documentclass{article}",
|
|
194
|
+
"\\title{Prose Isn't Policy: Measuring Whether Agent-Config Rules Are Enforceable}",
|
|
195
|
+
"\\begin{document}",
|
|
196
|
+
"\\maketitle",
|
|
189
197
|
"",
|
|
190
|
-
"
|
|
198
|
+
"\\begin{abstract}",
|
|
191
199
|
"Agent configuration files state rules in prose and assume the model obeys them. We compile a",
|
|
192
|
-
"corpus of real rules and measure what fraction can be mechanically enforced. We find that 84
|
|
200
|
+
"corpus of real rules and measure what fraction can be mechanically enforced. We find that 84\\%",
|
|
193
201
|
"of rules in our corpus are enforceable, and that LLM-authored checkers for the remainder leak",
|
|
194
|
-
"silently in 84
|
|
202
|
+
"silently in 84--96\\% of adversarial cases~\\cite{greshake2023}.",
|
|
203
|
+
"\\end{abstract}",
|
|
195
204
|
"",
|
|
196
|
-
"
|
|
205
|
+
"\\section{Introduction}",
|
|
197
206
|
"Every agent harness ships a natural-language rulebook. Nothing checks it. This is the same",
|
|
198
|
-
"mistake as a code comment that claims an invariant no test enforces
|
|
207
|
+
"mistake as a code comment that claims an invariant no test enforces~\\cite{thompson1984}.",
|
|
199
208
|
"",
|
|
200
|
-
"
|
|
209
|
+
"\\section{Method}",
|
|
201
210
|
"We gather rules from public repositories, classify each by enforceability, and build a",
|
|
202
211
|
"two-stage adversarial gate that validates a synthesized rule against a blind gold set.",
|
|
203
212
|
"",
|
|
204
|
-
"
|
|
205
|
-
"See Table
|
|
213
|
+
"\\section{Results}",
|
|
214
|
+
"See Table~1. The headline number is 84\\%.",
|
|
206
215
|
"",
|
|
207
|
-
"
|
|
216
|
+
"\\section{Discussion}",
|
|
208
217
|
"The result generalizes beyond our corpus in the sense that the mechanism is not corpus-specific,",
|
|
209
218
|
"though of course the specific percentages are, and it is important to note in this context that",
|
|
210
219
|
"the framing itself may be what carries, rather than the measurement.",
|
|
211
220
|
"",
|
|
212
|
-
"
|
|
221
|
+
"\\section{Threats to Validity}",
|
|
213
222
|
"Our corpus is drawn from public repositories and may not represent private configurations.",
|
|
214
223
|
"",
|
|
215
|
-
"
|
|
224
|
+
"\\section{Related Work}",
|
|
216
225
|
"TODO",
|
|
217
226
|
"",
|
|
218
|
-
"
|
|
227
|
+
"\\section{Conclusion}",
|
|
219
228
|
"Prose is not policy.",
|
|
229
|
+
"\\end{document}",
|
|
220
230
|
].join("\n"),
|
|
221
231
|
"paper/repro/README.md":
|
|
222
|
-
"# Reproduction artifact\n\n`python3 paper_numbers.py` recomputes every bolded figure in paper.
|
|
232
|
+
"# Reproduction artifact\n\n`python3 paper_numbers.py` recomputes every bolded figure in paper.tex.\n",
|
|
223
233
|
"paper/PIPELINE-STATUS.md":
|
|
224
234
|
"# Pipeline status\n\n| gate | state |\n|---|---|\n| numbers | pass |\n| structure | not run |\n| citations | not run |\n",
|
|
225
235
|
};
|
|
@@ -70,8 +70,8 @@ drive**.
|
|
|
70
70
|
|
|
71
71
|
"The PDF compiled" needs no taste. "The PDF is a good paper" is nothing but taste. "The number
|
|
72
72
|
recomputes from the committed data" needs no taste. "The number supports the sentence around it"
|
|
73
|
-
needed taste right up until we built
|
|
74
|
-
fourth rung.
|
|
73
|
+
needed taste right up until we built the numbers gate's `requires`/`forbids` guards, which is the
|
|
74
|
+
interesting move — see the fourth rung.
|
|
75
75
|
|
|
76
76
|
### Compound gates: split, never average
|
|
77
77
|
|
|
@@ -119,7 +119,7 @@ it caught a defect the judgement gates had passed:
|
|
|
119
119
|
|
|
120
120
|
| the taste question | the surrogate we built | what it caught |
|
|
121
121
|
| ----------------------------------------------- | --------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- |
|
|
122
|
-
| does this number support this sentence | `
|
|
122
|
+
| does this number support this sentence | `repro/numbers.tsv` guards — the arm word the paragraph must name, the condition it must not claim | the 2026-08-05 abstract defect, 29 hours before a deadline; five judgement gates had passed it |
|
|
123
123
|
| is this quantity really bound to its data | `repro/arm_permutation.py` — permute the arm labels, rebuild, an unchanged quantity was never bound | a whole class the artifact reviewer recomputes and passes, because the number is _correct_ |
|
|
124
124
|
| did the number survive into the delivered paper | `repro/delivered_pdf.py` — read the built PDF back | a superscript minus lost in typesetting |
|
|
125
125
|
| is this reference real | `extract-ref-facts.mjs` + rules `refs/*` — arXiv / CrossRef | nine fabricated titles carrying correct arXiv ids, and one invented award |
|
|
@@ -248,21 +248,19 @@ Everything in `.claude/skills/paper-pipeline/scripts/`, `.claude/lib/` and `<pap
|
|
|
248
248
|
construction — that is why it is trustworthy, and it is not a coincidence but a selection effect: a
|
|
249
249
|
program is what you can write only when the question is Type A.
|
|
250
250
|
|
|
251
|
-
| unit | question
|
|
252
|
-
| ---------------------------------------------------- |
|
|
253
|
-
| `repro/report-submission.py` | body pages, overfull boxes, unresolved refs, dropped characters
|
|
254
|
-
| `repro/check-anon.sh`, `check-figure.sh` | deanonymising strings; does the figure fit the column
|
|
255
|
-
| `
|
|
256
|
-
| `
|
|
257
|
-
| `
|
|
258
|
-
|
|
|
259
|
-
| `
|
|
260
|
-
| `
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
`structure.mjs` and `prose-lint.mjs` deserve their note. They put a **number with a threshold** on
|
|
265
|
-
questions that are Type B — does this section earn its place, is this sentence readable. That is not
|
|
251
|
+
| unit | question | rung |
|
|
252
|
+
| ---------------------------------------------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
|
|
253
|
+
| `repro/report-submission.py` | body pages, overfull boxes, unresolved refs, dropped characters | A (3) |
|
|
254
|
+
| `repro/check-anon.sh`, `check-figure.sh` | deanonymising strings; does the figure fit the column | A (3) |
|
|
255
|
+
| `prose-lint.mjs` | thresholds with published baselines | A (3) — _measuring_ a B question, which is legitimate and is not the same as answering it |
|
|
256
|
+
| rules `refs/*` (facts by `extract-ref-facts.mjs`) | does the registry have this record | A (3), **external adjudicator** |
|
|
257
|
+
| `paper-lint.mjs` | appendix ratio, shaved passages | A (3) |
|
|
258
|
+
| `repro/arm_permutation.py`, `repro/delivered_pdf.py` | is the quantity bound to its arm; did it survive typesetting | A (3), metamorphic |
|
|
259
|
+
| **`repro/paper_numbers.py` + `numbers.tsv`** | — | **rung 1: no PDF exists in which the number is wrong** |
|
|
260
|
+
| `paper-edit-guard.hook.ts` | writing to a paper from Bash | rung 2 — **leaky**, an interpreter reading its program from stdin still gets through (`papers/CLAUDE.md`, 2026-08-06) |
|
|
261
|
+
|
|
262
|
+
`prose-lint.mjs` deserves its note. It puts a **number with a threshold** on a question that is
|
|
263
|
+
Type B — is this sentence readable. That is not
|
|
266
264
|
cheating and it is not acquittal either: a threshold breach is a fact, and the fact drives a human or
|
|
267
265
|
a judgement pass. The failure to avoid is reading a green threshold as an acquittal, which is what
|
|
268
266
|
happened when a prose metric stood at 50.8 against a machine norm of 7.8 and printed as a neutral
|
|
@@ -318,7 +316,5 @@ questions into programs.
|
|
|
318
316
|
## See also
|
|
319
317
|
|
|
320
318
|
- `review-ratchet.md` — what a caught overclaim costs, and the pass that pays it back.
|
|
321
|
-
- `../scripts/round-diff.mjs` — the per-round edit gate: a driving pass may only change what it
|
|
322
|
-
declared, and the whole document's growth is measured every round.
|
|
323
319
|
- the papers tree's own `CLAUDE.md` — the four-rung ladder, and the rule that a rule written in prose is
|
|
324
320
|
a construction somebody did not build.
|
|
@@ -65,15 +65,14 @@ Observed: a paper's released bundle contained **no data at all** for the section
|
|
|
65
65
|
with. Six `harden-paper` passes and five review panels had not noticed. It surfaced because the
|
|
66
66
|
author asked a plain question — _"did we ever actually run the artifact?"_
|
|
67
67
|
|
|
68
|
-
**
|
|
69
|
-
Two checks, deliberately dumb:
|
|
68
|
+
**Check two things, deliberately dumb:**
|
|
70
69
|
|
|
71
70
|
1. every body section that prints a figure is named in the bundle's index (`NUMBERS.md`);
|
|
72
71
|
2. every path the index names exists in the bundle.
|
|
73
72
|
|
|
74
|
-
|
|
75
|
-
which the index never named — so a reviewer following the index would have
|
|
76
|
-
unreleased. Absence and unfindability fail the same way for a reader.
|
|
73
|
+
The first time they were checked they found **three** gaps: one real absence, and two experiments
|
|
74
|
+
whose data shipped but which the index never named — so a reviewer following the index would have
|
|
75
|
+
concluded they were unreleased. Absence and unfindability fail the same way for a reader.
|
|
77
76
|
|
|
78
77
|
## Running `aclpubcheck` locally
|
|
79
78
|
|
|
@@ -103,7 +103,7 @@ existed. Neither is expressible now. Keep new ids lowercase, one word, and say w
|
|
|
103
103
|
---
|
|
104
104
|
|
|
105
105
|
The template itself is a FILE, not this page: `templates/paper/PIPELINE-STATUS.md` at the package
|
|
106
|
-
root (beside `templates/paper/paper.tex`
|
|
106
|
+
root (beside `templates/paper/paper.tex`). `paperlint new <name>` copies it with the paper's
|
|
107
107
|
name filled in, and a project that keeps its own richer scorecard puts one at
|
|
108
108
|
`<papers>/.template/PIPELINE-STATUS.md`, which `paperlint new` prefers. The copy carries no `stages` — a new paper has shipped nothing.
|
|
109
109
|
|
|
@@ -7,12 +7,8 @@ installed package — see `consumer.mjs`).
|
|
|
7
7
|
|
|
8
8
|
| file | what it answers |
|
|
9
9
|
| ----------------------- | ----------------------------------------------------------------------------------------------------- |
|
|
10
|
-
| `artifact-coverage.mjs` | does the released bundle actually contain what the paper points at? |
|
|
11
|
-
| `check-provenance.mjs` | does the paper have a build-time numbers gate, and what does it still not cover? |
|
|
12
10
|
| `extract-ref-facts.mjs` | parse the bibliography, ask the registries, write facts to JSON for the `refs/*` ESLint rules |
|
|
13
11
|
| `generated-code.mjs` | lint the analysis scripts before they produce a number (unseeded randomness, absolute paths, …) |
|
|
14
|
-
| `population-map.mjs` | does the population registry keep its link to the body of the paper? |
|
|
15
|
-
| `round-diff.mjs` | did a review round change only what it declared? |
|
|
16
12
|
| `consumer.mjs` | where the consumer's repository is, and whether a script was executed or imported through the symlink |
|
|
17
13
|
|
|
18
14
|
The checkers with a `--flags-only` mode print nothing on a clean paper and exit 0 — that mode is
|
|
@@ -308,71 +308,12 @@ process.on("exit", () => rmSync(TMP, { recursive: true, force: true }));
|
|
|
308
308
|
);
|
|
309
309
|
}
|
|
310
310
|
|
|
311
|
-
// ── 7.
|
|
312
|
-
{
|
|
313
|
-
const md = [
|
|
314
|
-
"# Paper",
|
|
315
|
-
"",
|
|
316
|
-
"## 1. Intro",
|
|
317
|
-
"",
|
|
318
|
-
"Prose.",
|
|
319
|
-
"",
|
|
320
|
-
"## References",
|
|
321
|
-
"",
|
|
322
|
-
"1. C. Yang, Z. Zhao, L. Zhang. *KNighter: Transforming Static Analysis.* arXiv:2503.09002, 2025.",
|
|
323
|
-
"2. Z. Xiang et al. *Another Work.* ICSE 2024. arXiv:2401.00001",
|
|
324
|
-
"",
|
|
325
|
-
"## Appendix A",
|
|
326
|
-
"",
|
|
327
|
-
"1. Not a reference at all.",
|
|
328
|
-
"",
|
|
329
|
-
].join("\n");
|
|
330
|
-
const es = X.parseMarkdownRefs(md);
|
|
331
|
-
assert.equal(
|
|
332
|
-
es.length,
|
|
333
|
-
2,
|
|
334
|
-
`heading must close the reference list, parsed ${es.length}`,
|
|
335
|
-
);
|
|
336
|
-
assert.equal(
|
|
337
|
-
es[0].line,
|
|
338
|
-
9,
|
|
339
|
-
`line of first entry is ${es[0].line}, but it is ninth in file`,
|
|
340
|
-
);
|
|
341
|
-
assert.equal(es[0].title, "KNighter: Transforming Static Analysis");
|
|
342
|
-
assert.equal(es[1].truncated, true, "`et al.` must be marked as truncation");
|
|
343
|
-
assert.deepEqual(es[1].authors, ["Z. Xiang"]);
|
|
344
|
-
assert.equal(es[1].year, "2024");
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
// ── 8. `## References` inside ```-block does NOT open list ──────────────────
|
|
348
|
-
// Papers in this repo cite their own markup in pieces; the old `split` would open
|
|
349
|
-
// bibliography in the middle of a code example.
|
|
350
|
-
{
|
|
351
|
-
const md = [
|
|
352
|
-
"# P",
|
|
353
|
-
"",
|
|
354
|
-
"## Body",
|
|
355
|
-
"",
|
|
356
|
-
"```md",
|
|
357
|
-
"## References",
|
|
358
|
-
"",
|
|
359
|
-
"1. Fake. *X.* arXiv:1111.11111",
|
|
360
|
-
"```",
|
|
361
|
-
"",
|
|
362
|
-
].join("\n");
|
|
363
|
-
assert.deepEqual(
|
|
364
|
-
X.parseMarkdownRefs(md),
|
|
365
|
-
[],
|
|
366
|
-
"heading inside ```-block is not a heading",
|
|
367
|
-
);
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
// ── 9. CLI: ZERO ENTRIES — REFUSAL, not empty facts ─────────────────────────
|
|
311
|
+
// ── 7. CLI: ZERO ENTRIES — REFUSAL, not empty facts ─────────────────────────
|
|
371
312
|
{
|
|
372
313
|
const { spawnSync } = await import("node:child_process");
|
|
373
314
|
const dir = join(TMP, "empty");
|
|
374
315
|
mkdirSync(dir, { recursive: true });
|
|
375
|
-
writeFileSync(join(dir, "
|
|
316
|
+
writeFileSync(join(dir, "refs.bib"), "% no entries at all\n");
|
|
376
317
|
const r = spawnSync(
|
|
377
318
|
"node",
|
|
378
319
|
[join(HERE, "extract-ref-facts.mjs"), dir, "--offline"],
|
|
@@ -393,7 +334,7 @@ process.on("exit", () => rmSync(TMP, { recursive: true, force: true }));
|
|
|
393
334
|
);
|
|
394
335
|
}
|
|
395
336
|
|
|
396
|
-
// ──
|
|
337
|
+
// ── 8. CLI: no source — also refusal, not silence ─────────────────────────
|
|
397
338
|
{
|
|
398
339
|
const { spawnSync } = await import("node:child_process");
|
|
399
340
|
const dir = join(TMP, "nosrc");
|
|
@@ -410,20 +351,15 @@ process.on("exit", () => rmSync(TMP, { recursive: true, force: true }));
|
|
|
410
351
|
);
|
|
411
352
|
}
|
|
412
353
|
|
|
413
|
-
// ──
|
|
354
|
+
// ── 9. end-to-end: real paper → facts, freshness is computed ───────────────
|
|
414
355
|
{
|
|
415
356
|
const { spawnSync } = await import("node:child_process");
|
|
416
357
|
const dir = join(TMP, "e2e");
|
|
417
358
|
mkdirSync(join(dir, "repro"), { recursive: true });
|
|
418
|
-
const paper =
|
|
419
|
-
"
|
|
420
|
-
""
|
|
421
|
-
|
|
422
|
-
"",
|
|
423
|
-
"1. A. Lovelace, G. Hopper. *A Study.* arXiv:2503.09002, 2025.",
|
|
424
|
-
"",
|
|
425
|
-
].join("\n");
|
|
426
|
-
writeFileSync(join(dir, "paper.md"), paper);
|
|
359
|
+
const paper =
|
|
360
|
+
"@misc{lovelace2025,\n author={Lovelace, Ada and Hopper, Grace},\n title={A Study},\n" +
|
|
361
|
+
" note={arXiv:2503.09002},\n year={2025}}\n";
|
|
362
|
+
writeFileSync(join(dir, "refs.bib"), paper);
|
|
427
363
|
writeFileSync(
|
|
428
364
|
join(dir, "repro", "refs-cache.json"),
|
|
429
365
|
JSON.stringify({
|