paperlint 9.0.0 → 10.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. package/.github/CLAUDE.md +22 -0
  2. package/.github/CLAUDE.md.spec.ts +20 -0
  3. package/CLAUDE.md +59 -515
  4. package/CONTRIBUTING.md +2 -2
  5. package/README.md +68 -128
  6. package/dist/CLAUDE.md.spec.d.ts +3 -0
  7. package/dist/CLAUDE.md.spec.d.ts.map +1 -0
  8. package/dist/CLAUDE.md.spec.js +62 -0
  9. package/dist/CLAUDE.md.spec.js.map +1 -0
  10. package/dist/build.d.ts +1 -1
  11. package/dist/build.js +2 -2
  12. package/dist/cli.d.ts +5 -1
  13. package/dist/cli.d.ts.map +1 -1
  14. package/dist/cli.js +52 -6
  15. package/dist/cli.js.map +1 -1
  16. package/dist/domain/layout-tree.d.ts +19 -0
  17. package/dist/domain/layout-tree.d.ts.map +1 -0
  18. package/dist/domain/layout-tree.js +32 -0
  19. package/dist/domain/layout-tree.js.map +1 -0
  20. package/dist/domain/lookup-cache.d.ts +2 -2
  21. package/dist/domain/lookup-cache.js +2 -2
  22. package/dist/domain/paper-layout.d.ts +81 -0
  23. package/dist/domain/paper-layout.d.ts.map +1 -0
  24. package/dist/domain/paper-layout.js +259 -0
  25. package/dist/domain/paper-layout.js.map +1 -0
  26. package/dist/domain/repro-layout.d.ts +80 -0
  27. package/dist/domain/repro-layout.d.ts.map +1 -0
  28. package/dist/domain/repro-layout.js +247 -0
  29. package/dist/domain/repro-layout.js.map +1 -0
  30. package/dist/domain/scaffold-layout.d.ts +28 -0
  31. package/dist/domain/scaffold-layout.d.ts.map +1 -0
  32. package/dist/domain/scaffold-layout.js +81 -0
  33. package/dist/domain/scaffold-layout.js.map +1 -0
  34. package/dist/eslint-rules/CLAUDE.md.spec.d.ts +3 -0
  35. package/dist/eslint-rules/CLAUDE.md.spec.d.ts.map +1 -0
  36. package/dist/eslint-rules/CLAUDE.md.spec.js +39 -0
  37. package/dist/eslint-rules/CLAUDE.md.spec.js.map +1 -0
  38. package/dist/eslint-rules/install-path-literals.d.ts +1 -1
  39. package/dist/eslint-rules/install-path-literals.js +4 -4
  40. package/dist/eslint-rules/install-path-literals.js.map +1 -1
  41. package/dist/eslint-rules/pdf-last-page-balance.js +1 -1
  42. package/dist/eslint-rules/prereg-coverage.d.ts +98 -0
  43. package/dist/eslint-rules/prereg-coverage.d.ts.map +1 -0
  44. package/dist/eslint-rules/prereg-coverage.js +274 -0
  45. package/dist/eslint-rules/prereg-coverage.js.map +1 -0
  46. package/dist/folder-layout-hook.d.ts +21 -0
  47. package/dist/folder-layout-hook.d.ts.map +1 -0
  48. package/dist/folder-layout-hook.js +117 -0
  49. package/dist/folder-layout-hook.js.map +1 -0
  50. package/dist/folder-layout-rule.d.ts +22 -0
  51. package/dist/folder-layout-rule.d.ts.map +1 -0
  52. package/dist/folder-layout-rule.js +107 -0
  53. package/dist/folder-layout-rule.js.map +1 -0
  54. package/dist/init.js +1 -1
  55. package/dist/init.js.map +1 -1
  56. package/dist/lib/agent-cli-version.js +1 -1
  57. package/dist/lib/agent-cli-version.js.map +1 -1
  58. package/dist/new-paper.d.ts +4 -1
  59. package/dist/new-paper.d.ts.map +1 -1
  60. package/dist/new-paper.js +40 -6
  61. package/dist/new-paper.js.map +1 -1
  62. package/dist/references.d.ts +1 -1
  63. package/dist/references.d.ts.map +1 -1
  64. package/dist/references.js +2 -2
  65. package/dist/references.js.map +1 -1
  66. package/dist/repro-layout-rule.d.ts +16 -0
  67. package/dist/repro-layout-rule.d.ts.map +1 -0
  68. package/dist/repro-layout-rule.js +81 -0
  69. package/dist/repro-layout-rule.js.map +1 -0
  70. package/dist/tex-requirements.js +1 -1
  71. package/dist/toolchain.js +1 -1
  72. package/docs/ci.md +60 -0
  73. package/docs/configuration.md +2 -0
  74. package/docs/e2e.md +3 -3
  75. package/docs/incidents.md +5 -5
  76. package/docs/install.md +20 -16
  77. package/docs/layout.md +154 -0
  78. package/docs/optional-rules.md +5 -4
  79. package/docs/references.md +5 -2
  80. package/docs/rules/paper/allowed-files-only.md +192 -0
  81. package/docs/rules/repro/allowed-files-only.md +128 -0
  82. package/docs/rules/study/declare-sample.md +146 -0
  83. package/docs/rules.md +53 -50
  84. package/docs/skills.md +6 -3
  85. package/docs/venues.md +94 -0
  86. package/eslint-rules/CLAUDE.md +44 -0
  87. package/eslint-rules/CLAUDE.md.spec.ts +43 -0
  88. package/eslint-rules/install-path-literals.harness.mjs +1 -1
  89. package/eslint-rules/install-path-literals.ts +4 -4
  90. package/eslint-rules/pdf-last-page-balance.ts +1 -1
  91. package/eslint-rules/prereg-coverage.test.ts +448 -0
  92. package/eslint-rules/prereg-coverage.ts +419 -0
  93. package/eslint.config.mjs +6 -3
  94. package/hooks/CLAUDE.md +34 -0
  95. package/hooks/CLAUDE.md.spec.ts +30 -0
  96. package/lib/agent-cli-version.ts +1 -1
  97. package/lib/skill-trigger-cases.harness.mjs +51 -5
  98. package/lib/skill-trigger-cases.mjs +53 -15
  99. package/package.json +2 -2
  100. package/plugin/CLAUDE.md +22 -0
  101. package/plugin/CLAUDE.md.spec.ts +22 -0
  102. package/plugin/hooks/hooks.json +9 -0
  103. package/scripts/coverage-config.test.ts +2 -1
  104. package/skills/CLAUDE.md +19 -0
  105. package/skills/CLAUDE.md.spec.ts +18 -0
  106. package/skills/analyze-sibling-paper/SKILL.md +2 -2
  107. package/skills/analyze-sibling-paper/SKILL.md.spec.ts +1 -1
  108. package/skills/build-benchmark/SKILL.md +14 -1
  109. package/skills/build-benchmark/SKILL.md.spec.ts +13 -0
  110. package/skills/build-label-page/SKILL.md +118 -0
  111. package/skills/build-label-page/SKILL.md.spec.ts +129 -0
  112. package/skills/build-label-page/build-label-page.eval.mjs +15 -0
  113. package/skills/build-label-page/build-label-page.test.ts +959 -0
  114. package/skills/build-label-page/references/checklist.md +148 -0
  115. package/skills/build-label-page/references/example-generator.py +803 -0
  116. package/skills/find-venue/SKILL.md +5 -3
  117. package/skills/find-venue/SKILL.md.spec.ts +4 -2
  118. package/skills/grade-paper-writing/SKILL.md +7 -7
  119. package/skills/grade-paper-writing/SKILL.md.spec.ts +6 -6
  120. package/skills/map-prior-work/SKILL.md +2 -2
  121. package/skills/map-prior-work/SKILL.md.spec.ts +1 -1
  122. package/skills/paper-pipeline/SKILL.md +8 -6
  123. package/skills/paper-pipeline/SKILL.md.spec.ts +7 -5
  124. package/skills/paper-pipeline/grade-paper-writing-ablation.eval.mjs +19 -9
  125. package/skills/paper-pipeline/references/acceptance-gate.md +19 -19
  126. package/skills/paper-pipeline/references/artifact-checklist.md +5 -0
  127. package/skills/paper-pipeline/references/pipeline-status-template.md +30 -1
  128. package/skills/paper-pipeline/repro/2026-08-07-fork-check.log +1 -1
  129. package/skills/paper-pipeline/repro/2026-08-07-fork-check2.log +1 -1
  130. package/skills/paper-pipeline/repro/2026-08-07-language-eval-pilot.log +1 -1
  131. package/skills/paper-pipeline/repro/2026-08-07-language-eval-raw.log +1 -1
  132. package/skills/paper-pipeline/repro/2026-08-08-framing-vs-vocabulary-oracle.log +48 -48
  133. package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-A6-oracle.log +8 -8
  134. package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-oracle.log +48 -48
  135. package/skills/paper-pipeline/repro/2026-08-08-parent-replication-gpw.log +2 -2
  136. package/skills/paper-pipeline/repro/2026-08-08-parent-replication.mjs +3 -3
  137. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-after.log +81 -0
  138. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-before.log +81 -0
  139. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-setupdiff.log +59 -0
  140. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.jsonl +1 -0
  141. package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.log +22 -0
  142. package/skills/paper-pipeline/repro/README.md +4 -0
  143. package/skills/paper-pipeline/scripts/extract-ref-facts.harness.mjs +2 -2
  144. package/skills/paper-pipeline/scripts/extract-ref-facts.mjs +2 -2
  145. package/skills/paper-pipeline/scripts/extract-ref-facts.test.mjs +4 -2
  146. package/skills/paper-pipeline/scripts/generated-code.harness.mjs +7 -2
  147. package/skills/paper-pipeline/scripts/generated-code.mjs +8 -4
  148. package/skills/paper-pipeline/scripts/generated-code.test.ts +15 -1
  149. package/skills/paper-status/SKILL.md +2 -2
  150. package/skills/paper-status/SKILL.md.spec.ts +1 -1
  151. package/skills/plan-paper-timeline/plan-paper-timeline.effects.harness.mjs +21 -102
  152. package/skills/study-accepted-papers/SKILL.md +163 -161
  153. package/skills/study-accepted-papers/SKILL.md.spec.ts +167 -165
  154. package/skills/study-accepted-papers/references/base-rates.md +69 -0
  155. package/skills/study-accepted-papers/references/mechanisms.md +181 -0
  156. package/skills/verify-citations/SKILL.md +2 -2
  157. package/skills/verify-citations/SKILL.md.spec.ts +1 -1
  158. package/src/CLAUDE.md +23 -1
  159. package/src/CLAUDE.md.spec.ts +72 -0
  160. package/src/build.ts +2 -2
  161. package/src/cli.harness.mjs +42 -0
  162. package/src/cli.test.ts +90 -6
  163. package/src/cli.ts +78 -5
  164. package/src/domain/layout-tree.test.ts +32 -0
  165. package/src/domain/layout-tree.ts +47 -0
  166. package/src/domain/lookup-cache.ts +2 -2
  167. package/src/domain/paper-layout.test.ts +264 -0
  168. package/src/domain/paper-layout.ts +336 -0
  169. package/src/domain/repro-layout.test.ts +222 -0
  170. package/src/domain/repro-layout.ts +311 -0
  171. package/src/domain/scaffold-layout.ts +104 -0
  172. package/src/folder-layout-hook.test.ts +214 -0
  173. package/src/folder-layout-hook.ts +141 -0
  174. package/src/folder-layout-rule.test.ts +235 -0
  175. package/src/folder-layout-rule.ts +151 -0
  176. package/src/hooks-settings.harness.ts +9 -8
  177. package/src/init.ts +1 -1
  178. package/src/layout-docs.test.ts +52 -0
  179. package/src/new-paper.harness.mjs +23 -1
  180. package/src/new-paper.test.ts +299 -6
  181. package/src/new-paper.ts +51 -5
  182. package/src/paper-files.test.ts +9 -6
  183. package/src/references.test.ts +5 -5
  184. package/src/references.ts +2 -2
  185. package/src/repro-layout-rule.test.ts +114 -0
  186. package/src/repro-layout-rule.ts +121 -0
  187. package/src/rule-cases.test.ts +98 -0
  188. package/src/tex-requirements.ts +1 -1
  189. package/src/toolchain.ts +1 -1
  190. package/templates/paper/PIPELINE-STATUS.md +41 -7
@@ -0,0 +1,192 @@
1
+ # paper/allowed-files-only
2
+
3
+ **Level:** error · **Reads:** the paper folder's root (a directory listing), its `paperlint.json` ·
4
+ **Reported on:** the first line of the paper's `PIPELINE-STATUS.md`, once per stray entry
5
+
6
+ ## What it catches
7
+
8
+ A file or folder at the root of a paper folder that is not part of paperlint's layout. The root
9
+ holds a small, known set; every kind of material has one folder; anything else goes into `extra/`,
10
+ which is never judged.
11
+
12
+ It runs on every `paperlint lint` and in CI, from the moment `paperlint new` creates the folder; a
13
+ hook refuses a new stray entry before it is written. Which stage writes each place, and what checks
14
+ it: [the map](../../layout.md).
15
+
16
+ <!-- layout:begin — drawn from src/domain/paper-layout.ts (PAPER_LAYOUT_TREE); src/layout-docs.test.ts fails when this block differs -->
17
+
18
+ ```text
19
+ <paper>/
20
+ ├── PIPELINE-STATUS.md ← the paper's one live-state file: stages, plan, scorecard
21
+ ├── paper.tex ← the source; `paperlint build` compiles it
22
+ ├── paperlint.json ← the paper's settings
23
+ ├── README.md ← what the paper is
24
+ ├── CLAUDE.md ← the paper's agent rules
25
+ ├── CLAIMS.md ← the claims ledger
26
+ ├── NOTES.md ← each number in the paper → the data it comes from
27
+ ├── SUBMIT-CHECKLIST.md ← the venue's format and submission mechanics
28
+ ├── .gitignore ← keeps the build's byproducts out of git
29
+ ├── paper.pdf ← the built PDF
30
+ ├── *.tex *.bib *.cls *.sty *.bst ← what TeX and bibtex look up beside paper.tex
31
+ ├── _build/ ← the facts `paperlint build` writes
32
+ ├── versions/ ← each stage's frozen PDF and source
33
+ ├── reviews/ ← dated reviews and pass reports
34
+ ├── siblings/ ← the cards of competing papers
35
+ ├── research/ ← dated research and decision records
36
+ ├── repro/ ← the reproduction artifact
37
+ │ ├── README.md ← the index: one line per study
38
+ │ ├── .gitignore ← keeps the scripts' byproducts out of git
39
+ │ ├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)
40
+ │ ├── data/ ← shared data (tables, caches)
41
+ │ └── <slug>-YYYY-MM-DD/ ← one folder per study or spike (<slug>-YYYY-MM for one that ran over weeks)
42
+ │ └── README.md ← required: what it asks, how to run it, what it found
43
+ ├── figures/ ← graphics and their sources
44
+ ├── sections/ ← the body, when it is split
45
+ ├── talk/ ← the talk's files
46
+ ├── submission/ ← forms, portal records, mail with the venue, the anonymized upload
47
+ └── extra/ ← anything; never judged
48
+ ```
49
+
50
+ <!-- layout:end -->
51
+
52
+ `paperlint new` makes the folders a paper uses from its first day: `versions/`, `reviews/`, `research/`,
53
+ `repro/` (with its index, `tools/` and `data/`), `figures/` and `submission/`, each empty one with a
54
+ `.gitkeep` so the layout is in the first commit. It leaves out `siblings/` (made with the first
55
+ competitor's card), `sections/` (when the body is split), `talk/` (when the paper declares a talk),
56
+ `extra/` (never judged) and `_build/` (the build's). No rule reports a `.gitkeep` inside these folders.
57
+
58
+ `extra/` is never judged by any rule, not only this one: `paperlint lint` does not read a folder named
59
+ `extra/`, wherever it is. An archived snapshot of an older paper kept there, with its own
60
+ `PIPELINE-STATUS.md`, is not taken for a second paper.
61
+
62
+ Never reported: the byproducts a TeX run leaves beside `paper.tex` (`paper.aux`, `.log`, `.out`,
63
+ `.toc`, `.lof`, `.lot`, `.lol`, `.loa`, `.fls`, `.xdv`, `.dvi`, `.synctex.gz`; bibtex's `.bbl`, `.blg`;
64
+ makeindex's `.idx`, `.ind`, `.ilg`; glossaries' `.glo`, `.gls`, `.glg`, `.acn`, `.acr`, `.alg`, `.ist`,
65
+ `.nlo`, `.nls`; biber's `.bcf`, `.run.xml`; beamer's `.nav`, `.snm`, `.vrb`; latexmk's `.fdb_latexmk`;
66
+ the list follows latexmk's clean lists; only under the job name `paper`), the `.aux` that
67
+ `\include{chapter}` writes beside a root `chapter.tex`, and the operating system's `.DS_Store` and
68
+ `Thumbs.db`.
69
+
70
+ A venue's corpus of accepted papers is not in the paper folder at all: it is shared by every paper
71
+ that targets the venue, in `<papers-root>/venues/<venue>/` ([what it holds](../../venues.md)).
72
+
73
+ The paper's settings can move three entries, and what they name at the root is allowed while they
74
+ name it: `pdf` (the built PDF under another path), `talk.dir` (another talk folder), and a preset
75
+ file of the paper's own (`extends`, or the `extends` of any of its `cycles`).
76
+
77
+ ## Why
78
+
79
+ A paper folder collects files the way a desk does: a plan, mails, three drafts of the PDF, an old
80
+ attempt's checklist, a corpus. After a few months nobody can say which file is current, where the
81
+ evidence for a decision is, or where a new thing goes, and an agent asked to continue the work reads
82
+ the wrong one. A fixed root is what build tools settled on long ago (Cargo, Maven): a known set, one
83
+ folder per kind of material, and one place for the rest.
84
+
85
+ A structure linter over file names cannot do this from inside ESLint: ESLint is handed only the files
86
+ paperlint lints, so a stray `.pdf`, `.py` or `.md` is never one of them. The rule lists the folder
87
+ itself.
88
+
89
+ ## Examples
90
+
91
+ Failing: a plan beside the paper.
92
+
93
+ ```text
94
+ papers/agent-rule-drift/PIPELINE-STATUS.md
95
+ 1:1 error `PLAN.md` is at the root of a paper folder (the folder holding PIPELINE-STATUS.md),
96
+ and that root holds only paperlint's layout. Where it goes: a section of
97
+ PIPELINE-STATUS.md — the paper's one live-state file holds its stages, its plan
98
+ ("Plan / now") and its scorecard; move what is still live there and delete the file.
99
+ The layout (extra/ takes anything and is never judged; the build's paper.* byproducts are never reported):
100
+ <paper>/
101
+ ├── PIPELINE-STATUS.md ← the paper's one live-state file: stages, plan, scorecard
102
+ …
103
+ └── extra/ ← anything; never judged
104
+ Rule paper/allowed-files-only: https://github.com/zernie/paperlint/blob/main/docs/rules/paper/allowed-files-only.md paper/allowed-files-only
105
+ ```
106
+
107
+ Each message names the entry and where it goes, and draws the whole layout above, so it reads on its own. A
108
+ name met in real paper folders has one destination:
109
+
110
+ | entry | where it goes |
111
+ | -------------------------- | ---------------------------------------------------------------------------------------------------------- |
112
+ | `PLAN.md`, `plans/` | a section of `PIPELINE-STATUS.md` ("Plan / now") |
113
+ | `DECISION-LOG.md` | `research/` |
114
+ | `EMAILS.md` | `submission/` |
115
+ | `data/` | `repro/data/` |
116
+ | `artifact/` | `repro/` (scripts in `repro/tools/`, data in `repro/data/`); the anonymized upload copy in `submission/` |
117
+ | `archive/` | `versions/` for frozen PDFs, `submission/` for an earlier attempt's forms and mails, `extra/` for the rest |
118
+ | `venue-corpus/`, `venues/` | `<papers-root>/venues/<venue>/`, outside the paper folder |
119
+
120
+ Any other name gets the choice by what it is: a record or research → `research/`; about submitting
121
+ → `submission/`; the paper's live state → a section of `PIPELINE-STATUS.md`; reproduction → `repro/`;
122
+ a frozen PDF of a stage → `versions/`; unclear → `extra/`.
123
+
124
+ Passing: the folder `paperlint new` writes (`PIPELINE-STATUS.md`, `paper.tex`, `paperlint.json`),
125
+ and any folder holding only the entries above.
126
+
127
+ ## Options / preset fields
128
+
129
+ No rule options. The paper's `pdf`, `talk.dir` and own preset files add what they name (above).
130
+
131
+ ## What it does not check
132
+
133
+ - What is inside the allowed folders. `reviews/`, `siblings/` and `repro/` have their own rules
134
+ ([`repro/allowed-files-only`](../repro/allowed-files-only.md) for `repro/`); the rest are free.
135
+ - Whether a `.tex` at the root is included by anything: a fragment the body no longer `\input`s
136
+ passes. Old versions of the source belong in `versions/`.
137
+ - A folder without `PIPELINE-STATUS.md`: it is not a paper to paperlint.
138
+
139
+ ## The hook: a new stray entry is refused before it is written
140
+
141
+ `paperlint init` also wires the hook `paper-folder-layout` (PreToolUse on `Edit|Write|MultiEdit`,
142
+ `plugin/hooks/hooks.json`). When an agent is about to create a file that would add an entry the
143
+ layout does not allow — at a paper folder's root, or at the top of its `repro/`
144
+ ([`repro/allowed-files-only`](../repro/allowed-files-only.md)) — the hook denies the call, and the reason it gives is the
145
+ rule's own message: where the entry goes, and the layout drawn. The agent writes the file where it
146
+ belongs instead, in the same turn. The text comes from the functions the rules report with
147
+ (`strayTexts`, `reproTexts`); there is no second list.
148
+
149
+ What it never refuses:
150
+
151
+ - **a file that already exists** — editing is never blocked, wherever the file is, so a paper whose
152
+ folder predates the layout keeps working and the rule reports what is out of place;
153
+ - a new file inside an entry that already exists, even a stray one: the entry is not new, and the
154
+ rule names it once;
155
+ - anything outside a paper folder.
156
+
157
+ A new study folder in `repro/` is started with its `README.md`; any other first file in it is
158
+ refused with `repro/allowed-files-only`'s "without a README.md" message.
159
+
160
+ 🔴 **What it cannot see: the shell.** `mv`, `cp`, `git mv`, or a script that writes files are not an
161
+ Edit or a Write, so the hook never sees them. The guarantee is the lint: both rules fail `paperlint
162
+ lint`, and CI, whatever wrote the entry. The hook only saves the round trip. It does not read the
163
+ paper's `rules` either: a paper that switched a layout rule off is still refused a new stray entry
164
+ (an existing one is never refused).
165
+
166
+ Its first step is cheap: it looks for the nearest folder above the target file that holds
167
+ `PIPELINE-STATUS.md`, one file check per ancestor, and allows the call when there is none — no git,
168
+ no settings, no lint before that. Measured on 2026-10-12: that step takes about 19 µs. The process
169
+ around it is what every `paperlint hook` costs, the CLI's import: about 650 ms per write on the
170
+ measuring machine, with or without a paper folder (`paper-skills-nudge`, a vigiles hook, measured 730
171
+ to 850 ms on the same machine). The hook fails open: a payload it cannot read is reported on stderr
172
+ and the call goes ahead.
173
+
174
+ A project whose papers all live under one folder can skip the process for every other write by
175
+ giving the hook's entry in its own `.claude/settings.json` an `if`, Claude Code's permission-rule
176
+ filter, for example `"if": "Write(**/papers/**)"`. paperlint does not write one: where papers live is
177
+ a project's choice, and a filter that misses a papers folder would make the hook silently dead there.
178
+
179
+ ## How to fix
180
+
181
+ Move the entry where its message says, and update whatever names its path:
182
+
183
+ ```sh
184
+ git mv papers/agent-rule-drift/EMAILS.md papers/agent-rule-drift/submission/EMAILS.md
185
+ ```
186
+
187
+ To keep something as it is, move it under `extra/`. To switch the rule off for one paper, set it in
188
+ that paper's `paperlint.json`:
189
+
190
+ ```json
191
+ { "rules": { "paper/allowed-files-only": "off" } }
192
+ ```
@@ -0,0 +1,128 @@
1
+ # repro/allowed-files-only
2
+
3
+ **Level:** error · **Reads:** the listing of the paper's `repro/`, and whether each study folder in
4
+ it holds a `README.md` · **Reported on:** the first line of the paper's `PIPELINE-STATUS.md`, once
5
+ per misplaced entry
6
+
7
+ ## What it catches
8
+
9
+ An entry at the top of a paper's `repro/` that is not part of its layout, and a study folder there
10
+ without a `README.md`. `repro/` holds an index, the tools and data the whole paper shares, and one
11
+ dated folder per study. Stage 3 of the pipeline, the study, writes here; the rule runs on every
12
+ `paperlint lint` and in CI, and a hook refuses a new loose entry before it is written. The whole map:
13
+ [`layout.md`](../../layout.md).
14
+
15
+ <!-- layout:begin — drawn from src/domain/repro-layout.ts (REPRO_LAYOUT_TREE); src/layout-docs.test.ts fails when this block differs -->
16
+
17
+ ```text
18
+ repro/
19
+ ├── README.md ← the index: one line per study
20
+ ├── .gitignore ← keeps the scripts' byproducts out of git
21
+ ├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)
22
+ ├── data/ ← shared data (tables, caches)
23
+ └── <slug>-YYYY-MM-DD/ ← one folder per study or spike (<slug>-YYYY-MM for one that ran over weeks)
24
+ └── README.md ← required: what it asks, how to run it, what it found
25
+ ```
26
+
27
+ <!-- layout:end -->
28
+
29
+ `paperlint new` makes `repro/README.md` (a stub index), `repro/tools/` and `repro/data/` (each with a
30
+ `.gitkeep` until it holds a file). `repro/` itself gets no `.gitkeep`: its top allows only the entries
31
+ above, and it already holds the index.
32
+
33
+ A loose note belongs in the folder of the study it is about, or in the paper's `research/` when it is
34
+ research or a decision rather than the record of a run.
35
+
36
+ **The date in a study's name** is the day the study began: `ablation-2026-10-12`. A month is
37
+ accepted too, `human-labels-2026-10`: a study that ran over weeks (labelling, reading a corpus) has a
38
+ month as its honest date, and both forms sort by time. The date must be a real one — no month 13, no
39
+ 30 February. The slug is lowercase letters and digits, words joined by single hyphens.
40
+
41
+ Never reported: what interpreters, package managers, test runners and the operating system leave in
42
+ the folder they run in — `__pycache__/`, `.pytest_cache/`, `.mypy_cache/`, `.ruff_cache/`,
43
+ `.ipynb_checkpoints/`, `node_modules/`, `.venv/`, `.DS_Store`, `Thumbs.db`.
44
+
45
+ ## Why
46
+
47
+ `repro/` is where the evidence for every number lives, and it fills up like a desk. One real paper
48
+ had 120 entries at its top: about 70 study folders, some with no date, and about 50 loose scripts,
49
+ data files and notes. Nobody could say which script produces which number, which study is current,
50
+ or where a new run goes. Each kind of material now has one place, and each study says, in its
51
+ `README.md`, what it is.
52
+
53
+ ## Examples
54
+
55
+ Failing: a script loose at the top.
56
+
57
+ ```text
58
+ papers/agent-rule-drift/PIPELINE-STATUS.md
59
+ 1:1 error `repro/paper_numbers.py` is loose at the top of the paper's repro/ folder, which holds
60
+ only an index, the shared tools and data, and one dated folder per study. Where it
61
+ goes: repro/tools/ — the scripts used across the paper (numbers, figures, the anonymity
62
+ check, harnesses); a script only one study runs goes in that study's folder instead.
63
+ What repro/ holds (what interpreters and package managers leave there, like __pycache__/ and node_modules/, is never reported):
64
+ repro/
65
+ ├── README.md ← the index: one line per study
66
+ …
67
+ Rule repro/allowed-files-only: https://github.com/zernie/paperlint/blob/main/docs/rules/repro/allowed-files-only.md repro/allowed-files-only
68
+ ```
69
+
70
+ Each message names the entry and where it goes, and draws the layout above, so it reads on its own. Where
71
+ an entry goes depends on what it is:
72
+
73
+ | entry | where it goes |
74
+ | ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- |
75
+ | a script (`.py`, `.ipynb`, `.sh`, `.js`, `.mjs`, `.ts`, `.r`, `.jl`, `.sql` …), `Makefile`, `requirements.txt`, `package.json` | `repro/tools/`, or the folder of the one study that runs it |
76
+ | data (`.tsv`, `.csv`, `.json`, `.jsonl`, `.yaml`, `.txt`, `.log`, `.parquet`, `.zip` …) | `repro/data/`, or the folder of the one study that produced it |
77
+ | a note (`.md`) | the dated folder of the study it is about, or the paper's `research/` |
78
+ | a folder whose name is not `<slug>-YYYY-MM-DD` | rename it to `<slug>-YYYY-MM-DD` and give it a `README.md`; its first commit's date is a fine default |
79
+ | a study folder without `README.md` | add one: what the study asks, how to run it, what it found; and a line in `repro/README.md` |
80
+ | anything else | by what it is, as above |
81
+
82
+ Three files paperlint itself once kept at the top of `repro/` are named with their new place and the
83
+ command that moves them: `references-cache.json` → `repro/data/` (read by `paperlint build`),
84
+ `refs-cache.json` → `repro/data/`, `generated-code-grandfathered.txt` → `repro/tools/`.
85
+
86
+ To date a folder by its first commit, run in the paper folder:
87
+
88
+ ```sh
89
+ git log --reverse --format=%as -- repro/old-baseline | head -1
90
+ ```
91
+
92
+ Passing: a paper with no `repro/`, and a `repro/` holding only the entries above.
93
+
94
+ ## Options / preset fields
95
+
96
+ No rule options.
97
+
98
+ ## What it does not check
99
+
100
+ - What is inside `tools/`, `data/` and the study folders.
101
+ - Whether `repro/README.md` exists, or has a line for every study.
102
+ - What a study's `README.md` says.
103
+ - `repro/` itself: whether the paper has one is `paper/allowed-files-only`'s concern, and that rule never
104
+ looks inside it. Each entry is judged by exactly one of the two rules.
105
+
106
+ ## The hook
107
+
108
+ The hook `paper-folder-layout` refuses, before it is written, a new file that would add an entry
109
+ this rule reports — a loose file at the top of `repro/`, an undated folder, or a study folder whose
110
+ first file is not its `README.md` — with this rule's message as the reason. It never refuses a file
111
+ that already exists, and it cannot see a shell `mv` or `cp`: the lint is the guarantee. What it does
112
+ and does not do is on [`paper/allowed-files-only`'s page](../paper/allowed-files-only.md#the-hook-a-new-stray-entry-is-refused-before-it-is-written).
113
+
114
+ ## How to fix
115
+
116
+ Move or rename the entry where its message says, and update whatever names its path:
117
+
118
+ ```sh
119
+ git mv papers/agent-rule-drift/repro/paper_numbers.py papers/agent-rule-drift/repro/tools/
120
+ git mv papers/agent-rule-drift/repro/baseline papers/agent-rule-drift/repro/baseline-2026-08-05
121
+ ```
122
+
123
+ To switch the rule off for one paper — say, one whose `repro/` predates the layout — set it in that
124
+ paper's `paperlint.json`; `paper/allowed-files-only` still judges the paper's root:
125
+
126
+ ```json
127
+ { "rules": { "repro/allowed-files-only": "off" } }
128
+ ```
@@ -0,0 +1,146 @@
1
+ # study/declare-sample
2
+
3
+ > Experimental, off by default. Whether this belongs in a lint rule or in the study skills is open:
4
+ > see [#214](https://github.com/zernie/paperlint/issues/214).
5
+
6
+ **Level:** off unless turned on; a paper turns it on with `"rules": { "study/declare-sample": "error" }`
7
+ in its `paperlint.json` · **Reads:** `repro/<study>/prereg.json`, for every study folder at the top of the
8
+ paper's `repro/` that holds one (every folder there but the shared `tools/` and `data/`, whatever
9
+ its name) · **Reported on:** the first line of the paper's
10
+ `PIPELINE-STATUS.md`, once per problem
11
+
12
+ ## What it catches
13
+
14
+ A study's pre-registration that does not say how its sample relates to the population the paper
15
+ talks about. Every `prereg.json` in a study folder (`repro/<slug>-YYYY-MM-DD/`) must carry five
16
+ fields; every other key in the file is the author's and is not read.
17
+
18
+ | field | holds |
19
+ | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
20
+ | `population` | `about`: what the paper's claim is about, in its own words · `source`: where the units come from (census, crawl, registry, with its date) · `size`: how many units, a whole number above 0 (an estimate is fine; say how in `source`) |
21
+ | `strata` | `by`: the property the screens are counted by (rule type, language, …) · `values`: its values, at least one, each once |
22
+ | `screens` | one entry per screen applied between the population and the sample: `name` (what it keeps, as applied) and `removed`, an object with a count for **every** declared stratum value (0 where it removed none) and no other key. `[]` when no screen was applied |
23
+ | `coverage` | `share`: the share of the population the sample can represent, 0 to 1, or `null` with a `reason` · `unreachable`: the strata or kinds of units the sample cannot reach (`[]` only when it reaches them all; a `share` of 1 with a non-empty list is refused) |
24
+ | `author_check` | `who` looked at the drawn units before the run, `when` (YYYY-MM-DD), `units_seen` of `units_drawn` (the units drawn — rules, cases — not the items made from them), and the `verdict` to "is this what you meet in practice?": `representative`, `partly` or `not-representative` |
25
+
26
+ A verdict other than `representative` needs `author_check.decision`: `proceed` (true or false), `by`
27
+ whom, and `why`. A doubtful verdict with no recorded decision is an error; with one, either way, it
28
+ passes: the rule asks that the decision be made and written down, not which way it goes.
29
+
30
+ A study folder without a `prereg.json` owes nothing here.
31
+
32
+ **When it acts.** Inside stage 3, the study: `build-benchmark` writes the `prereg.json` before the
33
+ sample is drawn, the author checks the drawn units, and only then the run and any labelling
34
+ (`build-label-page`) follow. Once a paper turns the rule on, it runs on every `paperlint lint` and in CI, so a study whose
35
+ pre-registration lacks a field fails from the moment the file exists. The flow and the map:
36
+ [`layout.md`](../../layout.md).
37
+
38
+ ## Why
39
+
40
+ A measurement study of how language models read coding rules drew its rules from three pools, kept
41
+ only the rules judgeable from one code excerpt of at most 40 lines, restricted one pool to a
42
+ classifier's "enforceable" lane, and let a model pick short excerpts. Each screen was reasonable
43
+ alone; together they left a sample of crisp rules. The labeller found 93 % of the items designed as
44
+ borderline clear-cut, and the rules looked nothing like the ones met in practice. A spot-check of
45
+ ten items flagged it (seven of ten clear-cut), and the run went ahead.
46
+
47
+ Nothing in that design forced anyone to state what share of the real population the sample covered,
48
+ or which screens removed which kinds of rules. Those are the statements this rule requires, before
49
+ the sample is frozen, in the file that freezes it. The counts per stratum are what make the drift
50
+ visible: a screen that removes 120 crisp rules and 1,630 vague ones is a screen on the construct.
51
+ The author's check is on the drawn **units**, because ten items made from crisp rules look fine one
52
+ by one.
53
+
54
+ ## Examples
55
+
56
+ Failing, with the rule turned on: a pre-registration with hypotheses and arms, and none of the five fields.
57
+
58
+ ```text
59
+ papers/rule-reading/PIPELINE-STATUS.md
60
+ 1:1 error `repro/draw-2026-10-12/prereg.json`: `population` is missing — add `population`: what
61
+ the paper's claim is about (`about`), where its units come from (`source`) and how many
62
+ there are (`size`). Why: a study once screened its sample down to rules that are easy
63
+ to judge — its labeller found 93 % of the items designed as borderline clear-cut — and
64
+ nothing in its design made anyone state what share of the real population the sample
65
+ covered or which screens removed which kinds of units. Rule study/declare-sample: …
66
+ study/declare-sample
67
+ 1:1 error … `strata` is missing …
68
+ 1:1 error … `screens` is missing …
69
+ 1:1 error … `coverage` is missing …
70
+ 1:1 error … `author_check` is missing …
71
+ ```
72
+
73
+ Failing: a screen counted under a value `strata` does not declare, and a doubtful check with no
74
+ decision.
75
+
76
+ ```text
77
+ 1:1 error `repro/draw-2026-10-12/prereg.json`: `screens[0].removed.procedural` is not a declared
78
+ stratum value — `strata.values` declares crisp, parameterised, vague; add it there or
79
+ count it under one of them. …
80
+ 1:1 error `repro/draw-2026-10-12/prereg.json`: `author_check.decision` is missing — the verdict
81
+ is «partly», and a doubtful verdict with no recorded decision is how a sample the author
82
+ doubted gets run anyway: add `proceed` (true or false), `by` whom and `why`. …
83
+ ```
84
+
85
+ Passing:
86
+
87
+ ```json
88
+ {
89
+ "hypotheses": ["…the author's own keys, not read…"],
90
+ "population": {
91
+ "about": "rules written in agent instruction files of public repositories",
92
+ "source": "a census of instruction files, crawled 2026-07-28",
93
+ "size": 4210
94
+ },
95
+ "strata": {
96
+ "by": "rule type",
97
+ "values": ["crisp", "parameterised", "vague"]
98
+ },
99
+ "screens": [
100
+ {
101
+ "name": "judgeable from one code excerpt of at most 40 lines",
102
+ "removed": { "crisp": 120, "parameterised": 410, "vague": 1630 }
103
+ }
104
+ ],
105
+ "coverage": {
106
+ "share": 0.31,
107
+ "unreachable": ["vague", "rules about process rather than code"]
108
+ },
109
+ "author_check": {
110
+ "who": "first author",
111
+ "when": "2026-10-12",
112
+ "units_seen": 24,
113
+ "units_drawn": 24,
114
+ "verdict": "partly",
115
+ "decision": {
116
+ "proceed": true,
117
+ "by": "first author",
118
+ "why": "the claim is narrowed to crisp and parameterised rules; vague rules are named as out of reach"
119
+ }
120
+ }
121
+ }
122
+ ```
123
+
124
+ ## Options / preset fields
125
+
126
+ No rule options.
127
+
128
+ ## What it does not check
129
+
130
+ - Whether the numbers are true: that `removed` adds up to what the pools held, that `share` follows
131
+ from them, or that the author really looked. It checks that each was stated, in a shape a reader
132
+ and a script can use.
133
+ - That the paper's text matches `population.about` or names what `coverage.unreachable` lists. That
134
+ is reading, and the paper skills do it (`build-benchmark`, `draft-paper`'s threats section).
135
+ - A `prereg.json` deeper than a study folder's top (`repro/<study>/sub/prereg.json`), at the top of
136
+ `repro/` itself, or under any other name. The study's folder and its name are `repro/allowed-files-only`'s.
137
+ - A study without a `prereg.json`.
138
+
139
+ ## How to fix
140
+
141
+ Add the five fields to the study's `prereg.json` before the sample is frozen: count what each screen
142
+ removed per stratum value from the screening script's own output, and show the author the drawn
143
+ units with the question "is this what you meet in practice?". A pre-registration already frozen and
144
+ run gets the fields as a dated addendum, with the values measured now; a paper that cannot give them
145
+ leaves the rule off, which is the default; one that turned it on and cannot give them sets it back
146
+ to `"off"` in its `paperlint.json`, which says so in the record.