paperlint 9.0.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/CLAUDE.md +22 -0
- package/.github/CLAUDE.md.spec.ts +20 -0
- package/CLAUDE.md +59 -515
- package/CONTRIBUTING.md +2 -2
- package/README.md +68 -128
- package/dist/CLAUDE.md.spec.d.ts +3 -0
- package/dist/CLAUDE.md.spec.d.ts.map +1 -0
- package/dist/CLAUDE.md.spec.js +62 -0
- package/dist/CLAUDE.md.spec.js.map +1 -0
- package/dist/build.d.ts +1 -1
- package/dist/build.js +2 -2
- package/dist/cli.d.ts +5 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +52 -6
- package/dist/cli.js.map +1 -1
- package/dist/domain/layout-tree.d.ts +19 -0
- package/dist/domain/layout-tree.d.ts.map +1 -0
- package/dist/domain/layout-tree.js +32 -0
- package/dist/domain/layout-tree.js.map +1 -0
- package/dist/domain/lookup-cache.d.ts +2 -2
- package/dist/domain/lookup-cache.js +2 -2
- package/dist/domain/paper-layout.d.ts +81 -0
- package/dist/domain/paper-layout.d.ts.map +1 -0
- package/dist/domain/paper-layout.js +259 -0
- package/dist/domain/paper-layout.js.map +1 -0
- package/dist/domain/repro-layout.d.ts +80 -0
- package/dist/domain/repro-layout.d.ts.map +1 -0
- package/dist/domain/repro-layout.js +247 -0
- package/dist/domain/repro-layout.js.map +1 -0
- package/dist/domain/scaffold-layout.d.ts +28 -0
- package/dist/domain/scaffold-layout.d.ts.map +1 -0
- package/dist/domain/scaffold-layout.js +81 -0
- package/dist/domain/scaffold-layout.js.map +1 -0
- package/dist/eslint-rules/CLAUDE.md.spec.d.ts +3 -0
- package/dist/eslint-rules/CLAUDE.md.spec.d.ts.map +1 -0
- package/dist/eslint-rules/CLAUDE.md.spec.js +39 -0
- package/dist/eslint-rules/CLAUDE.md.spec.js.map +1 -0
- package/dist/eslint-rules/install-path-literals.d.ts +1 -1
- package/dist/eslint-rules/install-path-literals.js +4 -4
- package/dist/eslint-rules/install-path-literals.js.map +1 -1
- package/dist/eslint-rules/pdf-last-page-balance.js +1 -1
- package/dist/eslint-rules/prereg-coverage.d.ts +98 -0
- package/dist/eslint-rules/prereg-coverage.d.ts.map +1 -0
- package/dist/eslint-rules/prereg-coverage.js +274 -0
- package/dist/eslint-rules/prereg-coverage.js.map +1 -0
- package/dist/folder-layout-hook.d.ts +21 -0
- package/dist/folder-layout-hook.d.ts.map +1 -0
- package/dist/folder-layout-hook.js +117 -0
- package/dist/folder-layout-hook.js.map +1 -0
- package/dist/folder-layout-rule.d.ts +22 -0
- package/dist/folder-layout-rule.d.ts.map +1 -0
- package/dist/folder-layout-rule.js +107 -0
- package/dist/folder-layout-rule.js.map +1 -0
- package/dist/init.js +1 -1
- package/dist/init.js.map +1 -1
- package/dist/lib/agent-cli-version.js +1 -1
- package/dist/lib/agent-cli-version.js.map +1 -1
- package/dist/new-paper.d.ts +4 -1
- package/dist/new-paper.d.ts.map +1 -1
- package/dist/new-paper.js +40 -6
- package/dist/new-paper.js.map +1 -1
- package/dist/references.d.ts +1 -1
- package/dist/references.d.ts.map +1 -1
- package/dist/references.js +2 -2
- package/dist/references.js.map +1 -1
- package/dist/repro-layout-rule.d.ts +16 -0
- package/dist/repro-layout-rule.d.ts.map +1 -0
- package/dist/repro-layout-rule.js +81 -0
- package/dist/repro-layout-rule.js.map +1 -0
- package/dist/tex-requirements.js +1 -1
- package/dist/toolchain.js +1 -1
- package/docs/ci.md +60 -0
- package/docs/configuration.md +2 -0
- package/docs/e2e.md +3 -3
- package/docs/incidents.md +5 -5
- package/docs/install.md +20 -16
- package/docs/layout.md +154 -0
- package/docs/optional-rules.md +5 -4
- package/docs/references.md +5 -2
- package/docs/rules/paper/allowed-files-only.md +192 -0
- package/docs/rules/repro/allowed-files-only.md +128 -0
- package/docs/rules/study/declare-sample.md +146 -0
- package/docs/rules.md +53 -50
- package/docs/skills.md +6 -3
- package/docs/venues.md +94 -0
- package/eslint-rules/CLAUDE.md +44 -0
- package/eslint-rules/CLAUDE.md.spec.ts +43 -0
- package/eslint-rules/install-path-literals.harness.mjs +1 -1
- package/eslint-rules/install-path-literals.ts +4 -4
- package/eslint-rules/pdf-last-page-balance.ts +1 -1
- package/eslint-rules/prereg-coverage.test.ts +448 -0
- package/eslint-rules/prereg-coverage.ts +419 -0
- package/eslint.config.mjs +6 -3
- package/hooks/CLAUDE.md +34 -0
- package/hooks/CLAUDE.md.spec.ts +30 -0
- package/lib/agent-cli-version.ts +1 -1
- package/lib/skill-trigger-cases.harness.mjs +51 -5
- package/lib/skill-trigger-cases.mjs +53 -15
- package/package.json +2 -2
- package/plugin/CLAUDE.md +22 -0
- package/plugin/CLAUDE.md.spec.ts +22 -0
- package/plugin/hooks/hooks.json +9 -0
- package/scripts/coverage-config.test.ts +2 -1
- package/skills/CLAUDE.md +19 -0
- package/skills/CLAUDE.md.spec.ts +18 -0
- package/skills/analyze-sibling-paper/SKILL.md +2 -2
- package/skills/analyze-sibling-paper/SKILL.md.spec.ts +1 -1
- package/skills/build-benchmark/SKILL.md +14 -1
- package/skills/build-benchmark/SKILL.md.spec.ts +13 -0
- package/skills/build-label-page/SKILL.md +118 -0
- package/skills/build-label-page/SKILL.md.spec.ts +129 -0
- package/skills/build-label-page/build-label-page.eval.mjs +15 -0
- package/skills/build-label-page/build-label-page.test.ts +959 -0
- package/skills/build-label-page/references/checklist.md +148 -0
- package/skills/build-label-page/references/example-generator.py +803 -0
- package/skills/find-venue/SKILL.md +5 -3
- package/skills/find-venue/SKILL.md.spec.ts +4 -2
- package/skills/grade-paper-writing/SKILL.md +7 -7
- package/skills/grade-paper-writing/SKILL.md.spec.ts +6 -6
- package/skills/map-prior-work/SKILL.md +2 -2
- package/skills/map-prior-work/SKILL.md.spec.ts +1 -1
- package/skills/paper-pipeline/SKILL.md +8 -6
- package/skills/paper-pipeline/SKILL.md.spec.ts +7 -5
- package/skills/paper-pipeline/grade-paper-writing-ablation.eval.mjs +19 -9
- package/skills/paper-pipeline/references/acceptance-gate.md +19 -19
- package/skills/paper-pipeline/references/artifact-checklist.md +5 -0
- package/skills/paper-pipeline/references/pipeline-status-template.md +30 -1
- package/skills/paper-pipeline/repro/2026-08-07-fork-check.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-fork-check2.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-language-eval-pilot.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-07-language-eval-raw.log +1 -1
- package/skills/paper-pipeline/repro/2026-08-08-framing-vs-vocabulary-oracle.log +48 -48
- package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-A6-oracle.log +8 -8
- package/skills/paper-pipeline/repro/2026-08-08-grade-paper-writing-ablation-oracle.log +48 -48
- package/skills/paper-pipeline/repro/2026-08-08-parent-replication-gpw.log +2 -2
- package/skills/paper-pipeline/repro/2026-08-08-parent-replication.mjs +3 -3
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-after.log +81 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-preflight-before.log +81 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-ablation-setupdiff.log +59 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.jsonl +1 -0
- package/skills/paper-pipeline/repro/2026-10-13-grade-paper-writing-trigger.log +22 -0
- package/skills/paper-pipeline/repro/README.md +4 -0
- package/skills/paper-pipeline/scripts/extract-ref-facts.harness.mjs +2 -2
- package/skills/paper-pipeline/scripts/extract-ref-facts.mjs +2 -2
- package/skills/paper-pipeline/scripts/extract-ref-facts.test.mjs +4 -2
- package/skills/paper-pipeline/scripts/generated-code.harness.mjs +7 -2
- package/skills/paper-pipeline/scripts/generated-code.mjs +8 -4
- package/skills/paper-pipeline/scripts/generated-code.test.ts +15 -1
- package/skills/paper-status/SKILL.md +2 -2
- package/skills/paper-status/SKILL.md.spec.ts +1 -1
- package/skills/plan-paper-timeline/plan-paper-timeline.effects.harness.mjs +21 -102
- package/skills/study-accepted-papers/SKILL.md +163 -161
- package/skills/study-accepted-papers/SKILL.md.spec.ts +167 -165
- package/skills/study-accepted-papers/references/base-rates.md +69 -0
- package/skills/study-accepted-papers/references/mechanisms.md +181 -0
- package/skills/verify-citations/SKILL.md +2 -2
- package/skills/verify-citations/SKILL.md.spec.ts +1 -1
- package/src/CLAUDE.md +23 -1
- package/src/CLAUDE.md.spec.ts +72 -0
- package/src/build.ts +2 -2
- package/src/cli.harness.mjs +42 -0
- package/src/cli.test.ts +90 -6
- package/src/cli.ts +78 -5
- package/src/domain/layout-tree.test.ts +32 -0
- package/src/domain/layout-tree.ts +47 -0
- package/src/domain/lookup-cache.ts +2 -2
- package/src/domain/paper-layout.test.ts +264 -0
- package/src/domain/paper-layout.ts +336 -0
- package/src/domain/repro-layout.test.ts +222 -0
- package/src/domain/repro-layout.ts +311 -0
- package/src/domain/scaffold-layout.ts +104 -0
- package/src/folder-layout-hook.test.ts +214 -0
- package/src/folder-layout-hook.ts +141 -0
- package/src/folder-layout-rule.test.ts +235 -0
- package/src/folder-layout-rule.ts +151 -0
- package/src/hooks-settings.harness.ts +9 -8
- package/src/init.ts +1 -1
- package/src/layout-docs.test.ts +52 -0
- package/src/new-paper.harness.mjs +23 -1
- package/src/new-paper.test.ts +299 -6
- package/src/new-paper.ts +51 -5
- package/src/paper-files.test.ts +9 -6
- package/src/references.test.ts +5 -5
- package/src/references.ts +2 -2
- package/src/repro-layout-rule.test.ts +114 -0
- package/src/repro-layout-rule.ts +121 -0
- package/src/rule-cases.test.ts +98 -0
- package/src/tex-requirements.ts +1 -1
- package/src/toolchain.ts +1 -1
- package/templates/paper/PIPELINE-STATUS.md +41 -7
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
# paper/allowed-files-only
|
|
2
|
+
|
|
3
|
+
**Level:** error · **Reads:** the paper folder's root (a directory listing), its `paperlint.json` ·
|
|
4
|
+
**Reported on:** the first line of the paper's `PIPELINE-STATUS.md`, once per stray entry
|
|
5
|
+
|
|
6
|
+
## What it catches
|
|
7
|
+
|
|
8
|
+
A file or folder at the root of a paper folder that is not part of paperlint's layout. The root
|
|
9
|
+
holds a small, known set; every kind of material has one folder; anything else goes into `extra/`,
|
|
10
|
+
which is never judged.
|
|
11
|
+
|
|
12
|
+
It runs on every `paperlint lint` and in CI, from the moment `paperlint new` creates the folder; a
|
|
13
|
+
hook refuses a new stray entry before it is written. Which stage writes each place, and what checks
|
|
14
|
+
it: [the map](../../layout.md).
|
|
15
|
+
|
|
16
|
+
<!-- layout:begin — drawn from src/domain/paper-layout.ts (PAPER_LAYOUT_TREE); src/layout-docs.test.ts fails when this block differs -->
|
|
17
|
+
|
|
18
|
+
```text
|
|
19
|
+
<paper>/
|
|
20
|
+
├── PIPELINE-STATUS.md ← the paper's one live-state file: stages, plan, scorecard
|
|
21
|
+
├── paper.tex ← the source; `paperlint build` compiles it
|
|
22
|
+
├── paperlint.json ← the paper's settings
|
|
23
|
+
├── README.md ← what the paper is
|
|
24
|
+
├── CLAUDE.md ← the paper's agent rules
|
|
25
|
+
├── CLAIMS.md ← the claims ledger
|
|
26
|
+
├── NOTES.md ← each number in the paper → the data it comes from
|
|
27
|
+
├── SUBMIT-CHECKLIST.md ← the venue's format and submission mechanics
|
|
28
|
+
├── .gitignore ← keeps the build's byproducts out of git
|
|
29
|
+
├── paper.pdf ← the built PDF
|
|
30
|
+
├── *.tex *.bib *.cls *.sty *.bst ← what TeX and bibtex look up beside paper.tex
|
|
31
|
+
├── _build/ ← the facts `paperlint build` writes
|
|
32
|
+
├── versions/ ← each stage's frozen PDF and source
|
|
33
|
+
├── reviews/ ← dated reviews and pass reports
|
|
34
|
+
├── siblings/ ← the cards of competing papers
|
|
35
|
+
├── research/ ← dated research and decision records
|
|
36
|
+
├── repro/ ← the reproduction artifact
|
|
37
|
+
│ ├── README.md ← the index: one line per study
|
|
38
|
+
│ ├── .gitignore ← keeps the scripts' byproducts out of git
|
|
39
|
+
│ ├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
40
|
+
│ ├── data/ ← shared data (tables, caches)
|
|
41
|
+
│ └── <slug>-YYYY-MM-DD/ ← one folder per study or spike (<slug>-YYYY-MM for one that ran over weeks)
|
|
42
|
+
│ └── README.md ← required: what it asks, how to run it, what it found
|
|
43
|
+
├── figures/ ← graphics and their sources
|
|
44
|
+
├── sections/ ← the body, when it is split
|
|
45
|
+
├── talk/ ← the talk's files
|
|
46
|
+
├── submission/ ← forms, portal records, mail with the venue, the anonymized upload
|
|
47
|
+
└── extra/ ← anything; never judged
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
<!-- layout:end -->
|
|
51
|
+
|
|
52
|
+
`paperlint new` makes the folders a paper uses from its first day: `versions/`, `reviews/`, `research/`,
|
|
53
|
+
`repro/` (with its index, `tools/` and `data/`), `figures/` and `submission/`, each empty one with a
|
|
54
|
+
`.gitkeep` so the layout is in the first commit. It leaves out `siblings/` (made with the first
|
|
55
|
+
competitor's card), `sections/` (when the body is split), `talk/` (when the paper declares a talk),
|
|
56
|
+
`extra/` (never judged) and `_build/` (the build's). No rule reports a `.gitkeep` inside these folders.
|
|
57
|
+
|
|
58
|
+
`extra/` is never judged by any rule, not only this one: `paperlint lint` does not read a folder named
|
|
59
|
+
`extra/`, wherever it is. An archived snapshot of an older paper kept there, with its own
|
|
60
|
+
`PIPELINE-STATUS.md`, is not taken for a second paper.
|
|
61
|
+
|
|
62
|
+
Never reported: the byproducts a TeX run leaves beside `paper.tex` (`paper.aux`, `.log`, `.out`,
|
|
63
|
+
`.toc`, `.lof`, `.lot`, `.lol`, `.loa`, `.fls`, `.xdv`, `.dvi`, `.synctex.gz`; bibtex's `.bbl`, `.blg`;
|
|
64
|
+
makeindex's `.idx`, `.ind`, `.ilg`; glossaries' `.glo`, `.gls`, `.glg`, `.acn`, `.acr`, `.alg`, `.ist`,
|
|
65
|
+
`.nlo`, `.nls`; biber's `.bcf`, `.run.xml`; beamer's `.nav`, `.snm`, `.vrb`; latexmk's `.fdb_latexmk`;
|
|
66
|
+
the list follows latexmk's clean lists; only under the job name `paper`), the `.aux` that
|
|
67
|
+
`\include{chapter}` writes beside a root `chapter.tex`, and the operating system's `.DS_Store` and
|
|
68
|
+
`Thumbs.db`.
|
|
69
|
+
|
|
70
|
+
A venue's corpus of accepted papers is not in the paper folder at all: it is shared by every paper
|
|
71
|
+
that targets the venue, in `<papers-root>/venues/<venue>/` ([what it holds](../../venues.md)).
|
|
72
|
+
|
|
73
|
+
The paper's settings can move three entries, and what they name at the root is allowed while they
|
|
74
|
+
name it: `pdf` (the built PDF under another path), `talk.dir` (another talk folder), and a preset
|
|
75
|
+
file of the paper's own (`extends`, or the `extends` of any of its `cycles`).
|
|
76
|
+
|
|
77
|
+
## Why
|
|
78
|
+
|
|
79
|
+
A paper folder collects files the way a desk does: a plan, mails, three drafts of the PDF, an old
|
|
80
|
+
attempt's checklist, a corpus. After a few months nobody can say which file is current, where the
|
|
81
|
+
evidence for a decision is, or where a new thing goes, and an agent asked to continue the work reads
|
|
82
|
+
the wrong one. A fixed root is what build tools settled on long ago (Cargo, Maven): a known set, one
|
|
83
|
+
folder per kind of material, and one place for the rest.
|
|
84
|
+
|
|
85
|
+
A structure linter over file names cannot do this from inside ESLint: ESLint is handed only the files
|
|
86
|
+
paperlint lints, so a stray `.pdf`, `.py` or `.md` is never one of them. The rule lists the folder
|
|
87
|
+
itself.
|
|
88
|
+
|
|
89
|
+
## Examples
|
|
90
|
+
|
|
91
|
+
Failing: a plan beside the paper.
|
|
92
|
+
|
|
93
|
+
```text
|
|
94
|
+
papers/agent-rule-drift/PIPELINE-STATUS.md
|
|
95
|
+
1:1 error `PLAN.md` is at the root of a paper folder (the folder holding PIPELINE-STATUS.md),
|
|
96
|
+
and that root holds only paperlint's layout. Where it goes: a section of
|
|
97
|
+
PIPELINE-STATUS.md — the paper's one live-state file holds its stages, its plan
|
|
98
|
+
("Plan / now") and its scorecard; move what is still live there and delete the file.
|
|
99
|
+
The layout (extra/ takes anything and is never judged; the build's paper.* byproducts are never reported):
|
|
100
|
+
<paper>/
|
|
101
|
+
├── PIPELINE-STATUS.md ← the paper's one live-state file: stages, plan, scorecard
|
|
102
|
+
…
|
|
103
|
+
└── extra/ ← anything; never judged
|
|
104
|
+
Rule paper/allowed-files-only: https://github.com/zernie/paperlint/blob/main/docs/rules/paper/allowed-files-only.md paper/allowed-files-only
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Each message names the entry and where it goes, and draws the whole layout above, so it reads on its own. A
|
|
108
|
+
name met in real paper folders has one destination:
|
|
109
|
+
|
|
110
|
+
| entry | where it goes |
|
|
111
|
+
| -------------------------- | ---------------------------------------------------------------------------------------------------------- |
|
|
112
|
+
| `PLAN.md`, `plans/` | a section of `PIPELINE-STATUS.md` ("Plan / now") |
|
|
113
|
+
| `DECISION-LOG.md` | `research/` |
|
|
114
|
+
| `EMAILS.md` | `submission/` |
|
|
115
|
+
| `data/` | `repro/data/` |
|
|
116
|
+
| `artifact/` | `repro/` (scripts in `repro/tools/`, data in `repro/data/`); the anonymized upload copy in `submission/` |
|
|
117
|
+
| `archive/` | `versions/` for frozen PDFs, `submission/` for an earlier attempt's forms and mails, `extra/` for the rest |
|
|
118
|
+
| `venue-corpus/`, `venues/` | `<papers-root>/venues/<venue>/`, outside the paper folder |
|
|
119
|
+
|
|
120
|
+
Any other name gets the choice by what it is: a record or research → `research/`; about submitting
|
|
121
|
+
→ `submission/`; the paper's live state → a section of `PIPELINE-STATUS.md`; reproduction → `repro/`;
|
|
122
|
+
a frozen PDF of a stage → `versions/`; unclear → `extra/`.
|
|
123
|
+
|
|
124
|
+
Passing: the folder `paperlint new` writes (`PIPELINE-STATUS.md`, `paper.tex`, `paperlint.json`),
|
|
125
|
+
and any folder holding only the entries above.
|
|
126
|
+
|
|
127
|
+
## Options / preset fields
|
|
128
|
+
|
|
129
|
+
No rule options. The paper's `pdf`, `talk.dir` and own preset files add what they name (above).
|
|
130
|
+
|
|
131
|
+
## What it does not check
|
|
132
|
+
|
|
133
|
+
- What is inside the allowed folders. `reviews/`, `siblings/` and `repro/` have their own rules
|
|
134
|
+
([`repro/allowed-files-only`](../repro/allowed-files-only.md) for `repro/`); the rest are free.
|
|
135
|
+
- Whether a `.tex` at the root is included by anything: a fragment the body no longer `\input`s
|
|
136
|
+
passes. Old versions of the source belong in `versions/`.
|
|
137
|
+
- A folder without `PIPELINE-STATUS.md`: it is not a paper to paperlint.
|
|
138
|
+
|
|
139
|
+
## The hook: a new stray entry is refused before it is written
|
|
140
|
+
|
|
141
|
+
`paperlint init` also wires the hook `paper-folder-layout` (PreToolUse on `Edit|Write|MultiEdit`,
|
|
142
|
+
`plugin/hooks/hooks.json`). When an agent is about to create a file that would add an entry the
|
|
143
|
+
layout does not allow — at a paper folder's root, or at the top of its `repro/`
|
|
144
|
+
([`repro/allowed-files-only`](../repro/allowed-files-only.md)) — the hook denies the call, and the reason it gives is the
|
|
145
|
+
rule's own message: where the entry goes, and the layout drawn. The agent writes the file where it
|
|
146
|
+
belongs instead, in the same turn. The text comes from the functions the rules report with
|
|
147
|
+
(`strayTexts`, `reproTexts`); there is no second list.
|
|
148
|
+
|
|
149
|
+
What it never refuses:
|
|
150
|
+
|
|
151
|
+
- **a file that already exists** — editing is never blocked, wherever the file is, so a paper whose
|
|
152
|
+
folder predates the layout keeps working and the rule reports what is out of place;
|
|
153
|
+
- a new file inside an entry that already exists, even a stray one: the entry is not new, and the
|
|
154
|
+
rule names it once;
|
|
155
|
+
- anything outside a paper folder.
|
|
156
|
+
|
|
157
|
+
A new study folder in `repro/` is started with its `README.md`; any other first file in it is
|
|
158
|
+
refused with `repro/allowed-files-only`'s "without a README.md" message.
|
|
159
|
+
|
|
160
|
+
🔴 **What it cannot see: the shell.** `mv`, `cp`, `git mv`, or a script that writes files are not an
|
|
161
|
+
Edit or a Write, so the hook never sees them. The guarantee is the lint: both rules fail `paperlint
|
|
162
|
+
lint`, and CI, whatever wrote the entry. The hook only saves the round trip. It does not read the
|
|
163
|
+
paper's `rules` either: a paper that switched a layout rule off is still refused a new stray entry
|
|
164
|
+
(an existing one is never refused).
|
|
165
|
+
|
|
166
|
+
Its first step is cheap: it looks for the nearest folder above the target file that holds
|
|
167
|
+
`PIPELINE-STATUS.md`, one file check per ancestor, and allows the call when there is none — no git,
|
|
168
|
+
no settings, no lint before that. Measured on 2026-10-12: that step takes about 19 µs. The process
|
|
169
|
+
around it is what every `paperlint hook` costs, the CLI's import: about 650 ms per write on the
|
|
170
|
+
measuring machine, with or without a paper folder (`paper-skills-nudge`, a vigiles hook, measured 730
|
|
171
|
+
to 850 ms on the same machine). The hook fails open: a payload it cannot read is reported on stderr
|
|
172
|
+
and the call goes ahead.
|
|
173
|
+
|
|
174
|
+
A project whose papers all live under one folder can skip the process for every other write by
|
|
175
|
+
giving the hook's entry in its own `.claude/settings.json` an `if`, Claude Code's permission-rule
|
|
176
|
+
filter, for example `"if": "Write(**/papers/**)"`. paperlint does not write one: where papers live is
|
|
177
|
+
a project's choice, and a filter that misses a papers folder would make the hook silently dead there.
|
|
178
|
+
|
|
179
|
+
## How to fix
|
|
180
|
+
|
|
181
|
+
Move the entry where its message says, and update whatever names its path:
|
|
182
|
+
|
|
183
|
+
```sh
|
|
184
|
+
git mv papers/agent-rule-drift/EMAILS.md papers/agent-rule-drift/submission/EMAILS.md
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
To keep something as it is, move it under `extra/`. To switch the rule off for one paper, set it in
|
|
188
|
+
that paper's `paperlint.json`:
|
|
189
|
+
|
|
190
|
+
```json
|
|
191
|
+
{ "rules": { "paper/allowed-files-only": "off" } }
|
|
192
|
+
```
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# repro/allowed-files-only
|
|
2
|
+
|
|
3
|
+
**Level:** error · **Reads:** the listing of the paper's `repro/`, and whether each study folder in
|
|
4
|
+
it holds a `README.md` · **Reported on:** the first line of the paper's `PIPELINE-STATUS.md`, once
|
|
5
|
+
per misplaced entry
|
|
6
|
+
|
|
7
|
+
## What it catches
|
|
8
|
+
|
|
9
|
+
An entry at the top of a paper's `repro/` that is not part of its layout, and a study folder there
|
|
10
|
+
without a `README.md`. `repro/` holds an index, the tools and data the whole paper shares, and one
|
|
11
|
+
dated folder per study. Stage 3 of the pipeline, the study, writes here; the rule runs on every
|
|
12
|
+
`paperlint lint` and in CI, and a hook refuses a new loose entry before it is written. The whole map:
|
|
13
|
+
[`layout.md`](../../layout.md).
|
|
14
|
+
|
|
15
|
+
<!-- layout:begin — drawn from src/domain/repro-layout.ts (REPRO_LAYOUT_TREE); src/layout-docs.test.ts fails when this block differs -->
|
|
16
|
+
|
|
17
|
+
```text
|
|
18
|
+
repro/
|
|
19
|
+
├── README.md ← the index: one line per study
|
|
20
|
+
├── .gitignore ← keeps the scripts' byproducts out of git
|
|
21
|
+
├── tools/ ← scripts used across the paper (numbers, figures, anonymity check, harnesses)
|
|
22
|
+
├── data/ ← shared data (tables, caches)
|
|
23
|
+
└── <slug>-YYYY-MM-DD/ ← one folder per study or spike (<slug>-YYYY-MM for one that ran over weeks)
|
|
24
|
+
└── README.md ← required: what it asks, how to run it, what it found
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
<!-- layout:end -->
|
|
28
|
+
|
|
29
|
+
`paperlint new` makes `repro/README.md` (a stub index), `repro/tools/` and `repro/data/` (each with a
|
|
30
|
+
`.gitkeep` until it holds a file). `repro/` itself gets no `.gitkeep`: its top allows only the entries
|
|
31
|
+
above, and it already holds the index.
|
|
32
|
+
|
|
33
|
+
A loose note belongs in the folder of the study it is about, or in the paper's `research/` when it is
|
|
34
|
+
research or a decision rather than the record of a run.
|
|
35
|
+
|
|
36
|
+
**The date in a study's name** is the day the study began: `ablation-2026-10-12`. A month is
|
|
37
|
+
accepted too, `human-labels-2026-10`: a study that ran over weeks (labelling, reading a corpus) has a
|
|
38
|
+
month as its honest date, and both forms sort by time. The date must be a real one — no month 13, no
|
|
39
|
+
30 February. The slug is lowercase letters and digits, words joined by single hyphens.
|
|
40
|
+
|
|
41
|
+
Never reported: what interpreters, package managers, test runners and the operating system leave in
|
|
42
|
+
the folder they run in — `__pycache__/`, `.pytest_cache/`, `.mypy_cache/`, `.ruff_cache/`,
|
|
43
|
+
`.ipynb_checkpoints/`, `node_modules/`, `.venv/`, `.DS_Store`, `Thumbs.db`.
|
|
44
|
+
|
|
45
|
+
## Why
|
|
46
|
+
|
|
47
|
+
`repro/` is where the evidence for every number lives, and it fills up like a desk. One real paper
|
|
48
|
+
had 120 entries at its top: about 70 study folders, some with no date, and about 50 loose scripts,
|
|
49
|
+
data files and notes. Nobody could say which script produces which number, which study is current,
|
|
50
|
+
or where a new run goes. Each kind of material now has one place, and each study says, in its
|
|
51
|
+
`README.md`, what it is.
|
|
52
|
+
|
|
53
|
+
## Examples
|
|
54
|
+
|
|
55
|
+
Failing: a script loose at the top.
|
|
56
|
+
|
|
57
|
+
```text
|
|
58
|
+
papers/agent-rule-drift/PIPELINE-STATUS.md
|
|
59
|
+
1:1 error `repro/paper_numbers.py` is loose at the top of the paper's repro/ folder, which holds
|
|
60
|
+
only an index, the shared tools and data, and one dated folder per study. Where it
|
|
61
|
+
goes: repro/tools/ — the scripts used across the paper (numbers, figures, the anonymity
|
|
62
|
+
check, harnesses); a script only one study runs goes in that study's folder instead.
|
|
63
|
+
What repro/ holds (what interpreters and package managers leave there, like __pycache__/ and node_modules/, is never reported):
|
|
64
|
+
repro/
|
|
65
|
+
├── README.md ← the index: one line per study
|
|
66
|
+
…
|
|
67
|
+
Rule repro/allowed-files-only: https://github.com/zernie/paperlint/blob/main/docs/rules/repro/allowed-files-only.md repro/allowed-files-only
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Each message names the entry and where it goes, and draws the layout above, so it reads on its own. Where
|
|
71
|
+
an entry goes depends on what it is:
|
|
72
|
+
|
|
73
|
+
| entry | where it goes |
|
|
74
|
+
| ------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------- |
|
|
75
|
+
| a script (`.py`, `.ipynb`, `.sh`, `.js`, `.mjs`, `.ts`, `.r`, `.jl`, `.sql` …), `Makefile`, `requirements.txt`, `package.json` | `repro/tools/`, or the folder of the one study that runs it |
|
|
76
|
+
| data (`.tsv`, `.csv`, `.json`, `.jsonl`, `.yaml`, `.txt`, `.log`, `.parquet`, `.zip` …) | `repro/data/`, or the folder of the one study that produced it |
|
|
77
|
+
| a note (`.md`) | the dated folder of the study it is about, or the paper's `research/` |
|
|
78
|
+
| a folder whose name is not `<slug>-YYYY-MM-DD` | rename it to `<slug>-YYYY-MM-DD` and give it a `README.md`; its first commit's date is a fine default |
|
|
79
|
+
| a study folder without `README.md` | add one: what the study asks, how to run it, what it found; and a line in `repro/README.md` |
|
|
80
|
+
| anything else | by what it is, as above |
|
|
81
|
+
|
|
82
|
+
Three files paperlint itself once kept at the top of `repro/` are named with their new place and the
|
|
83
|
+
command that moves them: `references-cache.json` → `repro/data/` (read by `paperlint build`),
|
|
84
|
+
`refs-cache.json` → `repro/data/`, `generated-code-grandfathered.txt` → `repro/tools/`.
|
|
85
|
+
|
|
86
|
+
To date a folder by its first commit, run in the paper folder:
|
|
87
|
+
|
|
88
|
+
```sh
|
|
89
|
+
git log --reverse --format=%as -- repro/old-baseline | head -1
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Passing: a paper with no `repro/`, and a `repro/` holding only the entries above.
|
|
93
|
+
|
|
94
|
+
## Options / preset fields
|
|
95
|
+
|
|
96
|
+
No rule options.
|
|
97
|
+
|
|
98
|
+
## What it does not check
|
|
99
|
+
|
|
100
|
+
- What is inside `tools/`, `data/` and the study folders.
|
|
101
|
+
- Whether `repro/README.md` exists, or has a line for every study.
|
|
102
|
+
- What a study's `README.md` says.
|
|
103
|
+
- `repro/` itself: whether the paper has one is `paper/allowed-files-only`'s concern, and that rule never
|
|
104
|
+
looks inside it. Each entry is judged by exactly one of the two rules.
|
|
105
|
+
|
|
106
|
+
## The hook
|
|
107
|
+
|
|
108
|
+
The hook `paper-folder-layout` refuses, before it is written, a new file that would add an entry
|
|
109
|
+
this rule reports — a loose file at the top of `repro/`, an undated folder, or a study folder whose
|
|
110
|
+
first file is not its `README.md` — with this rule's message as the reason. It never refuses a file
|
|
111
|
+
that already exists, and it cannot see a shell `mv` or `cp`: the lint is the guarantee. What it does
|
|
112
|
+
and does not do is on [`paper/allowed-files-only`'s page](../paper/allowed-files-only.md#the-hook-a-new-stray-entry-is-refused-before-it-is-written).
|
|
113
|
+
|
|
114
|
+
## How to fix
|
|
115
|
+
|
|
116
|
+
Move or rename the entry where its message says, and update whatever names its path:
|
|
117
|
+
|
|
118
|
+
```sh
|
|
119
|
+
git mv papers/agent-rule-drift/repro/paper_numbers.py papers/agent-rule-drift/repro/tools/
|
|
120
|
+
git mv papers/agent-rule-drift/repro/baseline papers/agent-rule-drift/repro/baseline-2026-08-05
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
To switch the rule off for one paper — say, one whose `repro/` predates the layout — set it in that
|
|
124
|
+
paper's `paperlint.json`; `paper/allowed-files-only` still judges the paper's root:
|
|
125
|
+
|
|
126
|
+
```json
|
|
127
|
+
{ "rules": { "repro/allowed-files-only": "off" } }
|
|
128
|
+
```
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# study/declare-sample
|
|
2
|
+
|
|
3
|
+
> Experimental, off by default. Whether this belongs in a lint rule or in the study skills is open:
|
|
4
|
+
> see [#214](https://github.com/zernie/paperlint/issues/214).
|
|
5
|
+
|
|
6
|
+
**Level:** off unless turned on; a paper turns it on with `"rules": { "study/declare-sample": "error" }`
|
|
7
|
+
in its `paperlint.json` · **Reads:** `repro/<study>/prereg.json`, for every study folder at the top of the
|
|
8
|
+
paper's `repro/` that holds one (every folder there but the shared `tools/` and `data/`, whatever
|
|
9
|
+
its name) · **Reported on:** the first line of the paper's
|
|
10
|
+
`PIPELINE-STATUS.md`, once per problem
|
|
11
|
+
|
|
12
|
+
## What it catches
|
|
13
|
+
|
|
14
|
+
A study's pre-registration that does not say how its sample relates to the population the paper
|
|
15
|
+
talks about. Every `prereg.json` in a study folder (`repro/<slug>-YYYY-MM-DD/`) must carry five
|
|
16
|
+
fields; every other key in the file is the author's and is not read.
|
|
17
|
+
|
|
18
|
+
| field | holds |
|
|
19
|
+
| -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
20
|
+
| `population` | `about`: what the paper's claim is about, in its own words · `source`: where the units come from (census, crawl, registry, with its date) · `size`: how many units, a whole number above 0 (an estimate is fine; say how in `source`) |
|
|
21
|
+
| `strata` | `by`: the property the screens are counted by (rule type, language, …) · `values`: its values, at least one, each once |
|
|
22
|
+
| `screens` | one entry per screen applied between the population and the sample: `name` (what it keeps, as applied) and `removed`, an object with a count for **every** declared stratum value (0 where it removed none) and no other key. `[]` when no screen was applied |
|
|
23
|
+
| `coverage` | `share`: the share of the population the sample can represent, 0 to 1, or `null` with a `reason` · `unreachable`: the strata or kinds of units the sample cannot reach (`[]` only when it reaches them all; a `share` of 1 with a non-empty list is refused) |
|
|
24
|
+
| `author_check` | `who` looked at the drawn units before the run, `when` (YYYY-MM-DD), `units_seen` of `units_drawn` (the units drawn — rules, cases — not the items made from them), and the `verdict` to "is this what you meet in practice?": `representative`, `partly` or `not-representative` |
|
|
25
|
+
|
|
26
|
+
A verdict other than `representative` needs `author_check.decision`: `proceed` (true or false), `by`
|
|
27
|
+
whom, and `why`. A doubtful verdict with no recorded decision is an error; with one, either way, it
|
|
28
|
+
passes: the rule asks that the decision be made and written down, not which way it goes.
|
|
29
|
+
|
|
30
|
+
A study folder without a `prereg.json` owes nothing here.
|
|
31
|
+
|
|
32
|
+
**When it acts.** Inside stage 3, the study: `build-benchmark` writes the `prereg.json` before the
|
|
33
|
+
sample is drawn, the author checks the drawn units, and only then the run and any labelling
|
|
34
|
+
(`build-label-page`) follow. Once a paper turns the rule on, it runs on every `paperlint lint` and in CI, so a study whose
|
|
35
|
+
pre-registration lacks a field fails from the moment the file exists. The flow and the map:
|
|
36
|
+
[`layout.md`](../../layout.md).
|
|
37
|
+
|
|
38
|
+
## Why
|
|
39
|
+
|
|
40
|
+
A measurement study of how language models read coding rules drew its rules from three pools, kept
|
|
41
|
+
only the rules judgeable from one code excerpt of at most 40 lines, restricted one pool to a
|
|
42
|
+
classifier's "enforceable" lane, and let a model pick short excerpts. Each screen was reasonable
|
|
43
|
+
alone; together they left a sample of crisp rules. The labeller found 93 % of the items designed as
|
|
44
|
+
borderline clear-cut, and the rules looked nothing like the ones met in practice. A spot-check of
|
|
45
|
+
ten items flagged it (seven of ten clear-cut), and the run went ahead.
|
|
46
|
+
|
|
47
|
+
Nothing in that design forced anyone to state what share of the real population the sample covered,
|
|
48
|
+
or which screens removed which kinds of rules. Those are the statements this rule requires, before
|
|
49
|
+
the sample is frozen, in the file that freezes it. The counts per stratum are what make the drift
|
|
50
|
+
visible: a screen that removes 120 crisp rules and 1,630 vague ones is a screen on the construct.
|
|
51
|
+
The author's check is on the drawn **units**, because ten items made from crisp rules look fine one
|
|
52
|
+
by one.
|
|
53
|
+
|
|
54
|
+
## Examples
|
|
55
|
+
|
|
56
|
+
Failing, with the rule turned on: a pre-registration with hypotheses and arms, and none of the five fields.
|
|
57
|
+
|
|
58
|
+
```text
|
|
59
|
+
papers/rule-reading/PIPELINE-STATUS.md
|
|
60
|
+
1:1 error `repro/draw-2026-10-12/prereg.json`: `population` is missing — add `population`: what
|
|
61
|
+
the paper's claim is about (`about`), where its units come from (`source`) and how many
|
|
62
|
+
there are (`size`). Why: a study once screened its sample down to rules that are easy
|
|
63
|
+
to judge — its labeller found 93 % of the items designed as borderline clear-cut — and
|
|
64
|
+
nothing in its design made anyone state what share of the real population the sample
|
|
65
|
+
covered or which screens removed which kinds of units. Rule study/declare-sample: …
|
|
66
|
+
study/declare-sample
|
|
67
|
+
1:1 error … `strata` is missing …
|
|
68
|
+
1:1 error … `screens` is missing …
|
|
69
|
+
1:1 error … `coverage` is missing …
|
|
70
|
+
1:1 error … `author_check` is missing …
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Failing: a screen counted under a value `strata` does not declare, and a doubtful check with no
|
|
74
|
+
decision.
|
|
75
|
+
|
|
76
|
+
```text
|
|
77
|
+
1:1 error `repro/draw-2026-10-12/prereg.json`: `screens[0].removed.procedural` is not a declared
|
|
78
|
+
stratum value — `strata.values` declares crisp, parameterised, vague; add it there or
|
|
79
|
+
count it under one of them. …
|
|
80
|
+
1:1 error `repro/draw-2026-10-12/prereg.json`: `author_check.decision` is missing — the verdict
|
|
81
|
+
is «partly», and a doubtful verdict with no recorded decision is how a sample the author
|
|
82
|
+
doubted gets run anyway: add `proceed` (true or false), `by` whom and `why`. …
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Passing:
|
|
86
|
+
|
|
87
|
+
```json
|
|
88
|
+
{
|
|
89
|
+
"hypotheses": ["…the author's own keys, not read…"],
|
|
90
|
+
"population": {
|
|
91
|
+
"about": "rules written in agent instruction files of public repositories",
|
|
92
|
+
"source": "a census of instruction files, crawled 2026-07-28",
|
|
93
|
+
"size": 4210
|
|
94
|
+
},
|
|
95
|
+
"strata": {
|
|
96
|
+
"by": "rule type",
|
|
97
|
+
"values": ["crisp", "parameterised", "vague"]
|
|
98
|
+
},
|
|
99
|
+
"screens": [
|
|
100
|
+
{
|
|
101
|
+
"name": "judgeable from one code excerpt of at most 40 lines",
|
|
102
|
+
"removed": { "crisp": 120, "parameterised": 410, "vague": 1630 }
|
|
103
|
+
}
|
|
104
|
+
],
|
|
105
|
+
"coverage": {
|
|
106
|
+
"share": 0.31,
|
|
107
|
+
"unreachable": ["vague", "rules about process rather than code"]
|
|
108
|
+
},
|
|
109
|
+
"author_check": {
|
|
110
|
+
"who": "first author",
|
|
111
|
+
"when": "2026-10-12",
|
|
112
|
+
"units_seen": 24,
|
|
113
|
+
"units_drawn": 24,
|
|
114
|
+
"verdict": "partly",
|
|
115
|
+
"decision": {
|
|
116
|
+
"proceed": true,
|
|
117
|
+
"by": "first author",
|
|
118
|
+
"why": "the claim is narrowed to crisp and parameterised rules; vague rules are named as out of reach"
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Options / preset fields
|
|
125
|
+
|
|
126
|
+
No rule options.
|
|
127
|
+
|
|
128
|
+
## What it does not check
|
|
129
|
+
|
|
130
|
+
- Whether the numbers are true: that `removed` adds up to what the pools held, that `share` follows
|
|
131
|
+
from them, or that the author really looked. It checks that each was stated, in a shape a reader
|
|
132
|
+
and a script can use.
|
|
133
|
+
- That the paper's text matches `population.about` or names what `coverage.unreachable` lists. That
|
|
134
|
+
is reading, and the paper skills do it (`build-benchmark`, `draft-paper`'s threats section).
|
|
135
|
+
- A `prereg.json` deeper than a study folder's top (`repro/<study>/sub/prereg.json`), at the top of
|
|
136
|
+
`repro/` itself, or under any other name. The study's folder and its name are `repro/allowed-files-only`'s.
|
|
137
|
+
- A study without a `prereg.json`.
|
|
138
|
+
|
|
139
|
+
## How to fix
|
|
140
|
+
|
|
141
|
+
Add the five fields to the study's `prereg.json` before the sample is frozen: count what each screen
|
|
142
|
+
removed per stratum value from the screening script's own output, and show the author the drawn
|
|
143
|
+
units with the question "is this what you meet in practice?". A pre-registration already frozen and
|
|
144
|
+
run gets the fields as a dated addendum, with the values measured now; a paper that cannot give them
|
|
145
|
+
leaves the rule off, which is the default; one that turned it on and cannot give them sets it back
|
|
146
|
+
to `"off"` in its `paperlint.json`, which says so in the record.
|