paperlint 10.2.0 → 10.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +15 -4
- package/README.md +1 -0
- package/dist/adapters/git/index.d.ts +10 -0
- package/dist/adapters/git/index.d.ts.map +1 -0
- package/dist/adapters/git/index.js +151 -0
- package/dist/adapters/git/index.js.map +1 -0
- package/dist/adapters/memory/index.d.ts +6 -0
- package/dist/adapters/memory/index.d.ts.map +1 -1
- package/dist/adapters/memory/index.js +8 -0
- package/dist/adapters/memory/index.js.map +1 -1
- package/dist/adapters/node/index.d.ts +1 -0
- package/dist/adapters/node/index.d.ts.map +1 -1
- package/dist/adapters/node/index.js +1 -0
- package/dist/adapters/node/index.js.map +1 -1
- package/dist/adapters/node/list-entries.io.d.ts +3 -0
- package/dist/adapters/node/list-entries.io.d.ts.map +1 -0
- package/dist/adapters/node/list-entries.io.js +24 -0
- package/dist/adapters/node/list-entries.io.js.map +1 -0
- package/dist/adapters/study-registry/index.d.ts +412 -1
- package/dist/adapters/study-registry/index.d.ts.map +1 -1
- package/dist/adapters/study-registry/index.js +245 -8
- package/dist/adapters/study-registry/index.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +38 -6
- package/dist/cli.js.map +1 -1
- package/dist/domain/decide-first.d.ts +100 -0
- package/dist/domain/decide-first.d.ts.map +1 -0
- package/dist/domain/decide-first.js +102 -0
- package/dist/domain/decide-first.js.map +1 -0
- package/dist/domain/index-columns.d.ts +45 -0
- package/dist/domain/index-columns.d.ts.map +1 -0
- package/dist/domain/index-columns.js +62 -0
- package/dist/domain/index-columns.js.map +1 -0
- package/dist/domain/study-mode.d.ts +37 -0
- package/dist/domain/study-mode.d.ts.map +1 -0
- package/dist/domain/study-mode.js +45 -0
- package/dist/domain/study-mode.js.map +1 -0
- package/dist/domain/study-registry.d.ts +253 -10
- package/dist/domain/study-registry.d.ts.map +1 -1
- package/dist/domain/study-registry.js +325 -27
- package/dist/domain/study-registry.js.map +1 -1
- package/dist/outputs-before-decision-rule.d.ts +17 -0
- package/dist/outputs-before-decision-rule.d.ts.map +1 -0
- package/dist/outputs-before-decision-rule.js +90 -0
- package/dist/outputs-before-decision-rule.js.map +1 -0
- package/dist/ports/git-history.d.ts +44 -0
- package/dist/ports/git-history.d.ts.map +1 -0
- package/dist/ports/git-history.js +2 -0
- package/dist/ports/git-history.js.map +1 -0
- package/dist/ports/list-entries.d.ts +17 -0
- package/dist/ports/list-entries.d.ts.map +1 -0
- package/dist/ports/list-entries.js +2 -0
- package/dist/ports/list-entries.js.map +1 -0
- package/dist/ports/study-registry-codec.d.ts +9 -2
- package/dist/ports/study-registry-codec.d.ts.map +1 -1
- package/dist/studies.d.ts +4 -0
- package/dist/studies.d.ts.map +1 -1
- package/dist/studies.js +55 -24
- package/dist/studies.js.map +1 -1
- package/dist/study-gate.d.ts +14 -0
- package/dist/study-gate.d.ts.map +1 -0
- package/dist/study-gate.js +95 -0
- package/dist/study-gate.js.map +1 -0
- package/dist/study-index-input.d.ts +5 -0
- package/dist/study-index-input.d.ts.map +1 -0
- package/dist/study-index-input.js +13 -0
- package/dist/study-index-input.js.map +1 -0
- package/dist/study-modes.d.ts +45 -0
- package/dist/study-modes.d.ts.map +1 -0
- package/dist/study-modes.js +70 -0
- package/dist/study-modes.js.map +1 -0
- package/dist/study-registry-rule.d.ts +29 -6
- package/dist/study-registry-rule.d.ts.map +1 -1
- package/dist/study-registry-rule.js +91 -40
- package/dist/study-registry-rule.js.map +1 -1
- package/dist/study-run.d.ts +19 -0
- package/dist/study-run.d.ts.map +1 -0
- package/dist/study-run.js +9 -0
- package/dist/study-run.js.map +1 -0
- package/docs/design/study-modes.md +247 -0
- package/docs/design/study-registry.md +137 -15
- package/docs/e2e.md +8 -4
- package/docs/layout.md +26 -21
- package/docs/rules/repro/outputs-before-decision.md +119 -0
- package/docs/rules/repro/study-plan-name.md +70 -0
- package/docs/rules/repro/study-registered.md +206 -15
- package/docs/rules.md +54 -52
- package/docs/testing.md +1 -0
- package/package.json +2 -1
- package/scripts/check.ts +6 -0
- package/skills/build-benchmark/SKILL.md +24 -12
- package/skills/build-benchmark/SKILL.md.spec.ts +23 -11
- package/skills/build-label-page/SKILL.md +9 -7
- package/skills/build-label-page/SKILL.md.spec.ts +8 -6
- package/skills/camera-ready/SKILL.md +6 -1
- package/skills/camera-ready/SKILL.md.spec.ts +5 -0
- package/skills/draft-paper/SKILL.md +11 -3
- package/skills/draft-paper/SKILL.md.spec.ts +10 -2
- package/skills/harden-paper/SKILL.md +12 -2
- package/skills/harden-paper/SKILL.md.spec.ts +11 -1
- package/skills/osf-artifact-upload/SKILL.md +7 -1
- package/skills/osf-artifact-upload/SKILL.md.spec.ts +6 -0
- package/skills/pc-panel-review/SKILL.md +10 -3
- package/skills/pc-panel-review/SKILL.md.spec.ts +9 -2
- package/skills/run-agent-grid/SKILL.md +10 -5
- package/skills/run-agent-grid/SKILL.md.spec.ts +9 -4
- package/skills/tighten-paper/SKILL.md +5 -1
- package/skills/tighten-paper/SKILL.md.spec.ts +4 -0
- package/src/adapters/git/history.test.ts +121 -0
- package/src/adapters/git/index.test.ts +268 -0
- package/src/adapters/git/index.ts +227 -0
- package/src/adapters/memory/index.test.ts +15 -1
- package/src/adapters/memory/index.ts +13 -0
- package/src/adapters/node/index.ts +1 -0
- package/src/adapters/node/list-entries.io.ts +26 -0
- package/src/adapters/node/list-entries.test.ts +43 -0
- package/src/adapters/study-registry/index.test.ts +343 -0
- package/src/adapters/study-registry/index.ts +374 -10
- package/src/adapters/study-registry/mode.test.ts +162 -0
- package/src/cli.ts +52 -6
- package/src/domain/decide-first.test.ts +258 -0
- package/src/domain/decide-first.ts +243 -0
- package/src/domain/index-columns.test.ts +216 -0
- package/src/domain/index-columns.ts +105 -0
- package/src/domain/study-measurements.test.ts +633 -0
- package/src/domain/study-mode.test.ts +55 -0
- package/src/domain/study-mode.ts +78 -0
- package/src/domain/study-registry.test.ts +48 -32
- package/src/domain/study-registry.ts +621 -36
- package/src/new-paper.test.ts +6 -1
- package/src/outputs-before-decision-rule.test.ts +206 -0
- package/src/outputs-before-decision-rule.ts +124 -0
- package/src/ports/git-history.ts +44 -0
- package/src/ports/list-entries.ts +18 -0
- package/src/ports/study-registry-codec.ts +13 -2
- package/src/rule-cases.test.ts +103 -1
- package/src/studies-cli.test.ts +2 -2
- package/src/studies-index-modes.test.ts +129 -0
- package/src/studies.test.ts +39 -9
- package/src/studies.ts +99 -29
- package/src/study-gate.test.ts +316 -0
- package/src/study-gate.ts +161 -0
- package/src/study-index-input.ts +23 -0
- package/src/study-modes.ts +120 -0
- package/src/study-registry-rule.test.ts +333 -7
- package/src/study-registry-rule.ts +145 -41
- package/src/study-run.ts +21 -0
|
@@ -4,7 +4,7 @@ description: On acceptance, turn the anonymized-for-review paper into the de-ano
|
|
|
4
4
|
allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, WebSearch, WebFetch]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:66444fd7ce0e56a0 compiled from skills/camera-ready/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# camera-ready — de-anonymize an accepted paper into the final proceedings version
|
|
10
10
|
|
|
@@ -73,6 +73,11 @@ The review artifact was an OSF anonymized view-only link. The camera-ready needs
|
|
|
73
73
|
or `\doi{}`.
|
|
74
74
|
- Keep the artifact **self-checking** (it still recomputes every headline number and exits non-zero on
|
|
75
75
|
drift) — de-anonymizing it doesn't excuse it from reproducing the paper.
|
|
76
|
+
- **Ship each study's evidence of its mode**: its `plan.json`, its decision file, and
|
|
77
|
+
`repro/README.md` with the index regenerated (`npx paperlint studies index <paper-dir>`) — each study's mode,
|
|
78
|
+
decision commit and first output. The public repo keeps the **whole git history** — the order of
|
|
79
|
+
decision and outputs is read from it (`repro/outputs-before-decision`), and a squashed or shallow
|
|
80
|
+
copy loses it.
|
|
76
81
|
|
|
77
82
|
## 3. Responsible disclosure (security papers — BEFORE public release)
|
|
78
83
|
For the AISec "Safety Theater" paper the studied hooks are named-but-anonymized, and disclosure to the
|
|
@@ -88,6 +88,11 @@ The review artifact was an OSF anonymized view-only link. The camera-ready needs
|
|
|
88
88
|
or \`\\doi{}\`.
|
|
89
89
|
- Keep the artifact **self-checking** (it still recomputes every headline number and exits non-zero on
|
|
90
90
|
drift) — de-anonymizing it doesn't excuse it from reproducing the paper.
|
|
91
|
+
- **Ship each study's evidence of its mode**: its \`plan.json\`, its decision file, and
|
|
92
|
+
\`repro/README.md\` with the index regenerated (\`npx paperlint studies index <paper-dir>\`) — each study's mode,
|
|
93
|
+
decision commit and first output. The public repo keeps the **whole git history** — the order of
|
|
94
|
+
decision and outputs is read from it (\`repro/outputs-before-decision\`), and a squashed or shallow
|
|
95
|
+
copy loses it.
|
|
91
96
|
|
|
92
97
|
## 3. Responsible disclosure (security papers — BEFORE public release)
|
|
93
98
|
For the AISec "Safety Theater" paper the studied hooks are named-but-anonymized, and disclosure to the
|
|
@@ -4,7 +4,7 @@ description: Use when the numbers exist and it's time to WRITE the paper — "dr
|
|
|
4
4
|
allowed-tools: [Read, Write, Edit, Grep, Glob, Agent, Skill]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:a68aebea0917e4a6 compiled from skills/draft-paper/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# draft-paper — prose that survives review and gets cited
|
|
10
10
|
|
|
@@ -33,9 +33,17 @@ number in the body stays in the body**.
|
|
|
33
33
|
study, and — for security — the explicit threat model (attacker goals/capabilities/knowledge). A
|
|
34
34
|
fuzzy threat model is the fastest reject at a security venue.
|
|
35
35
|
4. **Method** — the reusable core. How the measurement/benchmark is *constructed* so someone else can
|
|
36
|
-
run it on a different target. This is the part that gets cited; give it the most care.
|
|
36
|
+
run it on a different target. This is the part that gets cited; give it the most care. **It states
|
|
37
|
+
each study's mode from the index, never from memory:** run `npx paperlint studies index <paper-dir>`, then read
|
|
38
|
+
each reported study's `mode`, `decision` and `first output` columns in `repro/README.md` and
|
|
39
|
+
write one sentence per study — decided before its run, with the decision's commit and date (and
|
|
40
|
+
its first output's, which came after); an exploratory study named exploratory, with its reason.
|
|
41
|
+
A study whose `decision` reads `not committed` was not decided before its run: say so.
|
|
37
42
|
5. **Results** — the numbers from `build-benchmark`, each tied to a claim. Report ranges/CIs, not
|
|
38
|
-
rounded maxes. One figure or table carries the paper — make it legible standalone.
|
|
43
|
+
rounded maxes. One figure or table carries the paper — make it legible standalone. A number from
|
|
44
|
+
an exploratory study says "exploratory" where it is reported; "pre-registered" or "decided before"
|
|
45
|
+
only for a study the index shows as `decide_before_run` with its decision committed before its
|
|
46
|
+
first output.
|
|
39
47
|
6. **Threats to validity** — construct / internal / external, honestly (see below). This section is a
|
|
40
48
|
trust signal, not a confession.
|
|
41
49
|
7. **Related work** — position against the *nearest* neighbor explicitly ("X measures A; we measure B
|
|
@@ -39,9 +39,17 @@ number in the body stays in the body**.
|
|
|
39
39
|
study, and — for security — the explicit threat model (attacker goals/capabilities/knowledge). A
|
|
40
40
|
fuzzy threat model is the fastest reject at a security venue.
|
|
41
41
|
4. **Method** — the reusable core. How the measurement/benchmark is *constructed* so someone else can
|
|
42
|
-
run it on a different target. This is the part that gets cited; give it the most care.
|
|
42
|
+
run it on a different target. This is the part that gets cited; give it the most care. **It states
|
|
43
|
+
each study's mode from the index, never from memory:** run \`npx paperlint studies index <paper-dir>\`, then read
|
|
44
|
+
each reported study's \`mode\`, \`decision\` and \`first output\` columns in \`repro/README.md\` and
|
|
45
|
+
write one sentence per study — decided before its run, with the decision's commit and date (and
|
|
46
|
+
its first output's, which came after); an exploratory study named exploratory, with its reason.
|
|
47
|
+
A study whose \`decision\` reads \`not committed\` was not decided before its run: say so.
|
|
43
48
|
5. **Results** — the numbers from \`build-benchmark\`, each tied to a claim. Report ranges/CIs, not
|
|
44
|
-
rounded maxes. One figure or table carries the paper — make it legible standalone.
|
|
49
|
+
rounded maxes. One figure or table carries the paper — make it legible standalone. A number from
|
|
50
|
+
an exploratory study says "exploratory" where it is reported; "pre-registered" or "decided before"
|
|
51
|
+
only for a study the index shows as \`decide_before_run\` with its decision committed before its
|
|
52
|
+
first output.
|
|
45
53
|
6. **Threats to validity** — construct / internal / external, honestly (see below). This section is a
|
|
46
54
|
trust signal, not a confession.
|
|
47
55
|
7. **Related work** — position against the *nearest* neighbor explicitly ("X measures A; we measure B
|
|
@@ -4,7 +4,7 @@ description: Use when asking "is this paper actually ready to submit?" / "what's
|
|
|
4
4
|
allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, WebSearch, WebFetch, Agent, Skill]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:85001dd768358bb5 compiled from skills/harden-paper/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# harden-paper — the multi-axis pre-submit gate
|
|
10
10
|
|
|
@@ -61,8 +61,18 @@ middle sags" — wasting the polish on possibly-cut sections.
|
|
|
61
61
|
there with no pointer reaching it, the body is overclaiming and "it's in the appendix" is not
|
|
62
62
|
a defence — reviewers are not required to read appendices, and ARR says so in as many words.
|
|
63
63
|
Rule and the sorting test: `paper-pipeline/references/body-vs-appendix.md`.
|
|
64
|
+
- 🔴 **A confirmatory claim on a study that was not confirmatory.** Every claim the paper presents
|
|
65
|
+
as tested — a prediction, a hypothesis confirmed, a headline effect — must rest on a study whose
|
|
66
|
+
`mode` is `decide_before_run` and whose `decision` commit precedes its `first output`, read
|
|
67
|
+
from the index in `repro/README.md` after `npx paperlint studies index <paper-dir>` (the plans and git, not
|
|
68
|
+
the prose). A confirmatory claim on an exploratory study, on one whose decision reads `not
|
|
69
|
+
committed`, or on one whose first output is not after its decision, is a **soundness** finding,
|
|
70
|
+
not a wording one: re-label the result exploratory, or rest the claim on a study decided first.
|
|
71
|
+
Flag every "pre-registered", "preregistered", "registered in advance" or "decided before" the
|
|
72
|
+
table does not support, and check the Methods sentence names each study's mode as the table does.
|
|
64
73
|
Mechanical leg: `tex/claim-provenance` (axis 7) reports every sentence that states a number and
|
|
65
|
-
does not say whose it is
|
|
74
|
+
does not say whose it is; `repro/outputs-before-decision` every output that entered history
|
|
75
|
+
before its decision.
|
|
66
76
|
|
|
67
77
|
3. **Novelty vs the nearest neighbor** — the scoop check. Search for the paper that *already did your
|
|
68
78
|
thing* (`verify-citations` covers metadata; here, actively hunt the closest empirical/method twin).
|
|
@@ -78,8 +78,18 @@ middle sags" — wasting the polish on possibly-cut sections.
|
|
|
78
78
|
there with no pointer reaching it, the body is overclaiming and "it's in the appendix" is not
|
|
79
79
|
a defence — reviewers are not required to read appendices, and ARR says so in as many words.
|
|
80
80
|
Rule and the sorting test: \`paper-pipeline/references/body-vs-appendix.md\`.
|
|
81
|
+
- 🔴 **A confirmatory claim on a study that was not confirmatory.** Every claim the paper presents
|
|
82
|
+
as tested — a prediction, a hypothesis confirmed, a headline effect — must rest on a study whose
|
|
83
|
+
\`mode\` is \`decide_before_run\` and whose \`decision\` commit precedes its \`first output\`, read
|
|
84
|
+
from the index in \`repro/README.md\` after \`npx paperlint studies index <paper-dir>\` (the plans and git, not
|
|
85
|
+
the prose). A confirmatory claim on an exploratory study, on one whose decision reads \`not
|
|
86
|
+
committed\`, or on one whose first output is not after its decision, is a **soundness** finding,
|
|
87
|
+
not a wording one: re-label the result exploratory, or rest the claim on a study decided first.
|
|
88
|
+
Flag every "pre-registered", "preregistered", "registered in advance" or "decided before" the
|
|
89
|
+
table does not support, and check the Methods sentence names each study's mode as the table does.
|
|
81
90
|
Mechanical leg: \`tex/claim-provenance\` (axis 7) reports every sentence that states a number and
|
|
82
|
-
does not say whose it is
|
|
91
|
+
does not say whose it is; \`repro/outputs-before-decision\` every output that entered history
|
|
92
|
+
before its decision.
|
|
83
93
|
|
|
84
94
|
3. **Novelty vs the nearest neighbor** — the scoop check. Search for the paper that *already did your
|
|
85
95
|
thing* (\`verify-citations\` covers metadata; here, actively hunt the closest empirical/method twin).
|
|
@@ -4,7 +4,7 @@ description: Upload or replace a paper's reproduction artifact file on an OSF pr
|
|
|
4
4
|
allowed-tools: [Read, Write, Grep, Glob, Bash]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:953fdd419dbc789c compiled from skills/osf-artifact-upload/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# osf-artifact-upload — push/replace a file on an OSF project via API
|
|
10
10
|
|
|
@@ -45,6 +45,12 @@ curl -g -s --retry 3 --cacert $CA -H "$AUTH" -X PUT --data-binary @"$ZIP" "${UP}
|
|
|
45
45
|
Verification that actually holds: **OSF-stored md5 == `md5sum $ZIP`**. The upload response also returns the
|
|
46
46
|
md5; matching it to the local file is proof even when the download-back call flakes.
|
|
47
47
|
|
|
48
|
+
## What the artifact ships about each study
|
|
49
|
+
Before zipping, the artifact carries, for every study the paper reports: its `plan.json` (its mode)
|
|
50
|
+
and its decision file; and `repro/README.md` with the index regenerated (`npx paperlint studies index <paper-dir>`)
|
|
51
|
+
— each study's mode, decision commit and first output — with the command that re-checks the order
|
|
52
|
+
(`npx paperlint lint` on a full clone). A zip without them asks a reviewer to take the modes on trust.
|
|
53
|
+
|
|
48
54
|
## After
|
|
49
55
|
- Confirm the paper's Availability `\url{}` still points at the same node (it does — same guid).
|
|
50
56
|
- Tell the user to **revoke the token** (osf.io/settings/tokens → Delete).
|
|
@@ -51,6 +51,12 @@ curl -g -s --retry 3 --cacert $CA -H "$AUTH" -X PUT --data-binary @"$ZIP" "\${UP
|
|
|
51
51
|
Verification that actually holds: **OSF-stored md5 == \`md5sum $ZIP\`**. The upload response also returns the
|
|
52
52
|
md5; matching it to the local file is proof even when the download-back call flakes.
|
|
53
53
|
|
|
54
|
+
## What the artifact ships about each study
|
|
55
|
+
Before zipping, the artifact carries, for every study the paper reports: its \`plan.json\` (its mode)
|
|
56
|
+
and its decision file; and \`repro/README.md\` with the index regenerated (\`npx paperlint studies index <paper-dir>\`)
|
|
57
|
+
— each study's mode, decision commit and first output — with the command that re-checks the order
|
|
58
|
+
(\`npx paperlint lint\` on a full clone). A zip without them asks a reviewer to take the modes on trust.
|
|
59
|
+
|
|
54
60
|
## After
|
|
55
61
|
- Confirm the paper's Availability \`\\url{}\` still points at the same node (it does — same guid).
|
|
56
62
|
- Tell the user to **revoke the token** (osf.io/settings/tokens → Delete).
|
|
@@ -4,7 +4,7 @@ description: Use when asking "what would the program committee decide?" / "simul
|
|
|
4
4
|
allowed-tools: [Read, Write, Grep, Glob, Bash, WebSearch, WebFetch, Agent, Skill]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:d92a744ec1cc1b05 compiled from skills/pc-panel-review/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# pc-panel-review — model the whole PC, not one reviewer
|
|
10
10
|
|
|
@@ -61,11 +61,18 @@ prose-only read can.
|
|
|
61
61
|
and calls the paper a blog post — the one reviewer a panel of careful full readers never
|
|
62
62
|
predicts. Give it the three sections as text and the venue's name, nothing else.
|
|
63
63
|
- **Methods/stats skeptic** — sample size, construct validity, multiple comparisons, overclaims,
|
|
64
|
-
whether the CIs support the claims
|
|
64
|
+
whether the CIs support the claims, and **whether every confirmatory claim rests on a study
|
|
65
|
+
decided before its run**. Run `npx paperlint studies index <paper-dir>` and give this reviewer the index in
|
|
66
|
+
`repro/README.md` (each study's `mode`, `decision` and `first output`) beside the paper: a
|
|
67
|
+
claim of a tested prediction on an exploratory study, on one whose decision reads `not
|
|
68
|
+
committed`, or on one whose first output is not after its decision is a soundness finding, and
|
|
69
|
+
so is a "pre-registered" or "decided before" the table does not support; the reviewer names it.
|
|
65
70
|
- **Domain practitioner** — relevance to the CFP, novelty vs the nearest prior work, does the
|
|
66
71
|
finding change behavior, generalizability, fairness/tone toward anyone named.
|
|
67
72
|
- **Artifact / reproducibility reviewer (MANDATORY if an artifact is attached)** — *actually
|
|
68
|
-
`cd` into the artifact and run it*; verify every headline number reproduces;
|
|
73
|
+
`cd` into the artifact and run it*; verify every headline number reproduces; re-run the order
|
|
74
|
+
check the paper's decided-before claims rest on (`npx paperlint lint` on a full clone); read the
|
|
75
|
+
code for
|
|
69
76
|
circularity (echoing stored fields vs recomputing); grep all files (incl. data + compiled
|
|
70
77
|
caches) for anonymization leaks; scope the badge (Available / Functional / Reproduced).
|
|
71
78
|
- **Security/ethics reviewer** (security venues) — threat model soundness, responsible disclosure,
|
|
@@ -77,11 +77,18 @@ prose-only read can.
|
|
|
77
77
|
and calls the paper a blog post — the one reviewer a panel of careful full readers never
|
|
78
78
|
predicts. Give it the three sections as text and the venue's name, nothing else.
|
|
79
79
|
- **Methods/stats skeptic** — sample size, construct validity, multiple comparisons, overclaims,
|
|
80
|
-
whether the CIs support the claims
|
|
80
|
+
whether the CIs support the claims, and **whether every confirmatory claim rests on a study
|
|
81
|
+
decided before its run**. Run \`npx paperlint studies index <paper-dir>\` and give this reviewer the index in
|
|
82
|
+
\`repro/README.md\` (each study's \`mode\`, \`decision\` and \`first output\`) beside the paper: a
|
|
83
|
+
claim of a tested prediction on an exploratory study, on one whose decision reads \`not
|
|
84
|
+
committed\`, or on one whose first output is not after its decision is a soundness finding, and
|
|
85
|
+
so is a "pre-registered" or "decided before" the table does not support; the reviewer names it.
|
|
81
86
|
- **Domain practitioner** — relevance to the CFP, novelty vs the nearest prior work, does the
|
|
82
87
|
finding change behavior, generalizability, fairness/tone toward anyone named.
|
|
83
88
|
- **Artifact / reproducibility reviewer (MANDATORY if an artifact is attached)** — *actually
|
|
84
|
-
\`cd\` into the artifact and run it*; verify every headline number reproduces;
|
|
89
|
+
\`cd\` into the artifact and run it*; verify every headline number reproduces; re-run the order
|
|
90
|
+
check the paper's decided-before claims rest on (\`npx paperlint lint\` on a full clone); read the
|
|
91
|
+
code for
|
|
85
92
|
circularity (echoing stored fields vs recomputing); grep all files (incl. data + compiled
|
|
86
93
|
caches) for anonymization leaks; scope the badge (Available / Functional / Reproduced).
|
|
87
94
|
- **Security/ethics reviewer** (security venues) — threat model soundness, responsible disclosure,
|
|
@@ -4,7 +4,7 @@ description: "Run a paid or counted agent experiment grid end to end — units
|
|
|
4
4
|
allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Agent]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:451354b8f781c409 compiled from skills/run-agent-grid/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# run-agent-grid — running a counted agent grid without spoiling it
|
|
10
10
|
|
|
@@ -25,10 +25,15 @@ and a driver loop that calls the guard: [`references/registry-and-ledger.md`](re
|
|
|
25
25
|
|
|
26
26
|
**The grid's study folder.** A grid is one study of the paper. Its folder is made by
|
|
27
27
|
`npx paperlint studies new <paper-dir> "<title>" --kind measurement` — the next id in
|
|
28
|
-
`repro/studies.json`, the folder `repro/<id>-<slug>/` — and `grid.json`,
|
|
29
|
-
|
|
30
|
-
`
|
|
31
|
-
|
|
28
|
+
`repro/studies.json`, the folder `repro/<id>-<slug>/` with the study's `plan.json` — and `grid.json`,
|
|
29
|
+
the runs and the ledger live in it. The plan says what the grid measures, the population and its
|
|
30
|
+
draw (`{ "kind": "seeded", "method", "seed" }`), the `arms` (each name once) and the `primary_outcome`, and is filled before the first counted
|
|
31
|
+
run; the author's approval of the drawn units goes into both `grid.json`'s `approval` (the guard reads
|
|
32
|
+
it) and the plan's `author_check` (the lint reads it). A grid that is one study of a larger
|
|
33
|
+
measurement names it in its row's `measurement`. When the grid ends, its row is updated (`status`,
|
|
34
|
+
`result`, `result_file`, `in_paper`) and `npx paperlint studies index <paper-dir>` regenerates the
|
|
35
|
+
index; `repro/study-registered` fails the lint until both are done, and on a running grid whose plan
|
|
36
|
+
is empty.
|
|
32
37
|
|
|
33
38
|
## The order — each gate is passed before the next phase starts
|
|
34
39
|
|
|
@@ -26,10 +26,15 @@ and a driver loop that calls the guard: [\`references/registry-and-ledger.md\`](
|
|
|
26
26
|
|
|
27
27
|
**The grid's study folder.** A grid is one study of the paper. Its folder is made by
|
|
28
28
|
\`npx paperlint studies new <paper-dir> "<title>" --kind measurement\` — the next id in
|
|
29
|
-
\`repro/studies.json\`, the folder \`repro/<id>-<slug>/\` — and \`grid.json\`,
|
|
30
|
-
|
|
31
|
-
\`
|
|
32
|
-
|
|
29
|
+
\`repro/studies.json\`, the folder \`repro/<id>-<slug>/\` with the study's \`plan.json\` — and \`grid.json\`,
|
|
30
|
+
the runs and the ledger live in it. The plan says what the grid measures, the population and its
|
|
31
|
+
draw (\`{ "kind": "seeded", "method", "seed" }\`), the \`arms\` (each name once) and the \`primary_outcome\`, and is filled before the first counted
|
|
32
|
+
run; the author's approval of the drawn units goes into both \`grid.json\`'s \`approval\` (the guard reads
|
|
33
|
+
it) and the plan's \`author_check\` (the lint reads it). A grid that is one study of a larger
|
|
34
|
+
measurement names it in its row's \`measurement\`. When the grid ends, its row is updated (\`status\`,
|
|
35
|
+
\`result\`, \`result_file\`, \`in_paper\`) and \`npx paperlint studies index <paper-dir>\` regenerates the
|
|
36
|
+
index; \`repro/study-registered\` fails the lint until both are done, and on a running grid whose plan
|
|
37
|
+
is empty.
|
|
33
38
|
|
|
34
39
|
## The order — each gate is passed before the next phase starts
|
|
35
40
|
|
|
@@ -4,7 +4,7 @@ description: "Use when a paper \"feels long / bloated / overly complex / hard to
|
|
|
4
4
|
allowed-tools: [Read, Write, Edit, Grep, Glob, Agent]
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
<!-- vigiles:sha256:
|
|
7
|
+
<!-- vigiles:sha256:ccf39ff26a2146e9 compiled from skills/tighten-paper/SKILL.md.spec.ts -->
|
|
8
8
|
|
|
9
9
|
# tighten-paper — the developmental edit
|
|
10
10
|
|
|
@@ -149,6 +149,10 @@ called it "fat for no reason" after a pass that cut almost nothing and left it 1
|
|
|
149
149
|
measurement paper's fat HIDES as explanation of real numbers, which is exactly why "it's all numbers" lets it
|
|
150
150
|
survive a lazy pass (GateBench 2026-07-25: a pass cut only 5% hiding behind this; the author correctly pushed
|
|
151
151
|
back that whole sections were over-explained, not just number-dense).
|
|
152
|
+
- **The study-mode sentence is never cut for length.** The Methods sentence that says a study was
|
|
153
|
+
decided before its run (with its decision's commit and date, from the index in `repro/README.md`)
|
|
154
|
+
or is exploratory — and the word "exploratory" beside an exploratory number — is the claim's
|
|
155
|
+
warrant, not its explanation: cut around it, never it.
|
|
152
156
|
- **"Fold to the artifact" is the PRIMARY length lever for a measurement/benchmark paper — not a last resort.**
|
|
153
157
|
FIRST verify the artifact exists and recomputes the numbers (it usually does for these papers). Then per-case
|
|
154
158
|
breakdowns, secondary analyses, exhaustive robustness sub-results, ablation walkthroughs, and every "we also
|
|
@@ -155,6 +155,10 @@ called it "fat for no reason" after a pass that cut almost nothing and left it 1
|
|
|
155
155
|
measurement paper's fat HIDES as explanation of real numbers, which is exactly why "it's all numbers" lets it
|
|
156
156
|
survive a lazy pass (GateBench 2026-07-25: a pass cut only 5% hiding behind this; the author correctly pushed
|
|
157
157
|
back that whole sections were over-explained, not just number-dense).
|
|
158
|
+
- **The study-mode sentence is never cut for length.** The Methods sentence that says a study was
|
|
159
|
+
decided before its run (with its decision's commit and date, from the index in \`repro/README.md\`)
|
|
160
|
+
or is exploratory — and the word "exploratory" beside an exploratory number — is the claim's
|
|
161
|
+
warrant, not its explanation: cut around it, never it.
|
|
158
162
|
- **"Fold to the artifact" is the PRIMARY length lever for a measurement/benchmark paper — not a last resort.**
|
|
159
163
|
FIRST verify the artifact exists and recomputes the numbers (it usually does for these papers). Then per-case
|
|
160
164
|
breakdowns, secondary analyses, exhaustive robustness sub-results, ablation walkthroughs, and every "we also
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The git ports over a REAL repository in a scratch directory: the committed state of a decision,
|
|
3
|
+
* and the order of every output's first commit against the decision's — after, the same commit, or
|
|
4
|
+
* not after — a deleted output included, and a shallow clone named as such.
|
|
5
|
+
*/
|
|
6
|
+
import { writeFileSync } from "node:fs";
|
|
7
|
+
import { join } from "node:path";
|
|
8
|
+
import { expect, it } from "vitest";
|
|
9
|
+
import { absolutePath } from "../../domain/paths.ts";
|
|
10
|
+
import { spawnProcess } from "../node/index.ts";
|
|
11
|
+
import { commit, git, initRepo } from "../../../test/git-repo.ts";
|
|
12
|
+
import { useTempDir } from "../../../test/support.ts";
|
|
13
|
+
import { gitHistory } from "./index.ts";
|
|
14
|
+
|
|
15
|
+
const root = useTempDir("git-history-");
|
|
16
|
+
const ports = gitHistory(spawnProcess(), { PATH: String(process.env.PATH) });
|
|
17
|
+
const T1 = "2026-10-16T10:00:00+05:00";
|
|
18
|
+
const T2 = "2026-10-16T11:00:00+05:00";
|
|
19
|
+
const T3 = "2026-10-16T12:00:00+05:00";
|
|
20
|
+
|
|
21
|
+
it("a decision committed, then changed, then never committed: three states", () => {
|
|
22
|
+
const repo = join(root, "committed");
|
|
23
|
+
initRepo(repo);
|
|
24
|
+
const sha = commit(repo, { "p/decision.md": "exclusions: none\n" }, T1);
|
|
25
|
+
const paper = absolutePath(join(repo, "p"));
|
|
26
|
+
const file = absolutePath(join(repo, "p/decision.md"));
|
|
27
|
+
// Guards: the commit and its date as git records them.
|
|
28
|
+
expect(ports.committed(paper, file)).toEqual({
|
|
29
|
+
kind: "committed",
|
|
30
|
+
commit: sha,
|
|
31
|
+
date: T1,
|
|
32
|
+
});
|
|
33
|
+
writeFileSync(file, "exclusions: run 7\n");
|
|
34
|
+
// Guards: an edit after the commit is not the recorded decision.
|
|
35
|
+
expect(ports.committed(paper, file)).toEqual({
|
|
36
|
+
kind: "changed",
|
|
37
|
+
commit: sha,
|
|
38
|
+
});
|
|
39
|
+
writeFileSync(join(repo, "p/new.md"), "x\n");
|
|
40
|
+
expect(ports.committed(paper, absolutePath(join(repo, "p/new.md")))).toEqual({
|
|
41
|
+
kind: "not-committed",
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it("outputs after the decision, in its commit, before it, and deleted since: each with its order", () => {
|
|
46
|
+
const repo = join(root, "order");
|
|
47
|
+
initRepo(repo);
|
|
48
|
+
const early = commit(repo, { "p/repro/S/early.csv": "1\n" }, T1);
|
|
49
|
+
const decided = commit(
|
|
50
|
+
repo,
|
|
51
|
+
{ "p/repro/S/decision.md": "d\n", "p/repro/S/same.csv": "2\n" },
|
|
52
|
+
T2,
|
|
53
|
+
);
|
|
54
|
+
const later = commit(
|
|
55
|
+
repo,
|
|
56
|
+
{ "p/repro/S/later.csv": "3\n", "p/repro/S/early.csv": null },
|
|
57
|
+
T3,
|
|
58
|
+
);
|
|
59
|
+
// Guards: the order is read from ancestry; a deleted early output is still evidence.
|
|
60
|
+
expect(
|
|
61
|
+
ports.order(
|
|
62
|
+
absolutePath(join(repo, "p/repro/S")),
|
|
63
|
+
absolutePath(join(repo, "p/repro/S/decision.md")),
|
|
64
|
+
),
|
|
65
|
+
).toEqual({
|
|
66
|
+
kind: "read",
|
|
67
|
+
decision: { commit: decided, date: T2 },
|
|
68
|
+
decisionDeleted: false,
|
|
69
|
+
files: [
|
|
70
|
+
{ path: "early.csv", commit: early, date: T1, order: "not-after" },
|
|
71
|
+
{ path: "decision.md", commit: decided, date: T2, order: "same" },
|
|
72
|
+
{ path: "same.csv", commit: decided, date: T2, order: "same" },
|
|
73
|
+
{ path: "later.csv", commit: later, date: T3, order: "after" },
|
|
74
|
+
],
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("a shallow clone is named, not judged", () => {
|
|
79
|
+
const repo = join(root, "deep");
|
|
80
|
+
initRepo(repo);
|
|
81
|
+
commit(repo, { "p/repro/S/decision.md": "d\n" }, T1);
|
|
82
|
+
commit(repo, { "p/repro/S/out.csv": "1\n" }, T2);
|
|
83
|
+
const shallow = join(root, "shallow");
|
|
84
|
+
git(root, ["clone", "-q", "--depth", "1", `file://${repo}`, shallow]);
|
|
85
|
+
// Guards: a CI checkout of depth 1 says so instead of reading every file as born in one commit.
|
|
86
|
+
expect(
|
|
87
|
+
ports.order(
|
|
88
|
+
absolutePath(join(shallow, "p/repro/S")),
|
|
89
|
+
absolutePath(join(shallow, "p/repro/S/decision.md")),
|
|
90
|
+
),
|
|
91
|
+
).toEqual({ kind: "shallow" });
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it("outside any repository, git's own refusal is the reason", () => {
|
|
95
|
+
const r = ports.order(absolutePath("/"), absolutePath("/decision.md"));
|
|
96
|
+
// Guards: no repository is "unknown", never an empty history.
|
|
97
|
+
expect(r.kind).toBe("unknown");
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("a decision committed and deleted in a later commit is not in HEAD: never `committed`", () => {
|
|
101
|
+
const repo = join(root, "deleted");
|
|
102
|
+
initRepo(repo);
|
|
103
|
+
const sha = commit(repo, { "p/repro/S/decision.md": "d\n" }, T1);
|
|
104
|
+
const gone = commit(repo, { "p/repro/S/decision.md": null }, T2);
|
|
105
|
+
const paper = absolutePath(join(repo, "p"));
|
|
106
|
+
const file = absolutePath(join(repo, "p/repro/S/decision.md"));
|
|
107
|
+
// Guards: `git log -1` names the deletion and the tree is clean, yet there is no decision to gate on;
|
|
108
|
+
// the order still reads the decision's first commit, and says it is gone.
|
|
109
|
+
expect([
|
|
110
|
+
ports.committed(paper, file),
|
|
111
|
+
ports.order(absolutePath(join(repo, "p/repro/S")), file),
|
|
112
|
+
]).toEqual([
|
|
113
|
+
{ kind: "deleted", commit: gone },
|
|
114
|
+
{
|
|
115
|
+
kind: "read",
|
|
116
|
+
decision: { commit: sha, date: T1 },
|
|
117
|
+
decisionDeleted: true,
|
|
118
|
+
files: [{ path: "decision.md", commit: sha, date: T1, order: "same" }],
|
|
119
|
+
},
|
|
120
|
+
]);
|
|
121
|
+
});
|