paperlint 2.0.0

package/skills/paper-pipeline/repro/2026-08-08-parent-replication-gpw.json

@@ -0,0 +1,139 @@
+{
+  "trials": 3,
+  "competitors": 36,
+  "results": [
+    {
+      "skill": "grade-paper-writing",
+      "cell": "AH",
+      "prompt": "grade the writing craft in this draft and name the offending sentence in each dimension",
+      "trials": 3,
+      "rate": 1,
+      "errored": 0,
+      "overlap": 0.875,
+      "lexArgmax": "grade-paper-writing",
+      "buckets": [
+        "TARGET",
+        "TARGET",
+        "TARGET"
+      ],
+      "othersFired": []
+    },
+    {
+      "skill": "grade-paper-writing",
+      "cell": "AH",
+      "prompt": "run the blind multi-grader panel over the prose clarity and jargon discipline",
+      "trials": 3,
+      "rate": 1,
+      "errored": 0,
+      "overlap": 1,
+      "lexArgmax": "grade-paper-writing",
+      "buckets": [
+        "TARGET",
+        "TARGET",
+        "TARGET"
+      ],
+      "othersFired": []
+    },
+    {
+      "skill": "grade-paper-writing",
+      "cell": "AH",
+      "prompt": "score the title, the abstract and the intro architecture one to five",
+      "trials": 3,
+      "rate": 1,
+      "errored": 0,
+      "overlap": 0.667,
+      "lexArgmax": "grade-paper-writing",
+      "buckets": [
+        "TARGET",
+        "TARGET",
+        "TARGET"
+      ],
+      "othersFired": []
+    },
+    {
+      "skill": "grade-paper-writing",
+      "cell": "AH",
+      "prompt": "grade how well this reads, hedge stacking included, and give me the fix for each",
+      "trials": 3,
+      "rate": 0,
+      "errored": 0,
+      "overlap": 0.857,
+      "lexArgmax": "grade-paper-writing",
+      "buckets": [
+        "SILENT",
+        "SILENT",
+        "SILENT"
+      ],
+      "othersFired": []
+    },
+    {
+      "skill": "grade-paper-writing",
+      "cell": "SH",
+      "prompt": "the abstract reads like a wall of jargon",
+      "trials": 3,
+      "rate": 0.6666666666666666,
+      "errored": 0,
+      "overlap": 1,
+      "lexArgmax": "grade-paper-writing",
+      "buckets": [
+        "SILENT",
+        "TARGET",
+        "TARGET"
+      ],
+      "othersFired": []
+    },
+    {
+      "skill": "grade-paper-writing",
+      "cell": "SH",
+      "prompt": "the intro architecture is fine but the prose clarity is not",
+      "trials": 3,
+      "rate": 0,
+      "errored": 0,
+      "overlap": 0.8,
+      "lexArgmax": "grade-paper-writing",
+      "buckets": [
+        "SILENT",
+        "SILENT",
+        "OTHER"
+      ],
+      "othersFired": [
+        "cold-read-diff"
+      ]
+    },
+    {
+      "skill": "grade-paper-writing",
+      "cell": "SH",
+      "prompt": "there is hedge stacking everywhere and the point never lands",
+      "trials": 3,
+      "rate": 0.3333333333333333,
+      "errored": 0,
+      "overlap": 0.667,
+      "lexArgmax": "grade-paper-writing",
+      "buckets": [
+        "OTHER",
+        "TARGET",
+        "OTHER"
+      ],
+      "othersFired": [
+        "tighten-paper",
+        "tighten-paper"
+      ]
+    },
+    {
+      "skill": "grade-paper-writing",
+      "cell": "SH",
+      "prompt": "the writing does not read well even though the draft is finished",
+      "trials": 3,
+      "rate": 0,
+      "errored": 0,
+      "overlap": 0.667,
+      "lexArgmax": "grade-paper-writing",
+      "buckets": [
+        "SILENT",
+        "SILENT",
+        "SILENT"
+      ],
+      "othersFired": []
+    }
+  ]
+}

package/skills/paper-pipeline/repro/2026-08-08-parent-replication-gpw.log

@@ -0,0 +1,98 @@
+patched: filter the MAIN LOOP to grade-paper-writing, cells AH+SH (guards untouched)
+patched: redirect the JSON output so the parent's own result file survives
+
+running the PARENT'S code, grade-paper-writing AH+SH only, 3 trials = 24 runs
+  parent : /home/user/mine/.claude/skills/paper-pipeline/framing-vs-vocabulary.eval.mjs
+  patched: /tmp/claude-0/-home-user-mine/8268acb1-66a2-55ac-858e-2b9e3c81f84e/scratchpad/parent-repl/replication.mjs
+
+
+37 skills installed → 36 competitors per run
+
+MEASURED VOCABULARY OVERLAP (prompt content-words found in the target description)
+skill                      AH    AL    SH    SL   HIGH-LOW
+tighten-paper            0.81  0.00  0.79  0.04   +0.78
+grade-paper-writing      0.85  0.14  0.78  0.05   +0.72
+argument-arc             0.86  0.00  0.69  0.00   +0.77
+
+mode=main  3 skills x 4 cells x 4 prompts x 3 trials = 144 runs
+
+=== grade-paper-writing  [ACTION  / HIGH overlap]
+  Spent: 404,155 tokens (18 in · 1.6k out · 357.3k cache) · ~$0.40 API-equivalent
+  Billed to: your Claude subscription — $0 metered ✅
+  1.00  T3 O0 S0  ov=0.88  grade the writing craft in this draft and name the offen
+  Spent: 498,692 tokens (22 in · 2.7k out · 448.8k cache) · ~$0.46 API-equivalent
+  Billed to: your Claude subscription — $0 metered ✅
+  Session so far: 902,847 tokens · ~$0.86 API-equivalent
+  1.00  T3 O0 S0  ov=1.00  run the blind multi-grader panel over the prose clarity 
+  Spent: 499,035 tokens (22 in · 3.4k out · 448.3k cache) · ~$0.47 API-equivalent
+  Billed to: your Claude subscription — $0 metered ✅
+  Session so far: 1,401,882 tokens · ~$1.33 API-equivalent
+  1.00  T3 O0 S0  ov=0.67  score the title, the abstract and the intro architecture
+  Spent: 133,709 tokens (6 in · 778 out · 90.1k cache) · ~$0.30 API-equivalent
+  Billed to: your Claude subscription — $0 metered ✅
+  Session so far: 1,535,591 tokens · ~$1.63 API-equivalent
+  0.00  T0 O0 S3  ov=0.86  grade how well this reads, hedge stacking included, and 
+
+=== grade-paper-writing  [SITUATION / HIGH overlap]
+  Spent: 587,085 tokens (26 in · 2.4k out · 537.8k cache) · ~$0.48 API-equivalent
+  Billed to: your Claude subscription — $0 metered ✅
+  Session so far: 2,122,676 tokens · ~$2.10 API-equivalent
+  0.67  T2 O0 S1  ov=1.00  the abstract reads like a wall of jargon
+  Spent: 894,628 tokens (38 in · 13.9k out · 821.6k cache) · ~$0.91 API-equivalent
+  Billed to: your Claude subscription — $0 metered ✅
+  Session so far: 3,017,304 tokens · ~$3.02 API-equivalent
+  0.00  T0 O1 S2  ov=0.80  the intro architecture is fine but the prose clarity is 
+         instead: cold-read-diff
+  Spent: 496,058 tokens (21 in · 2.2k out · 448.0k cache) · ~$0.44 API-equivalent
+  Billed to: your Claude subscription — $0 metered ✅
+  Session so far: 3,513,362 tokens · ~$3.46 API-equivalent
+  0.33  T1 O2 S0  ov=0.67  there is hedge stacking everywhere and the point never l
+         instead: tighten-paper
+  Spent: 134,038 tokens (6 in · 1.1k out · 90.1k cache) · ~$0.30 API-equivalent
+  Billed to: your Claude subscription — $0 metered ✅
+  Session so far: 3,647,400 tokens · ~$3.76 API-equivalent
+  0.00  T0 O0 S3  ov=0.67  the writing does not read well even though the draft is 
+
+==============================================================================
+THE 2x2 — target trigger rate
+
+                          HIGH overlap     LOW overlap
+ACTION                      75% (9/12)      NaN% (0/0)
+SITUATION                   25% (3/12)      NaN% (0/0)
+
+main effect of FRAMING   (AH+AL) - (SH+SL) = 50pp
+main effect of VOCABULARY (AH+SH) - (AL+SL) = NaNpp
+
+  A (lexical) predicts   AH high, SH high, AL low,  SL low
+  B (framing) predicts   AH high, AL high, SH low,  SL low
+  observed               AH 75%  AL NaN%  SH 25%  SL NaN%
+
+==============================================================================
+WHAT FIRED INSTEAD — the C discriminator
+
+cell                        target   other   silent
+ACTION  / HIGH overlap         75%      0%     25%
+ACTION  / LOW  overlap        NaN%    NaN%    NaN%
+SITUATION / HIGH overlap       25%     25%     50%
+SITUATION / LOW  overlap      NaN%    NaN%    NaN%
+
+  OTHER  = a competitor was selected and the target was not. The prompt routed somewhere;
+           the selector disagreed. Ambiguity (C) or a lexically nearer rival (A).
+  SILENT = nothing resolved. The target was never selected AND neither was anything else.
+           A description that does not cover the phrasing (A or B) — but see caveat 2:
+           a run that simply answered the question is also SILENT.
+
+WHICH competitors took the runs (all cells):
+    2  tighten-paper
+    1  cold-read-diff
+
+per-skill, target rate by cell:
+skill                       AH      AL      SH      SL
+tighten-paper             NaN%    NaN%    NaN%    NaN%
+grade-paper-writing        75%    NaN%     25%    NaN%
+argument-arc              NaN%    NaN%    NaN%    NaN%
+
+per-run rows → repro/2026-08-08-framing-vs-vocabulary.json
+
+Run --mode oracle before concluding C is dead: the 2x2 alone cannot tell "the description was
+not a candidate" from "no description could have been", and that is exactly what C claims.

package/skills/paper-pipeline/repro/2026-08-08-parent-replication.mjs

@@ -0,0 +1,92 @@
+/**
+ * 2026-08-08-parent-replication.mjs — DID THE PARENT'S 12% HOLD, OR WAS IT TIME-BOUND?
+ *
+ * Run:  node .claude/skills/paper-pipeline/repro/2026-08-08-parent-replication.mjs
+ *
+ * WHY THIS EXISTS. `grade-paper-writing-ablation.eval.mjs`'s CONTROL ARM (A0 — the verbatim
+ * description, the parent's own eight AH+SH prompts, the parent's fixture, the same 37 skills)
+ * returned 13/24 = 54%. The parent (`framing-vs-vocabulary.eval.mjs`, same day) measured those same
+ * eight prompts at AH 3/12 and SH 2/12 = 5/24 = 21%, and that 21%/12% is the entire premise of the
+ * ablation — and of two SKILL.md edits made earlier the same day that cite it as measured fact.
+ *
+ * A control that contradicts the experiment it was built to explain outranks every arm, so this
+ * settles which number is real, and it does it through the PARENT'S OWN CODE, not a reimplementation:
+ *
+ *   - It reads `framing-vs-vocabulary.eval.mjs` and applies exactly TWO textual patches, each of
+ *     which must match exactly once or this script aborts:
+ *       1. the MAIN LOOP is filtered to `grade-paper-writing` and to cells AH + SH;
+ *       2. the output JSON path is changed so the parent's own result file is never overwritten.
+ *     Everything else — fixture, roster, guards, allowedTools, trials, concurrency, spacing, timeout,
+ *     the `fired` predicate, the skillsDir (the REAL `.claude/skills`) — is the parent's, unmodified.
+ *     Note patch 1 deliberately filters the LOOP and not `CELLS`/`CASES` themselves: the parent's
+ *     preflight guards compute overlap contrasts across all four cells and all three skills, and
+ *     narrowing the constants would disable the guards instead of the runs.
+ *   - The patched copy is written to the session scratchpad with a `node_modules` symlink, so
+ *     `.claude/skills/` is not touched and the parent file is read-only throughout.
+ *
+ * READING IT
+ *   ~54%  the parent's 21% for this skill was time-bound or otherwise unstable, and the "dead skill"
+ *         premise is about MEASUREMENT STABILITY, not about the description. The two SKILL.md
+ *         paragraphs citing 12% need correcting.
+ *   ~21%  the difference lives in the ablation's harness setup (a copied skills dir, per-arm
+ *         packaging) and NO ablation arm is comparable to the parent's numbers — the arms remain
+ *         internally comparable to each other and to A0, and nothing else.
+ *
+ * 24 runs, ~$3.4 API-equivalent at the measured rate. n = 24 resolves a factor of two; that is
+ * exactly the size of the disagreement (21% vs 54%), which is why 24 is enough here.
+ */
+
+import { readFileSync, writeFileSync, mkdirSync, symlinkSync, existsSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const ROOT = process.env.CLAUDE_PROJECT_DIR ?? "/home/user/mine";
+const PARENT = join(ROOT, ".claude", "skills", "paper-pipeline", "framing-vs-vocabulary.eval.mjs");
+const SCRATCH =
+  process.env.ABLATION_SCRATCH ??
+  "/tmp/claude-0/-home-user-mine/8268acb1-66a2-55ac-858e-2b9e3c81f84e/scratchpad/parent-repl";
+
+const src = readFileSync(PARENT, "utf-8");
+
+const PATCHES = [
+  {
+    what: "filter the MAIN LOOP to grade-paper-writing, cells AH+SH (guards untouched)",
+    from: "const results = [];\nfor (const c of CASES) {\n  for (const k of CELLS) {",
+    to:
+      "const results = [];\n" +
+      'for (const c of CASES.filter((c) => c.skill === "grade-paper-writing")) {\n' +
+      '  for (const k of CELLS.filter((k) => k === "AH" || k === "SH")) {',
+  },
+  {
+    what: "redirect the JSON output so the parent's own result file survives",
+    from: '"2026-08-08-framing-vs-vocabulary.json"',
+    to: '"2026-08-08-parent-replication-gpw.json"',
+  },
+];
+
+let patched = src;
+for (const p of PATCHES) {
+  const n = patched.split(p.from).length - 1;
+  if (n !== 1)
+    throw new Error(`PATCH MATCHED ${n} TIMES (must be exactly 1): ${p.what}\n---\n${p.from}\n---`);
+  patched = patched.replace(p.from, p.to);
+  console.log(`patched: ${p.what}`);
+}
+
+rmSync(SCRATCH, { recursive: true, force: true });
+mkdirSync(SCRATCH, { recursive: true });
+const target = join(SCRATCH, "replication.mjs");
+writeFileSync(target, patched);
+if (!existsSync(join(SCRATCH, "node_modules")))
+  symlinkSync(join(ROOT, "node_modules"), join(SCRATCH, "node_modules"), "dir");
+
+console.log(`\nrunning the PARENT'S code, grade-paper-writing AH+SH only, 3 trials = 24 runs`);
+console.log(`  parent : ${PARENT}`);
+console.log(`  patched: ${target}\n`);
+
+const r = spawnSync(process.execPath, [target, "--mode", "main"], {
+  stdio: "inherit",
+  env: { ...process.env, CLAUDE_PROJECT_DIR: ROOT },
+  cwd: ROOT,
+});
+process.exit(r.status ?? 1);

package/skills/paper-pipeline/repro/README.md

@@ -0,0 +1,129 @@
+# repro/ — raw runs behind the pipeline's own measurements
+
+Every number this directory's evals report has its raw log here. The rule is the repo's: a claim
+whose run lives only in a scratchpad is not reproducible, and the scratchpad is erased at the end of
+the session.
+
+| file | what it backs | headline |
+|---|---|---|
+| `2026-08-07-language-eval-raw.log` | `pipeline-language.eval.mjs`, the full designed run | 32 matched pairs × 3 trials × 2 languages = 192 runs. EN 75/96 (78.1%) vs RU 58/96 (60.4%), −17.7pp; paired sign test 13v2, 17 tied, **p = 0.0074**. Effect present in BOTH translation directions. |
+| `2026-08-07-language-eval-pilot.log` | the same file's one-case pilot | `tighten-paper` at 2 trials, run before committing quota to 192 |
+| `analyze-language-eval.py` | the statistics for the above | Parses the raw log and recomputes everything. Statistics live HERE, downstream of the instrument, so the eval cannot print a number that confirms its own hypothesis. Run: `python3 analyze-language-eval.py 2026-08-07-language-eval-raw.log` |
+| `2026-08-07-description-language-control.log` | `description-language.eval.mjs`, control arm | 12 **fresh** matched pairs × 3 trials × 2 languages = 72 runs. RU 11/36 (31%) vs EN 9/36 (25%). 🔴 **The gap did not appear — and both arms collapsed** from 78%/60% to a quarter. This killed the bilingual-description intervention before it was built. |
+| `2026-08-07-fork-check.log` | the `context: fork` pilot | `cold-read-diff` en 67→75 / ru 67→67; `map-prior-work` en 67→75 / ru 75→75. No regression, which is the point — the risk was that forking changes what the model selects. |
+| `2026-08-07-fork-check2.log` | `context: fork` rollout check | `paper-adversarial-review`, the only one of the five rolled-out skills covered by an eval |
+| `2026-08-08-framing-vs-vocabulary-raw.log` | `framing-vs-vocabulary.eval.mjs`, main 2×2 | 3 skills × 4 cells × 4 prompts × 3 trials = **144 runs, 0 errored**. AH 64% (23/36) · AL 44% (16/36) · SH 61% (22/36) · SL 11% (4/36). Vocabulary main effect **35pp**, framing **18pp**. 🔴 **ONE** competitor misfire in 144 runs; SL is **86% SILENT**. |
+| `2026-08-08-framing-vs-vocabulary.json` | per-run rows for the above | Every run's TARGET/OTHER/SILENT bucket, measured overlap, lexical argmax, and which competitor fired instead. The bucket field is the signal neither earlier eval recorded. |
+| `2026-08-08-framing-vs-vocabulary-oracle.log` | the same file's `--mode oracle`, the C control | 48 closed-book picks over all 37 descriptions, no harness, no skills installed. AH **12/12** · AL 8/12 · SH **12/12** · SL **3/12** (6 answered NONE). Harness sits 14–39pp below the oracle in *every* cell. |
+| `2026-08-08-framing-vs-vocabulary-oracle.json` | per-pick rows for the above | Each prompt's oracle pick vs the intended skill. |
+| `2026-08-08-grade-paper-writing-ablation-raw.log` | `grade-paper-writing-ablation.eval.mjs`, arms A0–A5 | 6 arms × 8 prompts (the parent's AH+SH, verbatim) × 3 trials = **144 runs, 0 errored**. A0 baseline **54% (13/24)** · A1 short 50% · A2 no-subordination 50% · A3 no-negation 54% · A4 no-machinery 46% · A5 renamed **25%**. Every description-text arm inside noise (±~20pp at n=24). |
+| `2026-08-08-grade-paper-writing-ablation-A6-raw.log` | the same file, arm A6 added later | persona-only description (225 chars), 24 runs. **42% (10/24)** vs A1's rubric-only 50% — the pair does not separate the skill's two halves. |
+| `2026-08-08-grade-paper-writing-ablation.json` | per-run rows for both of the above | 56 prompt-rows across 7 arms: TARGET/OTHER/SILENT bucket, measured overlap, description length, competitors fired instead. |
+| `2026-08-08-grade-paper-writing-ablation-oracle.log` | the same file's `--mode oracle`, all 7 arms | 56 closed-book picks. 7/8 or 8/8 on every arm **except A5 at 0/8** — and on 7 of those 8 the oracle answered `grade-paper-writing`, a name absent from the roster it was shown. |
+| `2026-08-08-grade-paper-writing-ablation-oracle.json` | per-pick rows for the above | Each arm × prompt: the oracle's pick vs the arm's intended id. |
+| `2026-08-08-grade-paper-writing-ablation-setupdiff.log` | `--mode setupdiff`, **free** | The mechanical answer to "is the ablation's harness the same as the parent's": options, fixture block, `fired` predicate and `firedSkills` helper all identical; all 7 arm dirs 37 skills with every frontmatter sha256-identical to the real one bar the deliberate mutation. Plus the parent-vs-A0 row-by-row. |
+| `2026-08-08-parent-replication.mjs` | the replication driver | Runs `framing-vs-vocabulary.eval.mjs` **itself** with two asserted patches (main loop → this skill + AH/SH; output path), everything else untouched, against the real `.claude/skills`. Written because the ablation's control contradicted the parent. |
+| `2026-08-08-parent-replication-gpw.log` | that driver's run | 🔴 **The parent's own code returns AH 9/12 + SH 3/12 = 12/24 (50%)** hours after the same code returned 3/12 + 2/12 = 5/24 (21%). |
+| `2026-08-08-parent-replication-gpw.json` | per-run rows for the above | The parent's own row schema, so it diffs directly against `2026-08-08-framing-vs-vocabulary.json`. |
+
+## How to read these, and the trap in them
+
+**The two evals disagree, and the disagreement is the finding.** Same three skills, same fixture,
+same method:
+
+```
+prompts that NAME the action      ("paper got bloated, what to cut?")   78% EN / 60% RU
+prompts that describe a SITUATION ("text won't fit, need to decide
+    what to sacrifice and where to move it")                          25% EN / 31% RU
+```
+
+A factor of two to three. The second set was written deliberately to share **no vocabulary** with the
+skill descriptions — because a prompt reusing a description's words measures how well the description
+was copied, not whether it works. Remove the shared vocabulary and recall falls off a cliff.
+
+So the control does **not** refute the language finding: at 25–31% both arms sit on the floor and a
+6pp difference there has no power behind it. What it establishes is bigger and less welcome — these
+skills mostly fire when the user happens to use the description's own verb.
+
+**Do not compare a number from one log against a number from the other as if they were the same
+measurement.** They share a method and differ in the one variable that turned out to dominate.
+
+## 2026-08-08 — the disagreement above, explained
+
+`framing-vs-vocabulary.eval.mjs` crossed the two factors the disagreement confounded (every
+"names the action" prompt was *also* high-overlap) and recorded, for the first time, **what fired
+instead**. Result, in one line:
+
+> The selector fires when it has **either** the description's vocabulary **or** an explicit request
+> for the deliverable. It collapses only when it has neither — which is how people type.
+
+- **The symptom-vs-action framing hypothesis is refuted.** Pure symptom statements written in the
+  description's own words (`SH`, 61%) fire as well as explicit requests (`AH`, 64%) — 83% vs 83%
+  once the one dead skill is excluded. Descriptions already name symptoms; that is not what is
+  missing.
+- **Ambiguity is refuted too, in the form it was posed.** If the oblique prompts were losing to
+  plausible rivals, competitors would be taking the runs. They are not: **1 misfire in 144 runs**,
+  and 86% of the collapsed cell resolves *no skill at all*. Nothing competes; nothing matches.
+- **But the descriptions really are under-determining.** A closed-book reader handed all 37
+  descriptions also collapses on those prompts (3/12, six NONE). So the gap is **coverage**, not
+  ambiguity — different defect, different fix.
+- **Unexplained and possibly bigger:** the harness runs 14–39pp below that closed-book reader in
+  *every* cell, including the ones the reader gets 12/12. A third of the easiest prompts resolve
+  nothing. Size measured here; cause not.
+
+Full reading, caveats, and what it still cannot settle: the header of
+`../framing-vs-vocabulary.eval.mjs`. **No `SKILL.md` was changed on the strength of this run.**
+
+## 🔴 2026-08-08, later the same day — the "dead skill" premise did not replicate
+
+`grade-paper-writing-ablation.eval.mjs` set out to explain WHY `grade-paper-writing` fired only
+12% of the time, by mutating its description one factor at a time (7 arms × 8 prompts × 3 trials).
+Its **control arm broke the premise instead**:
+
+```
+grade-paper-writing, the parent's own eight AH+SH prompts, same fixture, same 36 competitors
+  parent, this morning                                3/12 AH · 2/12 SH   =  5/24   21%
+  ablation control A0, hours later, same description  7/12 AH · 6/12 SH   = 13/24   54%
+  THE PARENT'S OWN CODE, re-run, real .claude/skills  9/12 AH · 3/12 SH   = 12/24   50%
+```
+
+The third line is why this is not a setup bug: it is `framing-vs-vocabulary.eval.mjs` itself,
+patched only to run one skill and two cells, against the real skills directory. `--mode setupdiff`
+separately proved the two harnesses identical option by option and hash by hash.
+
+- **Every description-text arm is flat** — short −4pp, no-subordination −4pp, no-negation 0pp,
+  no-machinery −8pp, persona-only −12pp, against a ±20pp interval. The cause was not in the text.
+- **The one arm outside noise is an artifact.** Renaming the skill (description byte-identical)
+  cost 29pp and collapsed the oracle to 0/8 — because **six other descriptions name
+  `grade-paper-writing`** (`cold-read-diff`, `draft-paper`, `harden-paper`,
+  `paper-adversarial-review`, `pc-panel-review`, `tighten-paper`) and the reader follows the
+  cross-reference to a skill that no longer exists. A rename here is not a local edit.
+- **What did reproduce:** the closed-book oracle scores 7–8 of 8 on every non-artifact arm while
+  the harness delivers 42–54% on the same text. That 25–45pp deficit is the parent's point 5, and
+  no description edit touched it.
+
+⚠️ **The 12% figure is quoted as measured fact inside `harden-paper` and `pc-panel-review`**
+(commit `38a17ef`). The advice there — invoke the skill by name instead of relying on selection —
+survives at 50%; the number does not. Neither file was edited on the strength of this run either.
+
+Full reading and the confounds, including the un-pre-registered arm-order one:
+the header of `../grade-paper-writing-ablation.eval.mjs`.
+
+## ⚠️ These logs are gitignored — force-add them
+
+`.gitignore` line 73 is `*.log`, meant for LaTeX build artifacts. It silently swallows every raw log
+in this directory. The table above says "every number has its raw log here"; git tracks **none** of
+the `.log` files, so a fresh clone has the claims and not the evidence — exactly the artifact loss
+this directory exists to prevent. New logs must be added with:
+
+```
+git add -f .claude/skills/paper-pipeline/repro/<name>.log
+```
+
+The `2026-08-08-*` logs were added this way. The four `2026-08-07-*` logs are still untracked and
+should be force-added the next time someone touches this directory.
+
+## Cost
+
+Billed to a Claude subscription, $0 metered; ~$0.14 API-equivalent per run. The 192-run design cost
+~$26.59. That is why the pilot exists and why the `after` arm of the description A/B was never run.

package/skills/paper-pipeline/repro/analyze-language-eval.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+analyze-language-eval.py — the statistics behind the language finding, from the raw run log.
+
+    python3 .claude/skills/paper-pipeline/repro/analyze-language-eval.py \
+        .claude/skills/paper-pipeline/repro/2026-08-07-language-eval-raw.log
+
+WHY THIS FILE EXISTS. `pipeline-language.eval.mjs` prints rates and deltas; it deliberately does
+NOT print a p-value, because a gate or a statistic inside the measuring instrument is how a check
+that can only confirm its own hypothesis gets written. The inference lives here, downstream, and it
+reads the same log a human reads.
+
+WHY A SIGN TEST AND NOT A CHI-SQUARE ON THE TOTALS. The design is MATCHED PAIRS: each prompt exists
+in both languages with the same content, so the two arms are not independent samples and a
+two-proportion test on 96-vs-96 throws away the pairing that the design was built to get. The
+paired test asks the question the design actually poses — for a given question, does the English
+form fire more often than the Russian one — and tied pairs (the majority here) correctly contribute
+no evidence either way.
+
+The unpaired Fisher exact is printed alongside only as a sanity check that the two tests agree on
+direction and rough magnitude. It is NOT the headline number.
+
+WHAT THE NUMBERS CANNOT SAY, and the run they replace said louder:
+  - the prior single-trial run put the gap at 93% English vs 56% Russian (-37pp). This one puts it
+    at 78% vs 60% (-18pp). The Russian figure replicated; the ENGLISH one fell by 15 points. The
+    original gap was inflated by an English arm of 14 prompts measured once each.
+  - the translations in both directions were written by the same model family being measured, and
+    no native speaker has reviewed them.
+"""
+import math
+import re
+import sys
+from pathlib import Path
+
+TRIALS = 3
+
+
+def log_c(n, k):
+    return math.lgamma(n + 1) - math.lgamma(k + 1) - math.lgamma(n - k + 1)
+
+
+def fisher_two_sided(a, b, c, d):
+    """Two-sided Fisher exact on the 2x2 [[a,b],[c,d]], summing all tables no likelier than observed."""
+    n, r1, c1 = a + b + c + d, a + b, a + c
+
+    def pmf(x):
+        return math.exp(log_c(c1, x) + log_c(n - c1, r1 - x) - log_c(n, r1))
+
+    p0 = pmf(a)
+    lo, hi = max(0, r1 - (n - c1)), min(r1, c1)
+    return min(1.0, sum(p for x in range(lo, hi + 1) if (p := pmf(x)) <= p0 * (1 + 1e-9)))
+
+
+def binom_two_sided(k, m):
+    """Exact two-sided binomial test against p=0.5 — the sign test's p-value."""
+    if m == 0:
+        return float("nan")
+    pk = [math.exp(log_c(m, i) + m * math.log(0.5)) for i in range(m + 1)]
+    return min(1.0, sum(x for x in pk if x <= pk[k] * (1 + 1e-9)))
+
+
+def parse(path):
+    """Recover (skill, origin, en_fired, ru_fired) per matched pair from the run log."""
+    cases, cur, lang = [], None, None
+    for line in Path(path).read_text(encoding="utf-8").splitlines():
+        if m := re.match(r"^=== (\S+) —", line):
+            cur = {"skill": m.group(1), "en": [], "ru": []}
+            cases.append(cur)
+        elif m := re.match(r"^  (en|ru): ", line):
+            lang = m.group(1)
+        elif (m := re.match(r"^    ([01]\.\d\d)  \[(en|ru)\] (.*)$", line)) and cur and lang:
+            cur[lang].append((float(m.group(1)), m.group(2)))
+    # A case whose two arms did not both record four prompts is a truncated run, not a data point.
+    cases = [c for c in cases if len(c["en"]) == 4 and len(c["ru"]) == 4]
+    return [
+        (c["skill"], c["en"][i][1], round(c["en"][i][0] * TRIALS), round(c["ru"][i][0] * TRIALS))
+        for c in cases
+        for i in range(4)
+    ]
+
+
+def report(pairs, label):
+    en, ru = sum(p[2] for p in pairs), sum(p[3] for p in pairs)
+    n = len(pairs) * TRIALS
+    pos = sum(1 for p in pairs if p[2] > p[3])
+    neg = sum(1 for p in pairs if p[2] < p[3])
+    tie = len(pairs) - pos - neg
+    print(
+        f"{label:14} EN {en:3}/{n} ({en / n:5.1%})   RU {ru:3}/{n} ({ru / n:5.1%})   "
+        f"delta {(ru - en) / n * 100:+5.1f}pp   sign {pos}v{neg} (tied {tie}) "
+        f"p={binom_two_sided(pos, pos + neg):.4f}"
+    )
+    return en, ru, n
+
+
+def main():
+    path = sys.argv[1] if len(sys.argv) > 1 else Path(__file__).with_name("2026-08-07-language-eval-raw.log")
+    pairs = parse(path)
+    if not pairs:
+        sys.exit(f"no matched pairs parsed from {path} — is this a complete run log?")
+    print(f"{len(pairs)} matched pairs x {TRIALS} trials x 2 languages = {len(pairs) * TRIALS * 2} runs\n")
+
+    en, ru, n = report(pairs, "OVERALL")
+    print(f"{'':14} unpaired Fisher exact two-sided p = {fisher_two_sided(en, n - en, ru, n - ru):.4f} (sanity check only)\n")
+
+    print("SPLIT BY ORIGIN — does the gap survive when the RUSSIAN side is the original text?")
+    for origin in ("en", "ru"):
+        report([p for p in pairs if p[1] == origin], f"{origin}-original")
+
+    print("\nPER SKILL (read as scenery: n=12 per cell, one run moves a cell 8 points)")
+    for skill in dict.fromkeys(p[0] for p in pairs):
+        report([p for p in pairs if p[0] == skill], skill)
+
+
+if __name__ == "__main__":
+    main()