paperlint 10.0.0 → 10.1.1

package/lib/skill-trigger-cases.test.ts

@@ -0,0 +1,211 @@
+/**
+ * The shared trigger table is FROZEN while every skill moves its trigger cases into its own eval
+ * file (#218).
+ *
+ * A skill's trigger cases belong to the skill: `skills/<name>/<name>.eval.ts` default-exports
+ * `defineEval({ measureTriggerRate: { prompts, irrelevantPrompts, … } })`, and `vigiles eval`
+ * discovers and runs it. `lib/skill-trigger-cases.mjs` holds the skills that have not moved yet.
+ * This file keeps that table from growing (a new skill writes its own eval file, never a row),
+ * makes `FROZEN` shrink in step with the moves, and refuses a skill that is in both places.
+ *
+ * The table is read by RUNNING the module in a child process, as the evals do — not by importing
+ * it here, which would need types for a JavaScript module, and not by matching its text. An eval
+ * file is classified by PARSING it: importing one that still calls the paid kit would spend money.
+ *
+ * Each check is shown both ways: on the real files (passes) and on a planted input (fails).
+ */
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { globSync, readFileSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { pathToFileURL } from "node:url";
+import ts from "typescript";
+import { assertPromptDiversity } from "vigiles";
+import { test } from "vitest";
+import { z } from "zod";
+import { useTempDir, writeTree } from "../test/support.ts";
+import grid from "../skills/run-agent-grid/run-agent-grid.eval.ts";
+import labels from "../skills/label-agent-runs/label-agent-runs.eval.ts";
+
+const ROOT = join(import.meta.dirname, "..");
+
+/** The skills still in the shared table. It may only SHRINK: remove a name when its skill moves. */
+const FROZEN: readonly string[] = [
+  "analyze-sibling-paper",
+  "argument-arc",
+  "build-benchmark",
+  "build-label-page",
+  "camera-ready",
+  "cold-read-diff",
+  "draft-paper",
+  "extend-paper",
+  "find-venue",
+  "grade-paper-writing",
+  "harden-paper",
+  "map-prior-work",
+  "osf-artifact-upload",
+  "paper-adversarial-review",
+  "paper-pipeline",
+  "paper-status",
+  "pc-panel-review",
+  "plan-paper-timeline",
+  "render-paper",
+  "research-ideate",
+  "study-accepted-papers",
+  "submission-portal",
+  "submit-paper",
+  "sweep-design-space",
+  "tighten-paper",
+  "verify-citations",
+];
+
+const Row = z.object({ skill: z.string(), prompts: z.array(z.string()) });
+type Row = z.infer<typeof Row>;
+
+/** The shared table's rows, read by running the module. */
+function sharedTable(): Row[] {
+  const url = pathToFileURL(join(ROOT, "lib", "skill-trigger-cases.mjs")).href;
+  const r = spawnSync(
+    process.execPath,
+    [
+      "--input-type=module",
+      "-e",
+      `const m = await import(${JSON.stringify(url)});` +
+        "console.log(JSON.stringify(m.CASES.map(({ skill, prompts }) => ({ skill, prompts }))));",
+    ],
+    { cwd: ROOT, encoding: "utf8" },
+  );
+  assert.equal(r.status, 0, r.stderr);
+  return z.array(Row).parse(JSON.parse(r.stdout));
+}
+const SHARED = sharedTable();
+
+/** What is wrong with the shared table against the frozen set, as messages; [] when nothing. */
+function frozenProblems(
+  table: readonly string[],
+  frozen: readonly string[],
+): string[] {
+  return [
+    ...table
+      .filter((n) => !frozen.includes(n))
+      .map(
+        (n) =>
+          `${n} is a new row in lib/skill-trigger-cases.mjs, which is frozen: declare its trigger ` +
+          `cases in skills/${n}/${n}.eval.ts with defineEval({ measureTriggerRate }) instead`,
+      ),
+    ...frozen
+      .filter((n) => !table.includes(n))
+      .map(
+        (n) =>
+          `${n} left the shared table: remove it from FROZEN in lib/skill-trigger-cases.test.ts — ` +
+          `the frozen set only shrinks`,
+      ),
+  ];
+}
+
+/** Is this the `measureTriggerRate` key of an object literal? */
+const isTriggerKey = (p: ts.ObjectLiteralElementLike): boolean =>
+  p.name !== undefined &&
+  ts.isIdentifier(p.name) &&
+  p.name.text === "measureTriggerRate";
+
+/** Does this eval file default-export `defineEval({ measureTriggerRate: … })`? Parsed, not run. */
+function declaresTriggerRate(file: string): boolean {
+  const sf = ts.createSourceFile(
+    file,
+    readFileSync(file, "utf8"),
+    ts.ScriptTarget.Latest,
+  );
+  return sf.statements.some(
+    (s) =>
+      ts.isExportAssignment(s) &&
+      ts.isCallExpression(s.expression) &&
+      ts.isIdentifier(s.expression.expression) &&
+      s.expression.expression.text === "defineEval" &&
+      s.expression.arguments.some(
+        (a) =>
+          ts.isObjectLiteralExpression(a) && a.properties.some(isTriggerKey),
+      ),
+  );
+}
+
+/** Skills under `dir` whose own `<name>.eval.ts` declares their trigger cases. */
+const colocated = (dir: string): string[] =>
+  globSync("*/*.eval.ts", { cwd: dir })
+    .filter((f) => basename(f) === `${dirname(f)}.eval.ts`)
+    .filter((f) => declaresTriggerRate(join(dir, f)))
+    .map((f) => dirname(f))
+    .sort();
+
+test("the shared table names exactly the frozen skills", () => {
+  assert.deepEqual(
+    frozenProblems(
+      SHARED.map((r) => r.skill),
+      FROZEN,
+    ),
+    [],
+  );
+});
+
+test("a planted new row, and a frozen name that left the table, both fail", () => {
+  const names = SHARED.map((r) => r.skill);
+  assert.deepEqual(frozenProblems([...names, "new-skill"], FROZEN), [
+    "new-skill is a new row in lib/skill-trigger-cases.mjs, which is frozen: declare its trigger cases in skills/new-skill/new-skill.eval.ts with defineEval({ measureTriggerRate }) instead",
+  ]);
+  assert.deepEqual(
+    frozenProblems(
+      names.filter((n) => n !== "camera-ready"),
+      FROZEN,
+    ),
+    [
+      "camera-ready left the shared table: remove it from FROZEN in lib/skill-trigger-cases.test.ts — the frozen set only shrinks",
+    ],
+  );
+});
+
+test("no skill that declares its own trigger cases is also in the shared table", () => {
+  const mine = colocated(join(ROOT, "skills"));
+  assert.deepEqual(mine, ["label-agent-runs", "run-agent-grid"]);
+  assert.deepEqual(
+    mine.filter((s) => SHARED.some((r) => r.skill === s)),
+    [],
+  );
+});
+
+const WORK = useTempDir("trigger-cases-");
+
+test("an eval file is classified by its default export, parsed: defineEval yes, the kit no", () => {
+  const dir = writeTree(join(WORK, "skills"), {
+    "own/own.eval.ts":
+      'import { defineEval } from "vigiles";\nexport default defineEval({ measureTriggerRate: { prompts: [] } });\n',
+    "kit/kit.eval.ts":
+      'import { runSkillTriggerEval } from "../../lib/skill-eval-kit.mjs";\nawait runSkillTriggerEval("kit");\n',
+    "other/other.eval.ts":
+      'import { defineEval } from "vigiles";\nexport default defineEval({ measure: { task: "x" } });\n',
+    "named/elsewhere.eval.ts":
+      'import { defineEval } from "vigiles";\nexport default defineEval({ measureTriggerRate: { prompts: [] } });\n',
+  });
+  assert.deepEqual(colocated(dir), ["own"]);
+  // A planted double source: a skill that declares its own cases and is also a table row.
+  assert.deepEqual(
+    colocated(dir).filter((s) => [...FROZEN, "own"].includes(s)),
+    ["own"],
+  );
+});
+
+test("each new eval declares diverse prompts on both sides, none of them a table row's", () => {
+  const tablePrompts = new Set(SHARED.flatMap((r) => r.prompts));
+  for (const def of [grid, labels]) {
+    const spec = def.measureTriggerRate;
+    assert.ok(spec, "the eval declares measureTriggerRate");
+    assertPromptDiversity(spec.prompts, { minPrompts: 4, label: "prompts" });
+    assertPromptDiversity(spec.irrelevantPrompts ?? [], {
+      minPrompts: 4,
+      label: "irrelevantPrompts",
+    });
+    assert.deepEqual(
+      spec.prompts.filter((p) => tablePrompts.has(p)),
+      [],
+    );
+  }
+});

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "paperlint",
-  "version": "10.0.0",
+  "version": "10.1.1",
   "description": "A linter for scientific papers written in LaTeX: catches mistakes before you submit to a conference or journal.",
   "type": "module",
   "license": "MIT",
   "scripts": {
-    "build": "tsc -p tsconfig.pkg.json && tsc",
-    "prepare": "tsc -p tsconfig.pkg.json && tsc",
+    "build": "tsc -b",
+    "prepare": "tsc -b",
     "test": "node scripts/test.ts",
     "test:e2e": "vitest run --project 'e2e-*'",
     "test:e2e:install": "vitest run --project e2e-install",
@@ -101,6 +101,7 @@
     "action.yml",
     "bin",
     "dist",
+    "!dist/.tsbuildinfo",
     "docs",
     "eslint-rules",
     "eslint.config.mjs",
@@ -114,8 +115,11 @@
     "skills",
     "src",
     "templates",
+    "tsconfig.base.json",
     "tsconfig.json",
-    "tsconfig.pkg.json"
+    "tsconfig.pkg.json",
+    "tsconfig.skills.json",
+    "tsconfig.src.json"
   ],
   "imports": {
     "#lib/*": {

package/scripts/coverage-config.test.ts

@@ -165,7 +165,7 @@ test("every evalOnly exclusion is imported only by an eval or another evalOnly m
       : [];
   });
   const outsiders = importers.filter(
-    (f) => !f.endsWith(".eval.mjs") && !evalOnly.has(join(ROOT, f)),
+    (f) => !/\.eval\.[cm]?[jt]s$/.test(f) && !evalOnly.has(join(ROOT, f)),
   );
   assert.deepEqual(
     { outsiders, anyImporter: importers.length > 0 },
@@ -398,8 +398,8 @@ test("the exclude list is exactly the justified set", () => {
     // tests themselves
     "**/*.harness.*",
     "**/*.test.*",
-    // PAID LLM evaluations, never run by `npm test`
-    "**/*.eval.mjs",
+    // PAID LLM evaluations, never run by `npm test` (`.mjs` or `.ts`)
+    "**/*.eval.*",
     // skill and CLAUDE.md sources vigiles compiles; a glob's `**` skips a dot directory
     "**/*.spec.ts",
     ".github/*.spec.ts",
@@ -423,6 +423,8 @@ test("the exclude list is exactly the justified set", () => {
     "skills/**/repro/**",
     "skills/**/references/**",
     "docs/prior-art/repro/**",
+    // a skill's scripts compiled beside their .ts (tsconfig.skills.json); the .ts is what is measured
+    "skills/*/scripts/*.js",
     // the test runner's own configuration, read by vitest, not code the package runs
     "vitest.config.ts",
   ]);

package/skills/build-benchmark/SKILL.md

@@ -5,7 +5,7 @@ context: fork
 allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Agent]
 ---
 
-<!-- vigiles:sha256:5eb2ba9561e082d9 compiled from skills/build-benchmark/SKILL.md.spec.ts -->
+<!-- vigiles:sha256:eed7d0afdfee87a6 compiled from skills/build-benchmark/SKILL.md.spec.ts -->
 
 # build-benchmark — the study + the artifact reviewers can run
 
@@ -167,6 +167,9 @@ Before submitting, verify the artifact you built with the reviewer-side reproduc
 - **argument-arc (frame mode)** — 🔴 **hard input.** Owns row `frame`, the stated claim. No `frame`, no study.
 - **research-ideate** (upstream) — supplies the contribution and the pitfall to invert; this skill turns
   it into evidence.
+- **run-agent-grid** — when the study is a grid of agent runs: this skill designs it, that one runs
+  it (registry gate, sandbox probe, smoke, locked driver, overlap audit, committed runs).
+- **label-agent-runs** — the outcome labels of those runs, before any statistic here is computed.
 - **build-label-page** — when the study needs a person to label items (gold labels, a blind review of
   outputs): it builds the page, publishes it and exports the answers into the study's `data/`.
 - **draft-paper** — the study's stats, bound, and artifact-number table feed Methods/Results/Availability.

package/skills/build-benchmark/SKILL.md.spec.ts

@@ -173,6 +173,9 @@ Before submitting, verify the artifact you built with the reviewer-side reproduc
 - **argument-arc (frame mode)** — 🔴 **hard input.** Owns row \`frame\`, the stated claim. No \`frame\`, no study.
 - **research-ideate** (upstream) — supplies the contribution and the pitfall to invert; this skill turns
   it into evidence.
+- **run-agent-grid** — when the study is a grid of agent runs: this skill designs it, that one runs
+  it (registry gate, sandbox probe, smoke, locked driver, overlap audit, committed runs).
+- **label-agent-runs** — the outcome labels of those runs, before any statistic here is computed.
 - **build-label-page** — when the study needs a person to label items (gold labels, a blind review of
   outputs): it builds the page, publishes it and exports the answers into the study's \`data/\`.
 - **draft-paper** — the study's stats, bound, and artifact-number table feed Methods/Results/Availability.

package/skills/build-label-page/SKILL.md

@@ -4,7 +4,7 @@ description: Build a phone labelling page published as a claude.ai Artifact with
 allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Artifact, ArtifactData]
 ---
 
-<!-- vigiles:sha256:894d8a2df3f5c319 compiled from skills/build-label-page/SKILL.md.spec.ts -->
+<!-- vigiles:sha256:592d7b4f8f27c4ce compiled from skills/build-label-page/SKILL.md.spec.ts -->
 
 # build-label-page — a page the person labels on a phone, and the labels back in the repo
 
@@ -112,6 +112,8 @@ Do not iterate on a page you cannot see.
 - **build-benchmark** — owns the study design and the inter-rater rule: agreement with the person who
   wrote the codebook measures trainability, not reliability, so the labeller for gold labels is
   someone who did not write the scheme.
+- **label-agent-runs** — when the items are the runs of an agent grid: the codebook, which runs need
+  a hand label, the second rater and the label file this page's answers go into.
 - **draft-paper** — the exported labels, the familiarity filter and the complete-item count feed the
   Methods and the threats.
 - **submit-paper** — a published artifact page is not anonymous. Do not link it from an anonymized

package/skills/build-label-page/SKILL.md.spec.ts

@@ -122,6 +122,8 @@ Do not iterate on a page you cannot see.
 - **build-benchmark** — owns the study design and the inter-rater rule: agreement with the person who
   wrote the codebook measures trainability, not reliability, so the labeller for gold labels is
   someone who did not write the scheme.
+- **label-agent-runs** — when the items are the runs of an agent grid: the codebook, which runs need
+  a hand label, the second rater and the label file this page's answers go into.
 - **draft-paper** — the exported labels, the familiarity filter and the complete-item count feed the
   Methods and the threats.
 - **submit-paper** — a published artifact page is not anonymous. Do not link it from an anonymized

package/skills/label-agent-runs/SKILL.md

@@ -0,0 +1,103 @@
+---
+name: label-agent-runs
+description: "Label the outcomes of an agent experiment grid so the labels survive review: a codebook fixed before the first label (each label with a definition and a borderline example), the automatic classifier used only as a pre-sort, every run without artifact-level evidence read in full and labelled by hand, auto-vs-hand disagreements reported, a blind second rater on a different model with agreement (Cohen's kappa, percent with a CI), labels kept as data and never edited after the analysis without a dated note. Use after the runs of a grid are in and before any outcome number is computed: \"label the runs\", \"the classifier missed some outcomes\", \"second rater\", \"inter-rater agreement on the transcripts\". Ships labels.js (validate, agree). Composes with run-agent-grid (upstream), build-label-page (the hand-labelling page) and build-benchmark (the analysis)."
+allowed-tools: [Read, Write, Edit, Grep, Glob, Bash, Agent]
+---
+
+<!-- vigiles:sha256:91ebf75c83bbd845 compiled from skills/label-agent-runs/SKILL.md.spec.ts -->
+
+# label-agent-runs — turning finished agent runs into outcome labels a reviewer can trust
+
+After `run-agent-grid` has produced the runs, every outcome number in the paper comes from a label.
+This skill is the procedure for producing those labels: what is fixed before labelling, what may be
+decided by a script, what must be read by a person, how a second rater is run, and how the labels
+are kept. `build-benchmark` turns the labels into statistics; `build-label-page` is the page a person
+labels on.
+
+Each rule names the failure behind it and what holds it — a refusal in `skills/label-agent-runs/scripts/labels.ts` (run as the compiled `scripts/labels.js`) or
+**prose only**. Formats (codebook, label file, notes, rater instruction):
+[`references/formats.md`](references/formats.md).
+
+```bash
+node ${CLAUDE_SKILL_DIR}/scripts/labels.js validate labels.tsv --codebook codebook.json --ledger attempts.jsonl
+node ${CLAUDE_SKILL_DIR}/scripts/labels.js agree auto.tsv hand.tsv
+```
+
+Exit codes: 0 valid or reported · 1 a finding · 2 usage.
+
+## 1. The codebook comes first
+
+- Before the first label: every outcome label with a **definition** and a **borderline example** (the
+  case nearest the line, and which side it falls on). A codebook written while labelling bends to the
+  runs already seen.
+- **Held by:** `validate` refuses a codebook that is not committed, differs from its committed
+  version, lacks a definition or a borderline example for a label, or has ANY commit later than the
+  label file's first commit; and a label the codebook does not define. A codebook changed after
+  labelling began is refused rather than excused by a note: the labels already given were given
+  under the old definitions, so the change starts a NEW label file, committed after it, and every
+  run is labelled again under the new codebook.
+
+## 2. The automatic classifier is a pre-sort, not a label
+
+- An automatic outcome detector misses classes. The case that produced this rule: a detector of
+  "the agent handed the decision back to the user" keyed on a question mark, and missed most of the
+  deferrals in one grid — they were phrased as statements ("I'll leave that decision to you.").
+- **Mechanically certain** means artifact-level evidence decides the outcome: the task's check passes
+  or fails on the committed state, the required file exists or does not. Only those runs keep the
+  automatic label.
+- **Every other run is read in full** — the final message AND the action sequence (the tool calls,
+  in order) — and labelled by hand. Use `build-label-page` when there are more than a handful.
+- **The disagreements between the pre-sort and the hand labels are reported**, with counts per
+  label, not folded away: they are the measured error of the classifier.
+- **Held by:** `validate --ledger` refuses a label file that misses a completed run, or labels a cell
+  with no completed run; `agree auto.tsv hand.tsv` lists every disagreement. **Prose only:** that
+  the uncertain runs were actually read.
+
+## 3. A finished run still gets a check of what changed outside the task
+
+- A run that completed its task can also have changed what it was not asked to touch (deleted a
+  test, edited a fixture, rewrote a config). Before a "solved" label stands, list what the run
+  changed outside the task's paths:
+
+  ```bash
+  git -C <working-copy> diff --name-only <base> HEAD -- . ':(exclude)<task path>'
+  ```
+
+  Anything listed is read and either explained in the notes or reflected in the label.
+- **Prose only.**
+
+## 4. A second rater, blind, on a different model
+
+- A second rater labels **the same runs**: a different model from the first rater's, given **the same
+  instruction byte for byte** (one committed file; record its hash in the notes), and **blind** to the
+  first labels — it receives the runs' messages and actions, never another rater's label.
+- Report agreement: Cohen's kappa and percent agreement with its 95% interval, over n, with the
+  disagreements. Resolve disagreements by re-reading the run, and record each resolution in the notes.
+- A person may replace **one** rater, never all of them: at least one rater stays independent of the
+  person who wrote the codebook.
+- **Held by:** `agree` refuses two files that label different cells, and computes n, percent with a
+  Wilson interval, kappa (or says it is undefined when only one label occurs), and every
+  disagreement. **Prose only:** blindness, the different model, the identical instruction.
+
+## 5. Labels are data, and frozen after the analysis
+
+- The labels live in a TSV with the header `agent model cell label` (tab-separated), one row per
+  run, beside a notes file. Every relabel gets a note under a dated heading (`## YYYY-MM-DD`) naming
+  the cell and quoting the run verbatim (a `>` line) — the quote is the evidence for the label.
+- Once an analysis has read the labels, they are not edited without such a note.
+- **Held by:** `validate --analysed-at <rev> --notes notes.md` compares the cells at the revision the
+  analysis read with the cells now, and refuses every cell that was relabelled, ADDED or REMOVED
+  since and is not named in a dated section with a quote. Added cells count too: the analysis did
+  not see them. Tag the revision when the analysis runs (`git tag analysis-<date>`).
+
+## What to report
+
+The codebook (and its revision), the share of runs labelled automatically vs by hand, the
+pre-sort's disagreements with the hand labels, the second rater's agreement (kappa, percent with CI,
+n), and every relabel after the analysis with its note.
+
+## Compose with
+
+- **run-agent-grid** (upstream) — the runs, the attempt ledger and the cell ids the labels use.
+- **build-label-page** — the page a person labels on, and the export into the study's `data/`.
+- **build-benchmark** — the statistics computed from the labels.

package/skills/label-agent-runs/SKILL.md.spec.ts

@@ -0,0 +1,106 @@
+import { experimental_skill, file, prose } from "vigiles/spec";
+import { COMMANDS } from "./scripts/labels-commands.ts";
+
+export default experimental_skill({
+  name: "label-agent-runs",
+  description:
+    'Label the outcomes of an agent experiment grid so the labels survive review: a codebook fixed before the first label (each label with a definition and a borderline example), the automatic classifier used only as a pre-sort, every run without artifact-level evidence read in full and labelled by hand, auto-vs-hand disagreements reported, a blind second rater on a different model with agreement (Cohen\'s kappa, percent with a CI), labels kept as data and never edited after the analysis without a dated note. Use after the runs of a grid are in and before any outcome number is computed: "label the runs", "the classifier missed some outcomes", "second rater", "inter-rater agreement on the transcripts". Ships labels.js (validate, agree). Composes with run-agent-grid (upstream), build-label-page (the hand-labelling page) and build-benchmark (the analysis).',
+  tools: ["Read", "Write", "Edit", "Grep", "Glob", "Bash", "Agent"],
+  body: prose`
+# label-agent-runs — turning finished agent runs into outcome labels a reviewer can trust
+
+After \`run-agent-grid\` has produced the runs, every outcome number in the paper comes from a label.
+This skill is the procedure for producing those labels: what is fixed before labelling, what may be
+decided by a script, what must be read by a person, how a second rater is run, and how the labels
+are kept. \`build-benchmark\` turns the labels into statistics; \`build-label-page\` is the page a person
+labels on.
+
+Each rule names the failure behind it and what holds it — a refusal in ${file("skills/label-agent-runs/scripts/labels.ts")} (run as the compiled \`scripts/labels.js\`) or
+**prose only**. Formats (codebook, label file, notes, rater instruction):
+[\`references/formats.md\`](references/formats.md).
+
+\`\`\`bash
+node \${CLAUDE_SKILL_DIR}/scripts/labels.js ${COMMANDS.validate} labels.tsv --codebook codebook.json --ledger attempts.jsonl
+node \${CLAUDE_SKILL_DIR}/scripts/labels.js ${COMMANDS.agree} auto.tsv hand.tsv
+\`\`\`
+
+Exit codes: 0 valid or reported · 1 a finding · 2 usage.
+
+## 1. The codebook comes first
+
+- Before the first label: every outcome label with a **definition** and a **borderline example** (the
+  case nearest the line, and which side it falls on). A codebook written while labelling bends to the
+  runs already seen.
+- **Held by:** \`${COMMANDS.validate}\` refuses a codebook that is not committed, differs from its committed
+  version, lacks a definition or a borderline example for a label, or has ANY commit later than the
+  label file's first commit; and a label the codebook does not define. A codebook changed after
+  labelling began is refused rather than excused by a note: the labels already given were given
+  under the old definitions, so the change starts a NEW label file, committed after it, and every
+  run is labelled again under the new codebook.
+
+## 2. The automatic classifier is a pre-sort, not a label
+
+- An automatic outcome detector misses classes. The case that produced this rule: a detector of
+  "the agent handed the decision back to the user" keyed on a question mark, and missed most of the
+  deferrals in one grid — they were phrased as statements ("I'll leave that decision to you.").
+- **Mechanically certain** means artifact-level evidence decides the outcome: the task's check passes
+  or fails on the committed state, the required file exists or does not. Only those runs keep the
+  automatic label.
+- **Every other run is read in full** — the final message AND the action sequence (the tool calls,
+  in order) — and labelled by hand. Use \`build-label-page\` when there are more than a handful.
+- **The disagreements between the pre-sort and the hand labels are reported**, with counts per
+  label, not folded away: they are the measured error of the classifier.
+- **Held by:** \`validate --ledger\` refuses a label file that misses a completed run, or labels a cell
+  with no completed run; \`agree auto.tsv hand.tsv\` lists every disagreement. **Prose only:** that
+  the uncertain runs were actually read.
+
+## 3. A finished run still gets a check of what changed outside the task
+
+- A run that completed its task can also have changed what it was not asked to touch (deleted a
+  test, edited a fixture, rewrote a config). Before a "solved" label stands, list what the run
+  changed outside the task's paths:
+
+  \`\`\`bash
+  git -C <working-copy> diff --name-only <base> HEAD -- . ':(exclude)<task path>'
+  \`\`\`
+
+  Anything listed is read and either explained in the notes or reflected in the label.
+- **Prose only.**
+
+## 4. A second rater, blind, on a different model
+
+- A second rater labels **the same runs**: a different model from the first rater's, given **the same
+  instruction byte for byte** (one committed file; record its hash in the notes), and **blind** to the
+  first labels — it receives the runs' messages and actions, never another rater's label.
+- Report agreement: Cohen's kappa and percent agreement with its 95% interval, over n, with the
+  disagreements. Resolve disagreements by re-reading the run, and record each resolution in the notes.
+- A person may replace **one** rater, never all of them: at least one rater stays independent of the
+  person who wrote the codebook.
+- **Held by:** \`${COMMANDS.agree}\` refuses two files that label different cells, and computes n, percent with a
+  Wilson interval, kappa (or says it is undefined when only one label occurs), and every
+  disagreement. **Prose only:** blindness, the different model, the identical instruction.
+
+## 5. Labels are data, and frozen after the analysis
+
+- The labels live in a TSV with the header \`agent model cell label\` (tab-separated), one row per
+  run, beside a notes file. Every relabel gets a note under a dated heading (\`## YYYY-MM-DD\`) naming
+  the cell and quoting the run verbatim (a \`>\` line) — the quote is the evidence for the label.
+- Once an analysis has read the labels, they are not edited without such a note.
+- **Held by:** \`validate --analysed-at <rev> --notes notes.md\` compares the cells at the revision the
+  analysis read with the cells now, and refuses every cell that was relabelled, ADDED or REMOVED
+  since and is not named in a dated section with a quote. Added cells count too: the analysis did
+  not see them. Tag the revision when the analysis runs (\`git tag analysis-<date>\`).
+
+## What to report
+
+The codebook (and its revision), the share of runs labelled automatically vs by hand, the
+pre-sort's disagreements with the hand labels, the second rater's agreement (kappa, percent with CI,
+n), and every relabel after the analysis with its note.
+
+## Compose with
+
+- **run-agent-grid** (upstream) — the runs, the attempt ledger and the cell ids the labels use.
+- **build-label-page** — the page a person labels on, and the export into the study's \`data/\`.
+- **build-benchmark** — the statistics computed from the labels.
+`,
+});

package/skills/label-agent-runs/label-agent-runs.eval.ts

@@ -0,0 +1,55 @@
+/**
+ * label-agent-runs — the PAID tier: does this skill's description fire on its own requests, and
+ * stay quiet on a sibling's?
+ *
+ * Declared here, whole: the prompts it must fire on, the prompts it must not, and the
+ * measurement — no shared table, no kit. `vigiles eval` discovers this file by its name and runs
+ * it; nothing runs at import.
+ *
+ * The should-not-fire set is `run-agent-grid`'s own requests, imported from its eval file (the
+ * sibling it is most easily confused with), plus `build-label-page` (a page a person labels on)
+ * and `build-benchmark` (the statistics), written inline.
+ *
+ * The skills it competes with are every skill this package ships (the `skills/` directory this
+ * file sits in), the same set in a checkout and in `node_modules`.
+ *
+ * ⚠️ NEVER RUN. Written with the skill; its rates are UNKNOWN, not assumed good.
+ *
+ * Costs money; not CI.
+ *   npx vigiles eval skills/label-agent-runs/label-agent-runs.eval.ts
+ */
+import { dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+import { defineEval, skillResolved } from "vigiles";
+import { TRIGGER_PROMPTS as GRID } from "../run-agent-grid/run-agent-grid.eval.ts";
+
+/** The namespace vigiles installs a loose skills directory under. */
+const LOOSE_NS = "vigiles-loose-skills";
+
+export default defineEval({
+  measureTriggerRate: {
+    name: "label-agent-runs-trigger",
+    skillsDir: dirname(dirname(fileURLToPath(import.meta.url))),
+    prompts: [
+      "прогоны закончились, теперь надо разметить исходы каждого запуска агента",
+      "the auto-classifier missed outcomes where the agent handed the decision back without asking a question",
+      "нужен второй разметчик на другой модели и каппа Коэна по меткам прогонов",
+      "we already ran the analysis, can I still fix a few run labels and how do I record it",
+    ],
+    irrelevantPrompts: [
+      ...GRID.slice(0, 2),
+      "make me a page to label these, and put the answers back into the study folder",
+      "нужны парные прогоны, доверительные интервалы и поправка на множественные сравнения",
+    ],
+    fired: (trace) => skillResolved(trace, `${LOOSE_NS}:label-agent-runs`),
+    // Four named phrasings per side, deliberately, as for every skill of this package.
+    minPrompts: 4,
+    // Finished runs and a first label file, so "label the runs" has runs to refer to.
+    fixture: {
+      "runs/a1/done.json": '{"status":"complete"}\n',
+      "runs/a1/final-message.txt":
+        "I'll leave the final call on this to you.\n",
+      "labels.tsv": "agent\tmodel\tcell\tlabel\n",
+    },
+  },
+});