paperlint 10.4.0 → 11.0.0

package/scripts/refresh-deadlines/refresh.test.ts

@@ -0,0 +1,287 @@
+/**
+ * The refresh's logic: what one preset becomes, what a run prints, writes and returns — over a
+ * fake VenueDeadlines port that answers plain domain values (how HotCRP's page is parsed is
+ * src/adapters/hotcrp/deadlines.test.ts's) and presets held in memory.
+ */
+import { readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, test } from "vitest";
+import type { PortalDeadlines } from "#src/domain/deadline";
+import { err, ok, type Result } from "#src/domain/result";
+import type { PortalFailure } from "#src/domain/submission";
+import type { VenueDeadlines } from "#src/ports/venue-deadlines";
+import { parsePreset } from "#src/tex-requirements";
+import {
+  refreshPreset,
+  runRefresh,
+  withDeadlines,
+  type Outcome,
+  type PresetFiles,
+} from "./refresh.ts";
+
+const ROOT = dirname(dirname(dirname(fileURLToPath(import.meta.url))));
+const PRESETS = join(ROOT, "presets");
+const MSR = join(PRESETS, "msr.jsonc");
+const msrText = readFileSync(MSR, "utf8");
+const readings = (text: string) => parsePreset(text, MSR).deadlines ?? [];
+/** MSR with its submission deadline two days before the portal's. */
+const earlier = msrText.replace(
+  '"at": "2026-10-21T11:59:59Z"',
+  '"at": "2026-10-19T04:00:00Z"',
+);
+/** Presets held in memory, by name: what a run reads, and what it writes back. */
+function inMemory(files: Readonly<Record<string, string>>) {
+  const written: Record<string, string> = {};
+  const presets: PresetFiles = {
+    names: Object.keys(files).toSorted(),
+    read: (name) => files[name] ?? "",
+    write: (name, text) => {
+      written[name] = text;
+    },
+    path: (name) => join(PRESETS, name),
+  };
+  return { presets, written };
+}
+
+/** What MSR 2027's portal lists — the readings presets/msr.jsonc holds. */
+const MSR_PAGE: PortalDeadlines = {
+  now: 1792434476,
+  open: true,
+  blind: true,
+  deadlines: [
+    {
+      what: "submission",
+      label: "Submission deadline",
+      at: "2026-10-21T11:59:59Z",
+    },
+    {
+      what: "resubmission",
+      label: "Resubmission deadline",
+      at: "2026-10-24T11:59:59Z",
+    },
+  ],
+};
+/** What AIDC 2026's portal lists — the readings presets/aidc.jsonc holds. */
+const AIDC_PAGE: PortalDeadlines = {
+  now: 1791194489,
+  open: true,
+  blind: true,
+  deadlines: [
+    {
+      what: "submission",
+      label: "Submission deadline",
+      at: "2026-10-03T11:59:59Z",
+    },
+    { what: "camera-ready", label: "camera-ready", at: "2026-10-31T11:59:59Z" },
+  ],
+};
+/** A portal that answers `answer`. */
+const answering =
+  (answer: Result<PortalDeadlines, PortalFailure>) => (): VenueDeadlines => ({
+    read: () => Promise.resolve(answer),
+  });
+/** Each site's page, by the URL the preset names. */
+const bySite = (url: string): VenueDeadlines =>
+  answering(ok(url.includes("acsac") ? AIDC_PAGE : MSR_PAGE))();
+
+/** The outcome, known to be of `kind` — or the test fails naming what it was. */
+function as<K extends Outcome["kind"]>(
+  o: Outcome,
+  kind: K,
+): Extract<Outcome, { kind: K }> {
+  const isKind = (x: Outcome): x is Extract<Outcome, { kind: K }> =>
+    x.kind === kind;
+  if (!isKind(o)) throw new Error(`expected ${kind}, got ${JSON.stringify(o)}`);
+  return o;
+}
+
+describe("withDeadlines — the `deadlines` value replaced by jsonc-parser, everything else as written", () => {
+  test("writing back the readings the file holds gives the file's own bytes", async () => {
+    expect(await withDeadlines(msrText, MSR, readings(msrText))).toBe(msrText);
+  });
+});
+
+test("a preset with a portal and no `deadlines` yet: each reading added, the block right after `portal`", async () => {
+  const start = msrText.indexOf("  // Read from the portal");
+  const end = msrText.indexOf("  ],\n", start) + "  ],\n".length;
+  const o = as(
+    await refreshPreset({
+      file: MSR,
+      text: msrText.slice(0, start) + msrText.slice(end),
+      portal: answering(ok(MSR_PAGE)),
+      today: "2026-10-19",
+    }),
+    "changed",
+  );
+  expect(o.changes).toEqual([
+    "submission: added 2026-10-21T11:59:59Z",
+    "resubmission: added 2026-10-24T11:59:59Z",
+  ]);
+  expect(readings(o.text)).toEqual(readings(msrText));
+  expect(o.text).toMatch(
+    /"url": "https:\/\/msr2027\.hotcrp\.com",\n {2}\},\n {2}"deadlines": \[/,
+  );
+});
+
+describe("refreshPreset on MSR 2027's page", () => {
+  test("the portal says what the preset says: unchanged", async () => {
+    const o = await refreshPreset({
+      file: MSR,
+      text: msrText,
+      portal: answering(ok(MSR_PAGE)),
+      today: "2026-10-06",
+    });
+    expect(o).toEqual({
+      kind: "unchanged",
+      url: "https://msr2027.hotcrp.com/deadlines",
+      notes: [],
+    });
+  });
+
+  test("the portal moved the submission: only that reading changes, with today's read day", async () => {
+    const o = as(
+      await refreshPreset({
+        file: MSR,
+        text: earlier,
+        portal: answering(ok(MSR_PAGE)),
+        today: "2026-10-20",
+      }),
+      "changed",
+    );
+    expect(o.changes).toEqual([
+      "submission: 2026-10-19T04:00:00Z → 2026-10-21T11:59:59Z",
+    ]);
+    // Only the deadlines block differs: the comments and every other key are the file's own.
+    expect(o.text).toBe(
+      msrText.replace(
+        '"read": "2026-10-19",\n    },\n    {\n      "what": "resubmission"',
+        '"read": "2026-10-20",\n    },\n    {\n      "what": "resubmission"',
+      ),
+    );
+  });
+});
+
+describe("refreshPreset — what it skips, and a portal it cannot read", () => {
+  test("a preset without a portal, or with one paperlint has no adapter for, is skipped", async () => {
+    const aisec = join(PRESETS, "aisec.jsonc");
+    const skipped = { kind: "skipped" };
+    expect(
+      await refreshPreset({
+        file: aisec,
+        text: readFileSync(aisec, "utf8"),
+        portal: answering(ok(MSR_PAGE)),
+        today: "2026-10-05",
+      }),
+    ).toEqual(skipped);
+    expect(
+      await refreshPreset({
+        file: MSR,
+        text: msrText.replace('"kind": "hotcrp"', '"kind": "easychair"'),
+        portal: answering(ok(MSR_PAGE)),
+        today: "2026-10-05",
+      }),
+    ).toEqual(skipped);
+  });
+
+  test.each<[string, PortalFailure, string]>([
+    [
+      "a site that answers 404",
+      {
+        kind: "refused",
+        httpStatus: 404,
+        messages: [
+          { message: "No such conference", field: null, status: null },
+        ],
+      },
+      "the portal refused (HTTP 404): No such conference",
+    ],
+    [
+      "a site that does not answer",
+      { kind: "unreachable", detail: "connect ECONNREFUSED" },
+      "the portal cannot be reached: connect ECONNREFUSED",
+    ],
+    [
+      "a page that is not the one expected",
+      { kind: "malformed", httpStatus: 200, detail: "no init" },
+      "the portal answered HTTP 200, but not as its API does: no init",
+    ],
+  ])("%s is reported, never guessed", async (_what, failure, why) => {
+    expect(
+      await refreshPreset({
+        file: MSR,
+        text: msrText,
+        portal: answering(err(failure)),
+        today: "2026-10-05",
+      }),
+    ).toEqual({
+      kind: "unreadable",
+      url: "https://msr2027.hotcrp.com/deadlines",
+      why,
+    });
+  });
+});
+
+describe("runRefresh — what it prints, writes and returns", () => {
+  test("nothing changed: one line, no preset written, exit 0", async () => {
+    const { presets, written } = inMemory(
+      Object.fromEntries(
+        readdirSync(PRESETS)
+          .filter((f) => f.endsWith(".jsonc"))
+          .map((f) => [f, readFileSync(join(PRESETS, f), "utf8")]),
+      ),
+    );
+    const lines: string[] = [];
+    const code = await runRefresh({
+      presets,
+      portal: bySite,
+      today: "2026-10-06",
+      log: (l) => lines.push(l),
+      summary: null,
+    });
+    expect([code, written]).toStrictEqual([0, {}]);
+    expect(lines.at(-1)).toMatch(
+      /^✓ no deadline changed: 2 preset\(s\) with a portal read, \d+ skipped$/,
+    );
+  });
+
+  test("a preset that does not parse is a defect of the repository: exit 1, naming it", async () => {
+    const lines: string[] = [];
+    const summaries: string[] = [];
+    const code = await runRefresh({
+      presets: inMemory({ "bad.jsonc": "{ nope" }).presets,
+      portal: bySite,
+      today: "2026-10-06",
+      log: (l) => lines.push(l),
+      summary: (text) => summaries.push(text),
+    });
+    expect(code).toBe(1);
+    expect(lines[0]).toMatch(
+      /^::error title=deadlines::bad\.jsonc does not parse: /,
+    );
+    expect(summaries[0]).toMatch(/^- 🔴 `bad\.jsonc` does not parse: /m);
+  });
+});
+
+describe("runRefresh — a changed preset", () => {
+  test("a changed preset is written back, and the summary names it", async () => {
+    const { presets, written } = inMemory({ "msr.jsonc": earlier });
+    const lines: string[] = [];
+    const summaries: string[] = [];
+    const code = await runRefresh({
+      presets,
+      portal: bySite,
+      today: "2026-10-20",
+      log: (l) => lines.push(l),
+      summary: (text) => summaries.push(text),
+    });
+    expect(code).toBe(0);
+    expect(readings(written["msr.jsonc"] ?? "")[0]?.at).toBe(
+      "2026-10-21T11:59:59Z",
+    );
+    expect(lines.at(-1)).toBe("1 preset(s) changed");
+    expect(summaries[0]).toMatch(
+      /^- `msr\.jsonc` \(https:\/\/msr2027\.hotcrp\.com\/deadlines\)\n {2}- submission: 2026-10-19T04:00:00Z → 2026-10-21T11:59:59Z$/m,
+    );
+  });
+});

package/scripts/refresh-deadlines/refresh.ts

@@ -0,0 +1,183 @@
+/**
+ * THE SHIPPED PRESETS' DEADLINES, RE-READ FROM THEIR PORTALS — the refresh's logic, behind the
+ * VenueDeadlines port and the preset files it is handed (`main.ts` plugs in the real ones). For
+ * every preset whose `portal` paperlint has an adapter for, the portal's deadlines are compared
+ * with the preset's `deadlines` (`compare.ts`), and a preset that changed is rewritten — the
+ * `deadlines` value only, by jsonc-parser, then Prettier as `npm run fmt` would.
+ *
+ * IT REPORTS, IT DOES NOT BLOCK. A portal that is down, gone or not the page expected is a
+ * `::warning::` line and a line in the summary, and the run exits 0 — the venue's outage is not this
+ * repository's defect, and the next day's run reads it again. A preset that does not parse IS:
+ * `::error::`, exit 1.
+ */
+import { applyEdits, modify } from "jsonc-parser";
+import { format } from "prettier";
+import type { IsoDate, Reading } from "#src/domain/deadline";
+import { supportedPortal } from "#src/domain/submission";
+import { messageOf } from "#src/domain/text";
+import type { VenueDeadlines } from "#src/ports/venue-deadlines";
+import { failureText } from "#src/submission";
+import { parsePreset } from "#src/tex-requirements";
+import { refreshReadings } from "./compare.ts";
+
+/** What happened to one preset. */
+export type Outcome =
+  | { readonly kind: "skipped" }
+  | { readonly kind: "broken"; readonly why: string }
+  | { readonly kind: "unreadable"; readonly url: string; readonly why: string }
+  | {
+      readonly kind: "unchanged";
+      readonly url: string;
+      readonly notes: readonly string[];
+    }
+  | {
+      readonly kind: "changed";
+      readonly url: string;
+      readonly text: string;
+      readonly changes: readonly string[];
+      readonly notes: readonly string[];
+    };
+
+/** The presets of one directory, by file name. */
+export interface PresetFiles {
+  /** The `*.jsonc` presets' names, sorted. */
+  readonly names: readonly string[];
+  readonly read: (name: string) => string;
+  readonly write: (name: string, text: string) => void;
+  /** Where a preset lives, as a message names it and Prettier infers its format from. */
+  readonly path: (name: string) => string;
+}
+
+/** The preset's text with its `deadlines` value set to `readings` (inserted after `portal` when absent). */
+export async function withDeadlines(
+  text: string,
+  file: string,
+  readings: readonly Reading[],
+): Promise<string> {
+  const edits = modify(text, ["deadlines"], readings, {
+    formattingOptions: { insertSpaces: true, tabSize: 2 },
+    getInsertionIndex: (keys) => keys.indexOf("portal") + 1,
+  });
+  // Prettier, for what jsonc-parser does not print: the trailing commas `npm run fmt:check` requires.
+  return format(applyEdits(text, edits), { filepath: file });
+}
+
+/** One preset: its portal read, its readings compared, its new text when one changed. */
+export async function refreshPreset(o: {
+  readonly file: string;
+  readonly text: string;
+  readonly portal: (site: string) => VenueDeadlines;
+  readonly today: IsoDate;
+}): Promise<Outcome> {
+  const parsed = ((): ReturnType<typeof parsePreset> | string => {
+    try {
+      return parsePreset(o.text, o.file);
+    } catch (e) {
+      return messageOf(e);
+    }
+  })();
+  if (typeof parsed === "string") return { kind: "broken", why: parsed };
+  const portal = parsed.portal === null ? null : supportedPortal(parsed.portal);
+  if (portal === null || !portal.ok) return { kind: "skipped" };
+  const url = `${portal.value.url}/deadlines`;
+  const page = await o.portal(portal.value.url).read();
+  if (!page.ok)
+    return { kind: "unreadable", url, why: failureText(page.error) };
+  const r = refreshReadings(parsed.deadlines ?? [], page.value, url, o.today);
+  if (r.changes.length === 0) return { kind: "unchanged", url, notes: r.notes };
+  const text = await withDeadlines(o.text, o.file, r.readings);
+  return { kind: "changed", url, text, changes: r.changes, notes: r.notes };
+}
+
+/** The lines one preset's outcome contributes: to the log (annotations included) and to the summary. */
+function report(
+  name: string,
+  o: Outcome,
+): { readonly log: readonly string[]; readonly md: readonly string[] } {
+  switch (o.kind) {
+    case "skipped":
+      return { log: [], md: [] };
+    case "broken":
+      return {
+        log: [`::error title=deadlines::${name} does not parse: ${o.why}`],
+        md: [`- 🔴 \`${name}\` does not parse: ${o.why}`],
+      };
+    case "unreadable":
+      return {
+        log: [
+          `::warning title=deadlines::${name}: could not read ${o.url} — ${o.why}`,
+        ],
+        md: [`- ⚠️ \`${name}\`: could not read ${o.url} — ${o.why}`],
+      };
+    case "unchanged":
+      return { log: o.notes.map((n) => `${name}: ${n}`), md: [] };
+    case "changed": {
+      const lines = [...o.changes, ...o.notes];
+      return {
+        log: lines.map((c) => `${name}: ${c}`),
+        md: [`- \`${name}\` (${o.url})`, ...lines.map((c) => `  - ${c}`)],
+      };
+    }
+  }
+}
+
+/** The pull request's body. */
+const summaryOf = (today: IsoDate, lines: readonly string[]): string =>
+  [
+    `The venue presets' deadlines, re-read on ${today} from each preset's portal by \`.github/workflows/deadlines.yml\` (\`scripts/refresh-deadlines/\`). A reading whose value changed carries today's \`read\` day; the others keep theirs.`,
+    "",
+    ...lines,
+    "",
+    "Merging releases a patch; papers get the new dates through their normal update of paperlint.",
+    "",
+  ].join("\n");
+
+/** What a refresh run needs: the presets, how to read a portal, the day, where to print. */
+export interface RefreshDeps {
+  readonly presets: PresetFiles;
+  readonly portal: (site: string) => VenueDeadlines;
+  readonly today: IsoDate;
+  readonly log: (line: string) => void;
+  /** Where the pull request's body goes; null: nowhere. */
+  readonly summary: ((text: string) => void) | null;
+}
+
+/** Every preset, refreshed — a changed one written back. */
+const outcomesOf = (
+  d: RefreshDeps,
+): Promise<readonly { readonly name: string; readonly out: Outcome }[]> =>
+  Promise.all(
+    d.presets.names.map(async (name) => {
+      const out = await refreshPreset({
+        file: d.presets.path(name),
+        text: d.presets.read(name),
+        portal: d.portal,
+        today: d.today,
+      });
+      if (out.kind === "changed") d.presets.write(name, out.text);
+      return { name, out };
+    }),
+  );
+
+/** Every preset refreshed, reported to the log and the summary. 0 unless a preset does not parse. */
+export async function runRefresh(d: RefreshDeps): Promise<number> {
+  const outcomes = await outcomesOf(d);
+  const reports = outcomes.map(({ name, out }) => report(name, out));
+  const count = (k: Outcome["kind"]) =>
+    outcomes.filter((x) => x.out.kind === k).length;
+  [
+    ...reports.flatMap((r) => r.log),
+    count("changed") === 0
+      ? `✓ no deadline changed: ${String(count("unchanged"))} preset(s) with a portal read, ${String(count("skipped"))} skipped`
+      : `${String(count("changed"))} preset(s) changed`,
+  ].forEach((l) => {
+    d.log(l);
+  });
+  d.summary?.(
+    summaryOf(
+      d.today,
+      reports.flatMap((r) => r.md),
+    ),
+  );
+  return count("broken") > 0 ? 1 : 0;
+}

package/skills/grade-paper-writing/SKILL.md

@@ -4,7 +4,7 @@ description: Grade how a paper reads and fix it sentence by sentence. Use when a
 allowed-tools: [Read, Write, Edit, Grep, Glob, Agent, Skill]
 ---
 
-<!-- vigiles:sha256:1897016b69f3f9b8 compiled from skills/grade-paper-writing/SKILL.md.spec.ts -->
+<!-- vigiles:sha256:2b7193c739a374e6 compiled from skills/grade-paper-writing/SKILL.md.spec.ts -->
 
 # grade-paper-writing — grade how the paper READS, then fix it sentence by sentence
 
@@ -223,7 +223,9 @@ is what happened.
    ```
 
    It measures the prose a reader sees — the files the last build read spliced in, no preamble,
-   comments, floats or inline `.bib` — so its rates are the paper's, not the source file's. Run
+   comments, floats or inline `.bib` — so its rates are the paper's, not the source file's — and, on
+   its own line, the figure captions that float outside that prose, flagging a caption sentence over
+   40 words or a caption over 100. Run
    `npx paperlint build` first: with no current build record it reads `paper.tex` alone and says so. The skill is what must ACT on it.
    Its thresholds are sourced; a FLAG means the text sits above a published human or machine
    baseline, which is a defect with a number attached, not an opinion.

package/skills/grade-paper-writing/SKILL.md.spec.ts

@@ -229,7 +229,9 @@ is what happened.
    \`\`\`
 
    It measures the prose a reader sees — the files the last build read spliced in, no preamble,
-   comments, floats or inline \`.bib\` — so its rates are the paper's, not the source file's. Run
+   comments, floats or inline \`.bib\` — so its rates are the paper's, not the source file's — and, on
+   its own line, the figure captions that float outside that prose, flagging a caption sentence over
+   40 words or a caption over 100. Run
    \`npx paperlint build\` first: with no current build record it reads \`paper.tex\` alone and says so. The skill is what must ACT on it.
    Its thresholds are sourced; a FLAG means the text sits above a published human or machine
    baseline, which is a defect with a number attached, not an opinion.

package/skills/grade-paper-writing/__snapshots__/prose-lint.test.ts.snap

@@ -1,10 +1,19 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`--flags-only: a metric over its threshold (not a caption) is a finding too, and the report flags the same line 1`] = `
+"✍️  prose-lint — paper.tex — 1 finding(s):
+   sentences whose main claim is a negation: 75%
+   run prose-lint.mjs on the file without --flags-only for every metric
+"
+`;
+
 exports[`--flags-only: an over-long caption flags and exits 1 1`] = `
 "✍️  prose-lint — paper.tex — 2 finding(s):
-   figures/fig1.tex: caption sentence of 46 words (warn >40) — "Big. word0 word1 word2 word3 word4 word5 word6 word7 word8 word9 word10 word11 word12 word13 word14 word15 wor…"
-   figures/fig1.tex: caption is 106 words (warn >100) — a caption this long is a section under a picture
-   run prose-lint.mjs on the file without --flags-only for the sentences
+   all metadiscourse/1000w  363.6   (Hyland RA baseline 64.9, warn >90)
+   figure captions over the limit: 2
+      > figures/fig1.tex: caption sentence of 46 words (warn >40) — "Big. word0 word1 word2 word3 word4 word5 word6 word7 word8 word9 word10 word11 word12 word13 word14 word15 wor…"
+      > figures/fig1.tex: caption is 106 words (warn >100) — a caption this long is a section under a picture
+   run prose-lint.mjs on the file without --flags-only for every metric
 "
 `;
 
@@ -37,6 +46,7 @@ ok   all metadiscourse/1000w  0   (Hyland RA baseline 64.9, warn >90)
 ok   "rather than"/instead-of/10,000w  0   (warn >25; not epanorthosis, no published baseline)
 ok   sentences ending on a cross-ref/citation/hedge (wasted stress position): 0
 ok   sentences whose main claim is a negation: 0%
+ok   figure captions over the limit: 0
      topic positions (whose story is this?): "a rendered page." | "it was hyphenated across a" | "next page text here."
 
 (thresholds and their sources are in THRESHOLDS at the top of this file)
@@ -64,6 +74,9 @@ FLAG sentences ending on a cross-ref/citation/hedge (wasted stress position): 12
        > …], and that results from a handful of runs need honest statistics [1].
        > …g code [1], and models that memorize answers instead of reasoning [1].
 ok   sentences whose main claim is a negation: 37%
+FLAG figure captions over the limit: 2
+       > paper.tex: caption sentence of 52 words (warn >40) — "Two things make it different from how efficiency tools are usually measured: the tool is installed the way a u…"
+       > paper.tex: caption sentence of 44 words (warn >40) — "The advertised 65% is an output-token reduction; because output is only math of the dollar cost, even a perfec…"
      topic positions (whose story is this?): "a fast-growing ecosystem of plugins," | "the most popular single artifact---a" | "we show that such claims," | "efficiency tools are validated on" | "we introduce a reproducible, cost-aware," | "across 140 runs on two" | "output falls far short of" | "the ceiling is also structural:" | "we argue that trustworthy evaluation" | "ai coding agents and their" | "built on the reasoning-and-acting loops" | "assistants of this kind are" | "an unverified performance claim about" | "while the community has invested" | "we focus on the most" | "the flagship example is a" | "naming the commits matters more" | "two separate claims are bundled" | "the first is that the" | "that one has already been" | "the second claim is the" | "nobody had measured it." | "it fails for a reason" | "this paper is about the" | "rq: when a token-efficiency tool" | "the first half is measured," | "our contribution is not that" | "we identify why the headline" | "we build a cost-aware, correctness-gated" | "we measure two widely used" | "output falls far short of" | "the dollar bill does not" | "the broader point is about" | "a modern agent bills tokens" | "a multi-turn agentic session [1]" | "reading the upstream benchmark code" | "the reported number measures something" | "the fix is to measure" | "the compression literature these tools" | "the harness --- the ecosystem" | "we implement the evaluation as" | "figure 1 shows the whole" | "each tool is delivered as" | "a skill that ships as" | "this step is load-bearing: a" | "we use seven self-contained coding" | "the suite deliberately skews short" | "each task is run with" | "the runs date from july" | "the two tools are the" | "for every run we record" | "on a subscription the \`\`bill''" | "we report dollars---the weighting most" | "each task carries a deterministic" | "these are lightweight structural checks" | "a trial where the skill" | "for each task we report" | "we call an effect significant" | "we also note which effects" | "for the bill we report" | "averages taken over a few" | "this protocol is sized to" | "two things are still required" | "the flagship skill (claim 65%)" | "we sign bill changes so" | "for the flagship, the bill" | "either way the answer is" | "the second tool (claim 63%)" | "it is also not significant." | "its one per-task cell that" | "the intervals are wide: at" | "but they exclude anything near" | "whatever room remains is capped" | "correctness." | "both arms passed the correctness" | "a sharper signal hides in" | "this is a short-session effect" | "this is not a quirk" | "both shares below are measured," | "pooled across all tasks and" | "if output is a fraction" | "even a perfect 65% output" | "the advertised 63--75% and the" | "the flagship's other feature trims" | "output is a small fraction" | "we measure cost: latency and" | "we do not claim \`\`these" | "nor do we dispute the" | "our point is that they" | "we label each by its" | "we run five trials per" | "that is itself part of" | "across tasks the flagship's output" | "the honest quantity is therefore" | "our design refutes the advertised" | "what caps those is the" | "a formal equivalence test (tost---a" | "the harness is not toothless," | "at that scale we read" | "we test one model, and" | "anthropic's tiers all share a" | "the ceiling scales linearly with" | "even if output were priced" | "our tasks top out around" | "this cuts in our favor." | "a longer session re-reads a" | "that pushes the ceiling lower," | "one might object that terser" | "but that downstream saving is" | "so it adds a few" | "the compounded ceiling is --," | "the overhead runs the other" | "an always-loaded tool prompt adds" | "that fixed cost amortizes over" | "the durable long-session prediction is" | "an empirical long-session demonstration is" | "every run in the main" | "the harness decides how many" | "a wordier model would raise" | "the ceiling of section 1" | "a pilot on claude haiku" | "nearly double, and still about" | "both runs ship in the" | "a second harness is reachable" | "the flagship now ships wrappers" | "measuring a second harness therefore" | "wrapper list and the rejected" | "we do not study multi-agent" | "we test two tools, and" | "the harness can a/b any" | "the bias runs in our" | "our per-run bill uses the" | "a user can also invalidate" | "(the first two rewrite the" | "that is, they push cost" | "they lower the ceiling and" | "so our clean-caching tasks, if" | "the correctness gate is a" | "so \`\`zero regressions'' bounds gross" | "a stronger gate that runs" | "still, a gate that never" | "without it, a harness would" | "the efficiency-claim failure is a" | "a community norm of cost-aware," | "execution-based, correctness-gated benchmarks such as" | "we keep their correctness-first discipline" | "several lines of work push" | "we take the same stance---price" | "closest to us, [1] argue" | "we turn that lens on" | "a long line of work" | "capability benchmarks show the same" | "we aim the same critique" | "the industry a/b test cited" | "what we add over such" | "the efficiency-tool market runs on" | "these tools trim what the" | "we measured two of the" | "the numbers are not fabricated;" | "the output you are shown" | "trimming the model's replies to" | "we release the measurements with" | "before you install the next" | "the complete harness of section" | "the measurements are released per" | "individual trial rows are not"
 
 (thresholds and their sources are in THRESHOLDS at the top of this file)