rungate 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/lib/affected-tests.ts +398 -0
  2. package/lib/agent-audit.ts +2 -3
  3. package/lib/agent-worktree.ts +287 -0
  4. package/lib/agents-md-sections.ts +81 -16
  5. package/lib/behavioral-cache.ts +150 -2
  6. package/lib/branch-protection.ts +790 -0
  7. package/lib/brief-ac-alignment.ts +104 -0
  8. package/lib/brief-references.ts +205 -0
  9. package/lib/bun-version.ts +93 -14
  10. package/lib/check-remedy.ts +140 -0
  11. package/lib/ci-checkout.ts +17 -0
  12. package/lib/ci-history-depth.ts +159 -0
  13. package/lib/ci-setup-step.ts +96 -0
  14. package/lib/claude-code-action.ts +85 -0
  15. package/lib/claude-tool-vocabulary.ts +188 -0
  16. package/lib/cli-commands.ts +15 -0
  17. package/lib/commit-sha.ts +42 -0
  18. package/lib/commit-subject.ts +90 -0
  19. package/lib/compliance-report.ts +377 -45
  20. package/lib/compliance.ts +191 -17
  21. package/lib/config-loader.ts +39 -1
  22. package/lib/config-trackable.ts +391 -0
  23. package/lib/conformity.ts +1087 -100
  24. package/lib/containment.ts +238 -0
  25. package/lib/content-hash.ts +24 -0
  26. package/lib/create-sc.ts +249 -16
  27. package/lib/dev-liveness.ts +180 -0
  28. package/lib/discovery-branch-points.ts +432 -0
  29. package/lib/discovery-stability.ts +481 -0
  30. package/lib/distribution-currency.ts +186 -0
  31. package/lib/doc-routing.ts +157 -0
  32. package/lib/evidence-prevalidator.ts +224 -2
  33. package/lib/evidence-satisfiability.ts +534 -12
  34. package/lib/failure-intake.ts +317 -0
  35. package/lib/fallow-availability.ts +157 -0
  36. package/lib/fs-write-capabilities.ts +232 -0
  37. package/lib/gate-run-split.ts +147 -0
  38. package/lib/generators/agents-md.ts +140 -32
  39. package/lib/git-isolation.ts +140 -0
  40. package/lib/github.ts +15 -0
  41. package/lib/google-auth-action.ts +93 -0
  42. package/lib/harness-project.ts +48 -0
  43. package/lib/harness-provenance.ts +255 -0
  44. package/lib/hook-evidence.ts +235 -0
  45. package/lib/hook-scopes.ts +386 -0
  46. package/lib/hook-target-tree.ts +398 -0
  47. package/lib/hook-trees.ts +447 -0
  48. package/lib/js-top-level-code.ts +78 -10
  49. package/lib/junit-checks.ts +52 -0
  50. package/lib/launch-base.ts +253 -0
  51. package/lib/line-citation.ts +372 -0
  52. package/lib/orphan-reaper.ts +468 -0
  53. package/lib/parity-binding.ts +205 -0
  54. package/lib/per-test-timeouts.ts +219 -0
  55. package/lib/plan-coverage.ts +151 -0
  56. package/lib/prior-branch.ts +201 -8
  57. package/lib/process-table.ts +146 -0
  58. package/lib/project-state-derivation.ts +219 -0
  59. package/lib/project-state-root.ts +258 -0
  60. package/lib/published-artifact.ts +525 -0
  61. package/lib/readdir-sorted.ts +45 -0
  62. package/lib/review-execution-reading.ts +206 -0
  63. package/lib/review-tool-agreement.ts +433 -0
  64. package/lib/rule-promotion.ts +234 -0
  65. package/lib/rule-retirement.ts +525 -0
  66. package/lib/run-branch.ts +612 -0
  67. package/lib/rungate-schema.ts +19 -0
  68. package/lib/runner-home.ts +274 -0
  69. package/lib/sc-id-cross-tree.ts +19 -7
  70. package/lib/sc-ids.ts +121 -13
  71. package/lib/sc-reservations.ts +289 -0
  72. package/lib/scaffold/ai-review-findings.ts +403 -0
  73. package/lib/scaffold/ai-review-step.ts +4661 -64
  74. package/lib/scaffold/managed-workflow.ts +587 -192
  75. package/lib/scaffold/owned-files.ts +152 -45
  76. package/lib/scaffold/pre-push-hook.ts +49 -5
  77. package/lib/scaffold/steps.ts +1413 -244
  78. package/lib/scaffold/workflow-census.ts +217 -0
  79. package/lib/scaffold/workflow-drift.ts +217 -0
  80. package/lib/scaffold/write-steps.ts +4 -0
  81. package/lib/scan-budget.ts +247 -0
  82. package/lib/scanner.ts +43 -51
  83. package/lib/scratch-roots.ts +50 -5
  84. package/lib/screenshot-discovery.ts +121 -0
  85. package/lib/screenshot-upload.ts +41 -1
  86. package/lib/secret-patterns.ts +511 -8
  87. package/lib/ship-mode.ts +219 -0
  88. package/lib/source-slice.ts +294 -0
  89. package/lib/stale-cleanup.ts +37 -3
  90. package/lib/suite-flake.ts +278 -0
  91. package/lib/suite-measurement.ts +23 -4
  92. package/lib/suite-run-reading.ts +241 -1
  93. package/lib/suite-timeout-margin.ts +193 -0
  94. package/lib/test-case-bodies.ts +883 -0
  95. package/lib/test-case-deletion.ts +653 -0
  96. package/lib/test-census.ts +97 -29
  97. package/lib/test-command-positions.ts +274 -0
  98. package/lib/test-suite-lock.ts +159 -16
  99. package/lib/tool-probe.ts +31 -9
  100. package/lib/top-level-cd.ts +393 -0
  101. package/lib/tree-entries.ts +89 -0
  102. package/lib/ts-ast.ts +200 -0
  103. package/lib/walk-test-files.ts +97 -0
  104. package/lib/workflow-actions.ts +263 -0
  105. package/lib/worktree-collect.ts +177 -5
  106. package/lib/worktree-repo.ts +36 -1
  107. package/package.json +2 -1
  108. package/scripts/ai-review-findings.ts +45 -0
  109. package/scripts/create-sc.ts +21 -1
  110. package/scripts/extract-constraints.ts +107 -7
  111. package/scripts/generate-code-map.ts +23 -30
  112. package/scripts/scaffold-project.ts +8 -1
  113. package/scripts/sync-sc-status.ts +77 -16
@@ -0,0 +1,278 @@
1
+ /**
2
+ * #323 SC-5 — whether a series of suite runs says anything.
3
+ *
4
+ * `lib/suite-run-reading.ts` reads ONE run. This reads a SERIES, and the two
5
+ * questions are genuinely different: a run is clean or it is not, and a
6
+ * series is consistent, flaky, failing, or not a series at all.
7
+ *
8
+ * One binding, because the interesting part is not any single run — it is
9
+ * the relationship between them, and a caller composing its own verdict
10
+ * would get that relationship right on the series it had seen and wrong on
11
+ * the one that matters. `scripts/suite-flake-run.ts` executes the runs and
12
+ * decides nothing.
13
+ */
14
+ import { describeSuiteFailures, type SuiteRun } from "./suite-run-reading";
15
+
16
+ /**
17
+ * What tree a run was taken against.
18
+ *
19
+ * `head` alone is not enough: the suites #323 suspects write scratch
20
+ * directories under the repository, and a run that dirties the tree it is
21
+ * measuring has changed the input to the next run without moving HEAD. The
22
+ * dirty list is `git status --porcelain` paths.
23
+ */
24
+ export interface TreeSignature {
25
+ head: string;
26
+ dirty: string[];
27
+ }
28
+
29
+ /**
30
+ * How the process ended, which no reading of its TEXT can establish.
31
+ *
32
+ * `readSuiteRun` deliberately takes no exit code: #469's run exited 0 having
33
+ * measured nothing and run `39405680063` exited `null` having measured
34
+ * nothing, and nothing in either output says which way to resolve that. A
35
+ * SIGNAL is not ambiguous — but only the process's caller ever sees one, so
36
+ * it is carried here and read by the series.
37
+ */
38
+ export interface RunExit {
39
+ status: number | null;
40
+ signal: string | null;
41
+ /** The spawn's own error, e.g. `ENOBUFS`, which names the cause. */
42
+ error: string | null;
43
+ }
44
+
45
+ export interface FlakeRun {
46
+ /** 1-based, so a reason can name the run an operator will look for. */
47
+ index: number;
48
+ signature: TreeSignature;
49
+ run: SuiteRun;
50
+ /** Where the full output was kept. A failure that is not kept is #323. */
51
+ log: string;
52
+ seconds: number;
53
+ exit: RunExit;
54
+ }
55
+
56
+ /**
57
+ * Why a run's ENDING disqualifies it, or null.
58
+ *
59
+ * `bun test` exits 0 for a clean suite and 1 for a failing one. **1 must not
60
+ * be refused**: refusing every non-zero exit would turn every FLAKY and
61
+ * FAILING series into UNMEASURED and leave the instrument unable to report
62
+ * its own subject.
63
+ */
64
+ export function exitRefusal(exit: RunExit): string | null {
65
+ if (exit.error !== null && exit.error !== "") {
66
+ return `the spawn failed: ${exit.error}`;
67
+ }
68
+ if (exit.signal !== null) {
69
+ return (
70
+ `it was killed by ${exit.signal}, so its log ends wherever the process did — and a truncated log ` +
71
+ "still parses as a clean reading when the output it captured contains a nested suite's summary, " +
72
+ "which is how run 39405680063 reported CONSISTENT over ten killed runs (#323)"
73
+ );
74
+ }
75
+ if (exit.status === null) {
76
+ return "the spawn reported no exit status and no signal, so nothing says how it ended";
77
+ }
78
+ if (exit.status !== 0 && exit.status !== 1) {
79
+ return `it exited ${exit.status}, which is not a verdict about the tests — bun uses 0 for a clean suite and 1 for a failing one`;
80
+ }
81
+ return null;
82
+ }
83
+
84
+ /**
85
+ * CONSISTENT, FLAKY and FAILING are three facts, not two.
86
+ *
87
+ * A suite that fails every run is red — an ordinary thing to be, and
88
+ * somebody's build to fix. A suite that fails SOME runs is the defect #323
89
+ * is about. One word for both would leave the series unable to report its
90
+ * own subject.
91
+ */
92
+ export type FlakeVerdict = "CONSISTENT" | "FLAKY" | "FAILING" | "UNMEASURED";
93
+
94
+ /**
95
+ * Whether the runs agreed about how much work they did.
96
+ *
97
+ * Reported BESIDE the verdict and never folded into it. The 2026-10-11
98
+ * reading's only surviving lead is that the failing run executed 384 more
99
+ * `expect()` calls than either clean one — so a varying count is a finding.
100
+ * It is not, on its own, evidence that anything is wrong: a case enumerating
101
+ * a directory legitimately varies. Folding it into the verdict would make
102
+ * one of those two readings unavailable.
103
+ *
104
+ * `UNREPORTED` is first-class. A runner that printed no count and a runner
105
+ * whose count held steady are opposite facts, and `STABLE` is the value that
106
+ * would let a reader treat the first as the second.
107
+ */
108
+ export type AssertionStability = "STABLE" | "VARIED" | "UNREPORTED";
109
+
110
+ /**
111
+ * What `treeSignatureKey` can and cannot see, carried in the reading itself.
112
+ *
113
+ * A `CONSISTENT` verdict says the runs agreed over one tree, and a reader
114
+ * naturally hears "one set of inputs". It is not: the signature is HEAD plus
115
+ * `git status --porcelain`, so a gitignored path, anything under `/tmp` or
116
+ * `~/.rungate`, and `node_modules` can all change between runs with the
117
+ * signature unmoved.
118
+ *
119
+ * #799 is the measured instance rather than a worry. Series `39412289321`
120
+ * reported ten runs over one tree and was right to — while `test/phase-0.test.ts`
121
+ * collected its test list from a `/tmp` fixture the PREVIOUS run had left, so
122
+ * run 1 executed 15,194 tests and runs 2 through 10 executed 15,250.
123
+ *
124
+ * In the reading and not only in the printed sentence, because `flake.json`
125
+ * is what anybody reads months later, and a caveat that exists only in a
126
+ * console line is a caveat the artefact does not carry.
127
+ */
128
+ export const SIGNATURE_SCOPE =
129
+ "the tree signature is HEAD plus `git status --porcelain`, so it covers the working tree and nothing " +
130
+ "else — a gitignored path, anything under /tmp or ~/.rungate, and node_modules can all differ between " +
131
+ "runs while this series reports one tree (#799)";
132
+
133
+ export interface FlakeReading {
134
+ verdict: FlakeVerdict;
135
+ /** Why it is not CONSISTENT. Empty exactly when it is. */
136
+ reasons: string[];
137
+ assertions: AssertionStability;
138
+ /** One entry per run, in order, `null` where the runner did not say. */
139
+ assertionCounts: Array<number | null>;
140
+ runs: number;
141
+ /** What the tree comparison covered. Always present; see `SIGNATURE_SCOPE`. */
142
+ signatureScope: string;
143
+ }
144
+
145
+ /**
146
+ * One string per tree, so two readings can be compared without anybody
147
+ * deciding field by field what counts as the same tree.
148
+ *
149
+ * The dirty list is sorted: `git status --porcelain` order is not a property
150
+ * of the tree, and two runs differing only in it are one tree.
151
+ */
152
+ export function treeSignatureKey(signature: TreeSignature): string {
153
+ return `${signature.head}\n${[...signature.dirty].sort().join("\n")}`;
154
+ }
155
+
156
+ /** The runs that could be read at all. */
157
+ function measured(runs: FlakeRun[]) {
158
+ return runs.flatMap(r => (r.run.status === "MEASURED" ? [{ entry: r, run: r.run }] : []));
159
+ }
160
+
161
+ /**
162
+ * Read a series.
163
+ *
164
+ * FAILS CLOSED, like the single-run reader it sits on. Every path that
165
+ * cannot positively establish that `minimum` readable runs were taken over
166
+ * one tree returns UNMEASURED with reasons. There is no path that returns
167
+ * CONSISTENT from a series that does not say so.
168
+ */
169
+ export function flakeReading(runs: FlakeRun[], minimum: number): FlakeReading {
170
+ const assertionCounts = runs.map(r => (r.run.status === "MEASURED" ? r.run.expectCalls : null));
171
+ const reasons: string[] = [];
172
+
173
+ // A short series first, because it explains every other absence. A job
174
+ // that died after three runs has measured three runs, and ten is what was
175
+ // asked for — the instrument failing is not the subject passing (#60).
176
+ if (runs.length < minimum) {
177
+ reasons.push(
178
+ `only ${runs.length} of ${minimum} run(s) completed, so the series was cut short — ` +
179
+ "a short series is not a clean series, however green the runs in it were (#323 SC-5)",
180
+ );
181
+ }
182
+
183
+ // Every run that did not FINISH, before anything is read off its text.
184
+ // This is the guard run 39405680063 did not have: ten children killed by
185
+ // SIGTERM on a 1 MiB `maxBuffer` overflow, ten truncated logs whose only
186
+ // surviving summary belonged to a nested suite, and a CONSISTENT verdict
187
+ // over a suite of 5,500 tests reporting one pass.
188
+ for (const r of runs) {
189
+ const why = exitRefusal(r.exit);
190
+ if (why !== null) reasons.push(`run ${r.index} did not finish (log ${r.log}): ${why}`);
191
+ }
192
+
193
+ // Every unreadable run, by index. An UNMEASURED run is not a clean run:
194
+ // `readSuiteRun` returns it for a log that printed no summary, and this
195
+ // repository's logs carry nested suites' summaries, so such a log still
196
+ // contains the text a `grep "0 fail"` would match.
197
+ for (const r of runs) {
198
+ if (r.run.status !== "MEASURED") {
199
+ reasons.push(`run ${r.index} could not be read (log ${r.log}): ${r.run.reason}`);
200
+ }
201
+ }
202
+
203
+ // One tree, or it is not a series. Named by what differs, because
204
+ // "the tree moved" tells nobody where to look and a scratch directory
205
+ // appearing under the repository is SC-6's whole hypothesis.
206
+ const trees = new Map<string, FlakeRun[]>();
207
+ for (const r of runs) {
208
+ const key = treeSignatureKey(r.signature);
209
+ trees.set(key, [...(trees.get(key) ?? []), r]);
210
+ }
211
+ if (trees.size > 1) {
212
+ const described = [...trees.entries()].map(([key, group]) => {
213
+ const where = group.map(g => g.index).join(", ");
214
+ const [head, ...dirty] = key.split("\n");
215
+ const paths = dirty.filter(d => d !== "");
216
+ return `run(s) ${where} at ${head}${paths.length === 0 ? " with a clean tree" : ` with ${paths.join(", ")}`}`;
217
+ });
218
+ reasons.push(
219
+ `the tree changed during the series, so these runs did not measure one thing: ${described.join("; ")} ` +
220
+ "— ten green runs over three trees is not ten green runs (#323 SC-5)",
221
+ );
222
+ }
223
+
224
+ const counts = assertionCounts.filter((c): c is number => c !== null);
225
+ const assertions: AssertionStability =
226
+ counts.length !== runs.length || runs.length === 0
227
+ ? "UNREPORTED"
228
+ : new Set(counts).size === 1
229
+ ? "STABLE"
230
+ : "VARIED";
231
+
232
+ if (reasons.length > 0) {
233
+ return { verdict: "UNMEASURED", reasons, assertions, assertionCounts, runs: runs.length, signatureScope: SIGNATURE_SCOPE };
234
+ }
235
+
236
+ const readable = measured(runs);
237
+ const failing = readable.filter(r => r.run.fail > 0);
238
+ if (failing.length > 0) {
239
+ for (const f of failing) {
240
+ reasons.push(`run ${f.entry.index} (log ${f.entry.log}): ${describeSuiteFailures(f.run)}`);
241
+ }
242
+ }
243
+
244
+ const verdict: FlakeVerdict =
245
+ failing.length === 0 ? "CONSISTENT" : failing.length === readable.length ? "FAILING" : "FLAKY";
246
+
247
+ return { verdict, reasons, assertions, assertionCounts, runs: runs.length, signatureScope: SIGNATURE_SCOPE };
248
+ }
249
+
250
+ /** The sentence a caller prints instead of the verdict on its own. */
251
+ export function describeFlakeReading(reading: FlakeReading): string {
252
+ const { verdict, runs, assertions, assertionCounts, reasons } = reading;
253
+ const counts = assertionCounts.filter((c): c is number => c !== null);
254
+ const spread =
255
+ assertions === "VARIED"
256
+ ? ` The runs did not agree about how much work they did: ${Math.min(...counts)} to ` +
257
+ `${Math.max(...counts)} expect() calls, a spread of ${Math.max(...counts) - Math.min(...counts)}. ` +
258
+ "That is the only surviving lead from the 2026-10-11 reading (#323 SC-4)."
259
+ : assertions === "UNREPORTED"
260
+ ? " The runner did not report an assertion count for every run, so nothing is claimed about one."
261
+ : ` Every run executed ${counts[0]} expect() calls.`;
262
+
263
+ const failed = reasons.length;
264
+ const head =
265
+ verdict === "CONSISTENT"
266
+ ? `CONSISTENT: ${runs} consecutive run(s) on one tree, none failing.`
267
+ : verdict === "FLAKY"
268
+ ? `FLAKY: ${failed} of ${runs} consecutive run(s) on one tree failed — the same tree, different answers.`
269
+ : verdict === "FAILING"
270
+ ? `FAILING: all ${runs} run(s) failed. The suite is red; this is not flake.`
271
+ : `UNMEASURED: the series does not support a verdict.`;
272
+
273
+ // The scope goes on EVERY verdict, not only CONSISTENT. A reader who sees
274
+ // it once beside a clean series and never again would reasonably conclude
275
+ // the other verdicts had looked harder, and they did not — it is the same
276
+ // comparison in all four cases.
277
+ return [head, ...reasons.map(r => ` - ${r}`), spread.trim(), `Scope: ${reading.signatureScope}`].join("\n");
278
+ }
@@ -28,15 +28,21 @@ const SHA = /^[0-9a-f]{7,40}$/i;
28
28
 
29
29
  /**
30
30
  * UNMEASURED is not a result. It is the statement that this run has no suite
31
- * evidence, and it is deliberately not writable — see gates/schema.ts, which
32
- * accepts PASS and FAIL only. A recorded UNMEASURED would be a measurement
33
- * saying there was no measurement.
31
+ * evidence, and it is deliberately not writable — see gates/schema.ts, whose
32
+ * writable verdicts are PASS, FAIL and PARTIAL. A recorded UNMEASURED would
33
+ * be a measurement saying there was no measurement.
34
+ *
35
+ * PARTIAL is writable and is not a verdict THIS type has, which is the whole
36
+ * of #673's first half: a reading taken over less than the suite is a reading
37
+ * that was taken, so the recorder may write it — and it is not a statement
38
+ * about the suite, so every consumer must see UNMEASURED. The conversion
39
+ * happens in `readSuiteMeasurement` below and nowhere else.
34
40
  */
35
41
  export type SuiteVerdict = "PASS" | "FAIL" | "UNMEASURED";
36
42
 
37
43
  /** What a recorder persists at `suiteMeasurement` in workflow-state.json. */
38
44
  export interface SuiteMeasurement {
39
- verdict: "PASS" | "FAIL";
45
+ verdict: "PASS" | "FAIL" | "PARTIAL";
40
46
  /** The commit the suite actually ran against. */
41
47
  measuredSha: string;
42
48
  /** Failing tests. 0 for a PASS, at least 1 for a FAIL. */
@@ -136,6 +142,19 @@ export function readSuiteMeasurement(state: unknown): SuiteReading {
136
142
  }
137
143
 
138
144
  const verdict = raw.verdict;
145
+ // #673 — a reading the caller declared PARTIAL. Its own branch rather than
146
+ // falling through to the clause below, because "nobody recognises this
147
+ // word" and "this run measured some tests and said so" are different
148
+ // things to go and do something about: the first is a corrupt record and
149
+ // the second is a run that has not measured the suite YET.
150
+ if (verdict === "PARTIAL") {
151
+ return unmeasured(
152
+ `the suite measurement for ${measuredSha} was taken over PART of the suite: the caller ` +
153
+ `declared the run it recorded was less than the whole of it, so what it found is not a ` +
154
+ `statement about the suite (#673). ${describe(raw.tests)} test(s) collected is not a ` +
155
+ `reading a gate can act on, whatever the count`,
156
+ );
157
+ }
139
158
  if (verdict !== "PASS" && verdict !== "FAIL") {
140
159
  return unmeasured(
141
160
  `the suite measurement for ${measuredSha} carries no usable verdict (got ${describe(verdict)}) — ` +
@@ -53,6 +53,85 @@ const FAIL_LINE = /^\s*(\d+)\s+fail\s*$/gm;
53
53
  /** `Ran 11521 tests across 324 files.` — and `1 test across 1 file.` */
54
54
  const RAN_LINE = /^Ran\s+(\d+)\s+tests?\s+across\s+(\d+)\s+files?\./gm;
55
55
 
56
+ /**
57
+ * `13 skip` and `818 todo` — bun prints these only when they are non-zero.
58
+ *
59
+ * Read so that `pass + fail` and `tests` stop being two unexplained numbers
60
+ * for one event (#673). On run `wf_62d68d55-81c` the changelog said 13084
61
+ * and the structured field said 13915; the difference was exactly this
62
+ * run's `13 skip + 818 todo`, and nothing in the record said so. A reader
63
+ * comparing either against a previous run's number may be comparing
64
+ * different quantities.
65
+ *
66
+ * **Searched in the SUMMARY BLOCK, not the document.** `pass` and `fail` get
67
+ * away with `lastMatch` over the whole output because the outer run's
68
+ * summary is the last thing printed. These two cannot: bun OMITS them when
69
+ * the count is zero, so a last-match over the document walks BACKWARD past
70
+ * the summary into whatever the suite captured. That is not hypothetical —
71
+ * `~/.rungate/pai-harness-669/suite-output.txt` carries a nested
72
+ * `1 pass / 0 fail` at line 2600 of 2688, from a test that drives bun
73
+ * itself. A nested run's skips would have been recorded as the outer run's.
74
+ * `.claude/rules/checks-must-be-able-to-fail.md`: search the slice.
75
+ *
76
+ * **Absent means zero only when the producer's own total says so.** bun
77
+ * omits the line at zero, so for bun absent and zero are the same statement
78
+ * — but that is a fact about one runner, and a reading that asserts `skip:
79
+ * 0` over a runner with no concept of skipping is a number nobody measured.
80
+ * `pass + fail + skip + todo === tests` is the producer's own arithmetic,
81
+ * and it is what resolves the absence: when it closes, the missing counts
82
+ * really are zero; when it does not, they are `null` — not reported. Both
83
+ * findings were raised by the security review of this change.
84
+ */
85
+ const SKIP_LINE = /^\s*(\d+)\s+skip\s*$/gm;
86
+ const TODO_LINE = /^\s*(\d+)\s+todo\s*$/gm;
87
+
88
+ /**
89
+ * `1 error` — unhandled errors between tests (#323).
90
+ *
91
+ * In the summary block beside `skip` and `todo`, and omitted at zero like
92
+ * they are. It is read for one reason: **it is how a failure loses its
93
+ * name.** A module-scope throw counts toward `fail` and prints no `(fail)`
94
+ * line, because the test that would have carried the name never ran.
95
+ */
96
+ const ERROR_LINE = /^\s*(\d+)\s+errors?\s*$/gm;
97
+
98
+ /**
99
+ * `13648 expect() calls` — how much work the run actually did (#323 SC-4).
100
+ *
101
+ * Read because it is the only surviving lead from the 2026-10-11 reading
102
+ * this issue is about: three consecutive runs on one tree, one of which
103
+ * reported a failure, and the only difference between the failing run and
104
+ * either clean one is that it executed **384 more assertions** against an
105
+ * almost identical test count. That is not a test flipping its verdict; it
106
+ * is a test executing a substantially different amount of work.
107
+ *
108
+ * Nobody could say so at the time, because the number was in a terminal
109
+ * scrollback and nowhere else. As a field it is a reading, and
110
+ * `lib/suite-flake.ts` compares it across a series.
111
+ *
112
+ * In the summary block beside `skip`, `todo` and `error`, and for the same
113
+ * reason: this repository's suite drives `bun test` in subprocesses, so a
114
+ * last-match over the document walks backward out of the summary into a
115
+ * nested run's count.
116
+ */
117
+ const EXPECT_LINE = /^\s*(\d+)\s+expect\(\)\s+calls\s*$/gm;
118
+
119
+ /**
120
+ * `(fail) suite > case [0.12ms]` — one line per failing test.
121
+ *
122
+ * Anchored to the whole line, and the timing suffix is optional because bun
123
+ * omits it for a case that never started. The name is everything between.
124
+ *
125
+ * This cannot be cut to the summary block the way `skip` and `todo` are:
126
+ * bun prints these as the run goes, throughout the output. That is a real
127
+ * limitation and it is reported rather than papered over — see
128
+ * `describeSuiteFailures`, which refuses to list names when there are more
129
+ * of them than the summary counts, because this repository's own suite
130
+ * captures nested `bun test` output and a nested run's `(fail)` lines sit at
131
+ * column zero like everybody else's.
132
+ */
133
+ const FAIL_CASE_LINE = /^\s*\(fail\)\s+(\S.*?)(?:\s+\[[\d.]+\s*m?s\])?\s*$/gm;
134
+
56
135
  export interface MeasuredRun {
57
136
  status: "MEASURED";
58
137
  pass: number;
@@ -65,6 +144,20 @@ export interface MeasuredRun {
65
144
  * whole module is that a number nobody measured must not be reportable.
66
145
  */
67
146
  tests: number | null;
147
+ /**
148
+ * Tests the runner declined to execute, by kind (#673).
149
+ *
150
+ * These exist so that "tests that RAN" and "tests that EXIST" are both
151
+ * nameable without anybody subtracting to find out which is which.
152
+ *
153
+ * **null, not 0, when the runner did not say.** A missing line resolves to
154
+ * zero only when the runner's own `Ran N tests` total closes without it —
155
+ * see the patterns above. "This runner reports no skip count" and "this
156
+ * runner skipped none" are opposite facts, and `0` is the one value that
157
+ * would let a reader add them together.
158
+ */
159
+ skip: number | null;
160
+ todo: number | null;
68
161
  /**
69
162
  * Test files the runner discovered, or null when it did not say.
70
163
  *
@@ -74,6 +167,48 @@ export interface MeasuredRun {
74
167
  * *Silent Default Substitution*, pointed at the number #490 is about.
75
168
  */
76
169
  files: number | null;
170
+ /**
171
+ * The names of failing tests the runner printed, in the order it printed
172
+ * them (#323).
173
+ *
174
+ * Beside `fail` and not instead of it, because the two can disagree and
175
+ * the disagreement is the finding. Measured on bun 1.3.14: two test files,
176
+ * one ordinary failing assertion and one module-scope `throw`, report
177
+ * `2 fail` and **one** `(fail)` line — the throw's test never ran, so
178
+ * there is no name to print. The same shape appeared in this repository's
179
+ * own CI on 2026-10-20: `1 fail`, `1 error`, and `1 tests failed:`
180
+ * followed by an empty list.
181
+ *
182
+ * An empty array therefore does not mean "nothing failed" — `fail` means
183
+ * that. It means the runner named nothing, which over a non-zero `fail` is
184
+ * a fact an operator needs rather than a hole to fill in with silence.
185
+ * `describeSuiteFailures` is the one place that turns the two numbers into
186
+ * a sentence.
187
+ */
188
+ failingTests: string[];
189
+ /**
190
+ * Unhandled errors between tests, or null when the runner did not say.
191
+ *
192
+ * **null, never 0**, for the reason `skip` and `todo` give one field up: a
193
+ * runner that reports no error count and a runner that hit none are
194
+ * opposite facts. Unlike those two there is no total to resolve an absence
195
+ * against — bun's `Ran N tests` does not include errors, which is itself
196
+ * the point: an error is not a test.
197
+ */
198
+ errors: number | null;
199
+ /**
200
+ * Assertions the runner executed, or null when it did not say (#323).
201
+ *
202
+ * **null, never 0**, for the reason `skip`, `todo` and `errors` give
203
+ * above. A runner with no concept of assertion counting and a run that
204
+ * executed none are opposite facts, and `0` is the one value that would
205
+ * let a reader compare them.
206
+ *
207
+ * It is not part of any verdict. A run executing more assertions than the
208
+ * one before it is not a failure — see `lib/suite-flake.ts`, which reports
209
+ * the spread beside a verdict rather than inside one.
210
+ */
211
+ expectCalls: number | null;
77
212
  }
78
213
 
79
214
  export interface UnmeasuredRun {
@@ -151,6 +286,36 @@ export function readSuiteRun(output: unknown): SuiteRun {
151
286
  const tests = ran ? Number(ran[1]) : null;
152
287
  const files = ran ? Number(ran[2]) : null;
153
288
 
289
+ // The summary block, cut from the two lines that are always in it. bun
290
+ // prints `pass`, then `skip` and `todo` when non-zero, then `fail`, so the
291
+ // span between those two anchors is where a skip count belongs and
292
+ // anywhere else in the output is somebody else's run.
293
+ //
294
+ // #323 extended the END of the span to the `Ran N tests` line when there
295
+ // is one. Measured on bun 1.3.14 rather than assumed: `skip` and `todo`
296
+ // are printed BEFORE `fail` and `error` is printed AFTER it, so a span
297
+ // that stops at the `fail` line cannot see an error count at all — the
298
+ // first draft of this read `null` for a run that plainly said `1 error`.
299
+ // `Ran N tests across M files.` closes the block, so it is the anchor.
300
+ const at = (m: RegExpMatchArray) => m.index ?? 0;
301
+ const summary = output.slice(
302
+ Math.min(at(passMatch), at(failMatch)),
303
+ Math.max(
304
+ at(passMatch) + passMatch[0].length,
305
+ at(failMatch) + failMatch[0].length,
306
+ ran ? at(ran) + ran[0].length : 0,
307
+ ),
308
+ );
309
+ const skipMatch = lastMatch(SKIP_LINE, summary);
310
+ const todoMatch = lastMatch(TODO_LINE, summary);
311
+ const skipRead = skipMatch ? Number(skipMatch[1]) : null;
312
+ const todoRead = todoMatch ? Number(todoMatch[1]) : null;
313
+ // The runner's own total is what resolves an absent line. It closes only
314
+ // if the counts it did print account for everything it says it ran.
315
+ const absentIsZero = tests !== null && pass + fail + (skipRead ?? 0) + (todoRead ?? 0) === tests;
316
+ const skip = skipRead ?? (absentIsZero ? 0 : null);
317
+ const todo = todoRead ?? (absentIsZero ? 0 : null);
318
+
154
319
  // #490 SC-1, as a contradiction rather than as a second way of asking
155
320
  // whether anything ran. A runner claiming results it discovered no files to
156
321
  // produce is not reporting a measurement of this project, and it is the one
@@ -178,7 +343,82 @@ export function readSuiteRun(output: unknown): SuiteRun {
178
343
  );
179
344
  }
180
345
 
181
- return { status: "MEASURED", pass, fail, tests, files };
346
+ // #323. Read AFTER every refusal above, so a run with no result cannot
347
+ // arrive carrying a list of names: an UNMEASURED reading has no failures
348
+ // to name, and offering some would make the absence of a measurement look
349
+ // like a measurement with detail.
350
+ //
351
+ // The error count is read from the summary slice for the reason skip and
352
+ // todo are — a captured nested run's `1 error` is not this run's. The
353
+ // NAMES cannot be, and that asymmetry is reported rather than hidden.
354
+ const errorMatch = lastMatch(ERROR_LINE, summary);
355
+ const expectMatch = lastMatch(EXPECT_LINE, summary);
356
+ const failingTests = [...output.matchAll(FAIL_CASE_LINE)].map(m => m[1].trim()).filter(n => n !== "");
357
+
358
+ return {
359
+ status: "MEASURED",
360
+ pass,
361
+ fail,
362
+ tests,
363
+ files,
364
+ skip,
365
+ todo,
366
+ failingTests,
367
+ errors: errorMatch ? Number(errorMatch[1]) : null,
368
+ expectCalls: expectMatch ? Number(expectMatch[1]) : null,
369
+ };
370
+ }
371
+
372
+ /**
373
+ * The sentence a caller prints instead of `N fail` (#323).
374
+ *
375
+ * One binding, because the interesting part is not any one of the numbers —
376
+ * it is the relationship between `fail` and the names, and a caller that
377
+ * composed its own sentence would get that relationship right on the runs it
378
+ * had seen and wrong on the one that matters.
379
+ *
380
+ * Four readings, and the last two are the ones this exists for:
381
+ *
382
+ * - nothing failed;
383
+ * - every failure is named;
384
+ * - **the runner counted more failures than it named** — a module-scope
385
+ * error is the usual cause, so the error count travels with the sentence;
386
+ * - **more names were found than the summary counts**, which means the
387
+ * output contains another runner's `(fail)` lines. The names are not this
388
+ * run's to report, so they are withheld and the contradiction is stated.
389
+ * This repository's own suite drives `bun test` in subprocesses, so it is
390
+ * not hypothetical.
391
+ */
392
+ export function describeSuiteFailures(run: SuiteRun): string {
393
+ if (run.status !== "MEASURED") return run.reason;
394
+ const { fail, failingTests, errors } = run;
395
+ const named = failingTests.length;
396
+
397
+ if (named > fail) {
398
+ return (
399
+ `the output names ${named} failing test(s) but the summary counts ${fail} — it carries another ` +
400
+ "runner's output as well as this one's, so the names cannot be attributed to this run (#323)"
401
+ );
402
+ }
403
+ if (fail === 0) {
404
+ return named === 0
405
+ ? "0 failing tests"
406
+ : `0 failing tests by the summary, yet the output names ${named} — the counts contradict each other (#323)`;
407
+ }
408
+
409
+ const unnamed = fail - named;
410
+ const names = named === 0 ? "" : `: ${failingTests.join(", ")}`;
411
+ if (unnamed === 0) return `${fail} failing test(s)${names}`;
412
+
413
+ // The case a bare count cannot express, and the one #323 was filed over.
414
+ const why =
415
+ errors === null
416
+ ? "and the runner reported no error count, so what happened to the other name(s) is unrecorded"
417
+ : errors === 0
418
+ ? "and the runner reported 0 unhandled errors, so the missing name(s) are unexplained"
419
+ : `and the runner reported ${errors} unhandled error(s) between tests — a module-scope ` +
420
+ "failure has no test name, because the test that would carry it never ran";
421
+ return `${fail} failing test(s), ${unnamed} of them unnamed${names} — ${why} (#323)`;
182
422
  }
183
423
 
184
424
  /**