rungate 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/affected-tests.ts +398 -0
- package/lib/agent-audit.ts +2 -3
- package/lib/agent-worktree.ts +287 -0
- package/lib/agents-md-sections.ts +81 -16
- package/lib/behavioral-cache.ts +150 -2
- package/lib/branch-protection.ts +790 -0
- package/lib/brief-ac-alignment.ts +104 -0
- package/lib/brief-references.ts +205 -0
- package/lib/bun-version.ts +93 -14
- package/lib/check-remedy.ts +140 -0
- package/lib/ci-checkout.ts +17 -0
- package/lib/ci-history-depth.ts +159 -0
- package/lib/ci-setup-step.ts +96 -0
- package/lib/claude-code-action.ts +85 -0
- package/lib/claude-tool-vocabulary.ts +188 -0
- package/lib/cli-commands.ts +15 -0
- package/lib/commit-sha.ts +42 -0
- package/lib/commit-subject.ts +90 -0
- package/lib/compliance-report.ts +377 -45
- package/lib/compliance.ts +191 -17
- package/lib/config-loader.ts +39 -1
- package/lib/config-trackable.ts +391 -0
- package/lib/conformity.ts +1087 -100
- package/lib/containment.ts +238 -0
- package/lib/content-hash.ts +24 -0
- package/lib/create-sc.ts +249 -16
- package/lib/dev-liveness.ts +180 -0
- package/lib/discovery-branch-points.ts +432 -0
- package/lib/discovery-stability.ts +481 -0
- package/lib/distribution-currency.ts +186 -0
- package/lib/doc-routing.ts +157 -0
- package/lib/evidence-prevalidator.ts +224 -2
- package/lib/evidence-satisfiability.ts +534 -12
- package/lib/failure-intake.ts +317 -0
- package/lib/fallow-availability.ts +157 -0
- package/lib/fs-write-capabilities.ts +232 -0
- package/lib/gate-run-split.ts +147 -0
- package/lib/generators/agents-md.ts +140 -32
- package/lib/git-isolation.ts +140 -0
- package/lib/github.ts +15 -0
- package/lib/google-auth-action.ts +93 -0
- package/lib/harness-project.ts +48 -0
- package/lib/harness-provenance.ts +255 -0
- package/lib/hook-evidence.ts +235 -0
- package/lib/hook-scopes.ts +386 -0
- package/lib/hook-target-tree.ts +398 -0
- package/lib/hook-trees.ts +447 -0
- package/lib/js-top-level-code.ts +78 -10
- package/lib/junit-checks.ts +52 -0
- package/lib/launch-base.ts +253 -0
- package/lib/line-citation.ts +372 -0
- package/lib/orphan-reaper.ts +468 -0
- package/lib/parity-binding.ts +205 -0
- package/lib/per-test-timeouts.ts +219 -0
- package/lib/plan-coverage.ts +151 -0
- package/lib/prior-branch.ts +201 -8
- package/lib/process-table.ts +146 -0
- package/lib/project-state-derivation.ts +219 -0
- package/lib/project-state-root.ts +258 -0
- package/lib/published-artifact.ts +525 -0
- package/lib/readdir-sorted.ts +45 -0
- package/lib/review-execution-reading.ts +206 -0
- package/lib/review-tool-agreement.ts +433 -0
- package/lib/rule-promotion.ts +234 -0
- package/lib/rule-retirement.ts +525 -0
- package/lib/run-branch.ts +612 -0
- package/lib/rungate-schema.ts +19 -0
- package/lib/runner-home.ts +274 -0
- package/lib/sc-id-cross-tree.ts +19 -7
- package/lib/sc-ids.ts +121 -13
- package/lib/sc-reservations.ts +289 -0
- package/lib/scaffold/ai-review-findings.ts +403 -0
- package/lib/scaffold/ai-review-step.ts +4661 -64
- package/lib/scaffold/managed-workflow.ts +587 -192
- package/lib/scaffold/owned-files.ts +152 -45
- package/lib/scaffold/pre-push-hook.ts +49 -5
- package/lib/scaffold/steps.ts +1413 -244
- package/lib/scaffold/workflow-census.ts +217 -0
- package/lib/scaffold/workflow-drift.ts +217 -0
- package/lib/scaffold/write-steps.ts +4 -0
- package/lib/scan-budget.ts +247 -0
- package/lib/scanner.ts +43 -51
- package/lib/scratch-roots.ts +50 -5
- package/lib/screenshot-discovery.ts +121 -0
- package/lib/screenshot-upload.ts +41 -1
- package/lib/secret-patterns.ts +511 -8
- package/lib/ship-mode.ts +219 -0
- package/lib/source-slice.ts +294 -0
- package/lib/stale-cleanup.ts +37 -3
- package/lib/suite-flake.ts +278 -0
- package/lib/suite-measurement.ts +23 -4
- package/lib/suite-run-reading.ts +241 -1
- package/lib/suite-timeout-margin.ts +193 -0
- package/lib/test-case-bodies.ts +883 -0
- package/lib/test-case-deletion.ts +653 -0
- package/lib/test-census.ts +97 -29
- package/lib/test-command-positions.ts +274 -0
- package/lib/test-suite-lock.ts +159 -16
- package/lib/tool-probe.ts +31 -9
- package/lib/top-level-cd.ts +393 -0
- package/lib/tree-entries.ts +89 -0
- package/lib/ts-ast.ts +200 -0
- package/lib/walk-test-files.ts +97 -0
- package/lib/workflow-actions.ts +263 -0
- package/lib/worktree-collect.ts +177 -5
- package/lib/worktree-repo.ts +36 -1
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +45 -0
- package/scripts/create-sc.ts +21 -1
- package/scripts/extract-constraints.ts +107 -7
- package/scripts/generate-code-map.ts +23 -30
- package/scripts/scaffold-project.ts +8 -1
- package/scripts/sync-sc-status.ts +77 -16
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #323 SC-5 — whether a series of suite runs says anything.
|
|
3
|
+
*
|
|
4
|
+
* `lib/suite-run-reading.ts` reads ONE run. This reads a SERIES, and the two
|
|
5
|
+
* questions are genuinely different: a run is clean or it is not, and a
|
|
6
|
+
* series is consistent, flaky, failing, or not a series at all.
|
|
7
|
+
*
|
|
8
|
+
* One binding, because the interesting part is not any single run — it is
|
|
9
|
+
* the relationship between them, and a caller composing its own verdict
|
|
10
|
+
* would get that relationship right on the series it had seen and wrong on
|
|
11
|
+
* the one that matters. `scripts/suite-flake-run.ts` executes the runs and
|
|
12
|
+
* decides nothing.
|
|
13
|
+
*/
|
|
14
|
+
import { describeSuiteFailures, type SuiteRun } from "./suite-run-reading";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* What tree a run was taken against.
|
|
18
|
+
*
|
|
19
|
+
* `head` alone is not enough: the suites #323 suspects write scratch
|
|
20
|
+
* directories under the repository, and a run that dirties the tree it is
|
|
21
|
+
* measuring has changed the input to the next run without moving HEAD. The
|
|
22
|
+
* dirty list is `git status --porcelain` paths.
|
|
23
|
+
*/
|
|
24
|
+
export interface TreeSignature {
|
|
25
|
+
head: string;
|
|
26
|
+
dirty: string[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* How the process ended, which no reading of its TEXT can establish.
|
|
31
|
+
*
|
|
32
|
+
* `readSuiteRun` deliberately takes no exit code: #469's run exited 0 having
|
|
33
|
+
* measured nothing and run `39405680063` exited `null` having measured
|
|
34
|
+
* nothing, and nothing in either output says which way to resolve that. A
|
|
35
|
+
* SIGNAL is not ambiguous — but only the process's caller ever sees one, so
|
|
36
|
+
* it is carried here and read by the series.
|
|
37
|
+
*/
|
|
38
|
+
export interface RunExit {
|
|
39
|
+
status: number | null;
|
|
40
|
+
signal: string | null;
|
|
41
|
+
/** The spawn's own error, e.g. `ENOBUFS`, which names the cause. */
|
|
42
|
+
error: string | null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface FlakeRun {
|
|
46
|
+
/** 1-based, so a reason can name the run an operator will look for. */
|
|
47
|
+
index: number;
|
|
48
|
+
signature: TreeSignature;
|
|
49
|
+
run: SuiteRun;
|
|
50
|
+
/** Where the full output was kept. A failure that is not kept is #323. */
|
|
51
|
+
log: string;
|
|
52
|
+
seconds: number;
|
|
53
|
+
exit: RunExit;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Why a run's ENDING disqualifies it, or null.
|
|
58
|
+
*
|
|
59
|
+
* `bun test` exits 0 for a clean suite and 1 for a failing one. **1 must not
|
|
60
|
+
* be refused**: refusing every non-zero exit would turn every FLAKY and
|
|
61
|
+
* FAILING series into UNMEASURED and leave the instrument unable to report
|
|
62
|
+
* its own subject.
|
|
63
|
+
*/
|
|
64
|
+
export function exitRefusal(exit: RunExit): string | null {
|
|
65
|
+
if (exit.error !== null && exit.error !== "") {
|
|
66
|
+
return `the spawn failed: ${exit.error}`;
|
|
67
|
+
}
|
|
68
|
+
if (exit.signal !== null) {
|
|
69
|
+
return (
|
|
70
|
+
`it was killed by ${exit.signal}, so its log ends wherever the process did — and a truncated log ` +
|
|
71
|
+
"still parses as a clean reading when the output it captured contains a nested suite's summary, " +
|
|
72
|
+
"which is how run 39405680063 reported CONSISTENT over ten killed runs (#323)"
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
if (exit.status === null) {
|
|
76
|
+
return "the spawn reported no exit status and no signal, so nothing says how it ended";
|
|
77
|
+
}
|
|
78
|
+
if (exit.status !== 0 && exit.status !== 1) {
|
|
79
|
+
return `it exited ${exit.status}, which is not a verdict about the tests — bun uses 0 for a clean suite and 1 for a failing one`;
|
|
80
|
+
}
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* CONSISTENT, FLAKY and FAILING are three facts, not two.
|
|
86
|
+
*
|
|
87
|
+
* A suite that fails every run is red — an ordinary thing to be, and
|
|
88
|
+
* somebody's build to fix. A suite that fails SOME runs is the defect #323
|
|
89
|
+
* is about. One word for both would leave the series unable to report its
|
|
90
|
+
* own subject.
|
|
91
|
+
*/
|
|
92
|
+
export type FlakeVerdict = "CONSISTENT" | "FLAKY" | "FAILING" | "UNMEASURED";
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Whether the runs agreed about how much work they did.
|
|
96
|
+
*
|
|
97
|
+
* Reported BESIDE the verdict and never folded into it. The 2026-10-11
|
|
98
|
+
* reading's only surviving lead is that the failing run executed 384 more
|
|
99
|
+
* `expect()` calls than either clean one — so a varying count is a finding.
|
|
100
|
+
* It is not, on its own, evidence that anything is wrong: a case enumerating
|
|
101
|
+
* a directory legitimately varies. Folding it into the verdict would make
|
|
102
|
+
* one of those two readings unavailable.
|
|
103
|
+
*
|
|
104
|
+
* `UNREPORTED` is first-class. A runner that printed no count and a runner
|
|
105
|
+
* whose count held steady are opposite facts, and `STABLE` is the value that
|
|
106
|
+
* would let a reader treat the first as the second.
|
|
107
|
+
*/
|
|
108
|
+
export type AssertionStability = "STABLE" | "VARIED" | "UNREPORTED";
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* What `treeSignatureKey` can and cannot see, carried in the reading itself.
|
|
112
|
+
*
|
|
113
|
+
* A `CONSISTENT` verdict says the runs agreed over one tree, and a reader
|
|
114
|
+
* naturally hears "one set of inputs". It is not: the signature is HEAD plus
|
|
115
|
+
* `git status --porcelain`, so a gitignored path, anything under `/tmp` or
|
|
116
|
+
* `~/.rungate`, and `node_modules` can all change between runs with the
|
|
117
|
+
* signature unmoved.
|
|
118
|
+
*
|
|
119
|
+
* #799 is the measured instance rather than a worry. Series `39412289321`
|
|
120
|
+
* reported ten runs over one tree and was right to — while `test/phase-0.test.ts`
|
|
121
|
+
* collected its test list from a `/tmp` fixture the PREVIOUS run had left, so
|
|
122
|
+
* run 1 executed 15,194 tests and runs 2 through 10 executed 15,250.
|
|
123
|
+
*
|
|
124
|
+
* In the reading and not only in the printed sentence, because `flake.json`
|
|
125
|
+
* is what anybody reads months later, and a caveat that exists only in a
|
|
126
|
+
* console line is a caveat the artefact does not carry.
|
|
127
|
+
*/
|
|
128
|
+
export const SIGNATURE_SCOPE =
|
|
129
|
+
"the tree signature is HEAD plus `git status --porcelain`, so it covers the working tree and nothing " +
|
|
130
|
+
"else — a gitignored path, anything under /tmp or ~/.rungate, and node_modules can all differ between " +
|
|
131
|
+
"runs while this series reports one tree (#799)";
|
|
132
|
+
|
|
133
|
+
export interface FlakeReading {
|
|
134
|
+
verdict: FlakeVerdict;
|
|
135
|
+
/** Why it is not CONSISTENT. Empty exactly when it is. */
|
|
136
|
+
reasons: string[];
|
|
137
|
+
assertions: AssertionStability;
|
|
138
|
+
/** One entry per run, in order, `null` where the runner did not say. */
|
|
139
|
+
assertionCounts: Array<number | null>;
|
|
140
|
+
runs: number;
|
|
141
|
+
/** What the tree comparison covered. Always present; see `SIGNATURE_SCOPE`. */
|
|
142
|
+
signatureScope: string;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* One string per tree, so two readings can be compared without anybody
|
|
147
|
+
* deciding field by field what counts as the same tree.
|
|
148
|
+
*
|
|
149
|
+
* The dirty list is sorted: `git status --porcelain` order is not a property
|
|
150
|
+
* of the tree, and two runs differing only in it are one tree.
|
|
151
|
+
*/
|
|
152
|
+
export function treeSignatureKey(signature: TreeSignature): string {
|
|
153
|
+
return `${signature.head}\n${[...signature.dirty].sort().join("\n")}`;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** The runs that could be read at all. */
|
|
157
|
+
function measured(runs: FlakeRun[]) {
|
|
158
|
+
return runs.flatMap(r => (r.run.status === "MEASURED" ? [{ entry: r, run: r.run }] : []));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Read a series.
|
|
163
|
+
*
|
|
164
|
+
* FAILS CLOSED, like the single-run reader it sits on. Every path that
|
|
165
|
+
* cannot positively establish that `minimum` readable runs were taken over
|
|
166
|
+
* one tree returns UNMEASURED with reasons. There is no path that returns
|
|
167
|
+
* CONSISTENT from a series that does not say so.
|
|
168
|
+
*/
|
|
169
|
+
export function flakeReading(runs: FlakeRun[], minimum: number): FlakeReading {
|
|
170
|
+
const assertionCounts = runs.map(r => (r.run.status === "MEASURED" ? r.run.expectCalls : null));
|
|
171
|
+
const reasons: string[] = [];
|
|
172
|
+
|
|
173
|
+
// A short series first, because it explains every other absence. A job
|
|
174
|
+
// that died after three runs has measured three runs, and ten is what was
|
|
175
|
+
// asked for — the instrument failing is not the subject passing (#60).
|
|
176
|
+
if (runs.length < minimum) {
|
|
177
|
+
reasons.push(
|
|
178
|
+
`only ${runs.length} of ${minimum} run(s) completed, so the series was cut short — ` +
|
|
179
|
+
"a short series is not a clean series, however green the runs in it were (#323 SC-5)",
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Every run that did not FINISH, before anything is read off its text.
|
|
184
|
+
// This is the guard run 39405680063 did not have: ten children killed by
|
|
185
|
+
// SIGTERM on a 1 MiB `maxBuffer` overflow, ten truncated logs whose only
|
|
186
|
+
// surviving summary belonged to a nested suite, and a CONSISTENT verdict
|
|
187
|
+
// over a suite of 5,500 tests reporting one pass.
|
|
188
|
+
for (const r of runs) {
|
|
189
|
+
const why = exitRefusal(r.exit);
|
|
190
|
+
if (why !== null) reasons.push(`run ${r.index} did not finish (log ${r.log}): ${why}`);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Every unreadable run, by index. An UNMEASURED run is not a clean run:
|
|
194
|
+
// `readSuiteRun` returns it for a log that printed no summary, and this
|
|
195
|
+
// repository's logs carry nested suites' summaries, so such a log still
|
|
196
|
+
// contains the text a `grep "0 fail"` would match.
|
|
197
|
+
for (const r of runs) {
|
|
198
|
+
if (r.run.status !== "MEASURED") {
|
|
199
|
+
reasons.push(`run ${r.index} could not be read (log ${r.log}): ${r.run.reason}`);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// One tree, or it is not a series. Named by what differs, because
|
|
204
|
+
// "the tree moved" tells nobody where to look and a scratch directory
|
|
205
|
+
// appearing under the repository is SC-6's whole hypothesis.
|
|
206
|
+
const trees = new Map<string, FlakeRun[]>();
|
|
207
|
+
for (const r of runs) {
|
|
208
|
+
const key = treeSignatureKey(r.signature);
|
|
209
|
+
trees.set(key, [...(trees.get(key) ?? []), r]);
|
|
210
|
+
}
|
|
211
|
+
if (trees.size > 1) {
|
|
212
|
+
const described = [...trees.entries()].map(([key, group]) => {
|
|
213
|
+
const where = group.map(g => g.index).join(", ");
|
|
214
|
+
const [head, ...dirty] = key.split("\n");
|
|
215
|
+
const paths = dirty.filter(d => d !== "");
|
|
216
|
+
return `run(s) ${where} at ${head}${paths.length === 0 ? " with a clean tree" : ` with ${paths.join(", ")}`}`;
|
|
217
|
+
});
|
|
218
|
+
reasons.push(
|
|
219
|
+
`the tree changed during the series, so these runs did not measure one thing: ${described.join("; ")} ` +
|
|
220
|
+
"— ten green runs over three trees is not ten green runs (#323 SC-5)",
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const counts = assertionCounts.filter((c): c is number => c !== null);
|
|
225
|
+
const assertions: AssertionStability =
|
|
226
|
+
counts.length !== runs.length || runs.length === 0
|
|
227
|
+
? "UNREPORTED"
|
|
228
|
+
: new Set(counts).size === 1
|
|
229
|
+
? "STABLE"
|
|
230
|
+
: "VARIED";
|
|
231
|
+
|
|
232
|
+
if (reasons.length > 0) {
|
|
233
|
+
return { verdict: "UNMEASURED", reasons, assertions, assertionCounts, runs: runs.length, signatureScope: SIGNATURE_SCOPE };
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const readable = measured(runs);
|
|
237
|
+
const failing = readable.filter(r => r.run.fail > 0);
|
|
238
|
+
if (failing.length > 0) {
|
|
239
|
+
for (const f of failing) {
|
|
240
|
+
reasons.push(`run ${f.entry.index} (log ${f.entry.log}): ${describeSuiteFailures(f.run)}`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const verdict: FlakeVerdict =
|
|
245
|
+
failing.length === 0 ? "CONSISTENT" : failing.length === readable.length ? "FAILING" : "FLAKY";
|
|
246
|
+
|
|
247
|
+
return { verdict, reasons, assertions, assertionCounts, runs: runs.length, signatureScope: SIGNATURE_SCOPE };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** The sentence a caller prints instead of the verdict on its own. */
|
|
251
|
+
export function describeFlakeReading(reading: FlakeReading): string {
|
|
252
|
+
const { verdict, runs, assertions, assertionCounts, reasons } = reading;
|
|
253
|
+
const counts = assertionCounts.filter((c): c is number => c !== null);
|
|
254
|
+
const spread =
|
|
255
|
+
assertions === "VARIED"
|
|
256
|
+
? ` The runs did not agree about how much work they did: ${Math.min(...counts)} to ` +
|
|
257
|
+
`${Math.max(...counts)} expect() calls, a spread of ${Math.max(...counts) - Math.min(...counts)}. ` +
|
|
258
|
+
"That is the only surviving lead from the 2026-10-11 reading (#323 SC-4)."
|
|
259
|
+
: assertions === "UNREPORTED"
|
|
260
|
+
? " The runner did not report an assertion count for every run, so nothing is claimed about one."
|
|
261
|
+
: ` Every run executed ${counts[0]} expect() calls.`;
|
|
262
|
+
|
|
263
|
+
const failed = reasons.length;
|
|
264
|
+
const head =
|
|
265
|
+
verdict === "CONSISTENT"
|
|
266
|
+
? `CONSISTENT: ${runs} consecutive run(s) on one tree, none failing.`
|
|
267
|
+
: verdict === "FLAKY"
|
|
268
|
+
? `FLAKY: ${failed} of ${runs} consecutive run(s) on one tree failed — the same tree, different answers.`
|
|
269
|
+
: verdict === "FAILING"
|
|
270
|
+
? `FAILING: all ${runs} run(s) failed. The suite is red; this is not flake.`
|
|
271
|
+
: `UNMEASURED: the series does not support a verdict.`;
|
|
272
|
+
|
|
273
|
+
// The scope goes on EVERY verdict, not only CONSISTENT. A reader who sees
|
|
274
|
+
// it once beside a clean series and never again would reasonably conclude
|
|
275
|
+
// the other verdicts had looked harder, and they did not — it is the same
|
|
276
|
+
// comparison in all four cases.
|
|
277
|
+
return [head, ...reasons.map(r => ` - ${r}`), spread.trim(), `Scope: ${reading.signatureScope}`].join("\n");
|
|
278
|
+
}
|
package/lib/suite-measurement.ts
CHANGED
|
@@ -28,15 +28,21 @@ const SHA = /^[0-9a-f]{7,40}$/i;
|
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
30
|
* UNMEASURED is not a result. It is the statement that this run has no suite
|
|
31
|
-
* evidence, and it is deliberately not writable — see gates/schema.ts,
|
|
32
|
-
*
|
|
33
|
-
* saying there was no measurement.
|
|
31
|
+
* evidence, and it is deliberately not writable — see gates/schema.ts, whose
|
|
32
|
+
* writable verdicts are PASS, FAIL and PARTIAL. A recorded UNMEASURED would
|
|
33
|
+
* be a measurement saying there was no measurement.
|
|
34
|
+
*
|
|
35
|
+
* PARTIAL is writable and is not a verdict THIS type has, which is the whole
|
|
36
|
+
* of #673's first half: a reading taken over less than the suite is a reading
|
|
37
|
+
* that was taken, so the recorder may write it — and it is not a statement
|
|
38
|
+
* about the suite, so every consumer must see UNMEASURED. The conversion
|
|
39
|
+
* happens in `readSuiteMeasurement` below and nowhere else.
|
|
34
40
|
*/
|
|
35
41
|
export type SuiteVerdict = "PASS" | "FAIL" | "UNMEASURED";
|
|
36
42
|
|
|
37
43
|
/** What a recorder persists at `suiteMeasurement` in workflow-state.json. */
|
|
38
44
|
export interface SuiteMeasurement {
|
|
39
|
-
verdict: "PASS" | "FAIL";
|
|
45
|
+
verdict: "PASS" | "FAIL" | "PARTIAL";
|
|
40
46
|
/** The commit the suite actually ran against. */
|
|
41
47
|
measuredSha: string;
|
|
42
48
|
/** Failing tests. 0 for a PASS, at least 1 for a FAIL. */
|
|
@@ -136,6 +142,19 @@ export function readSuiteMeasurement(state: unknown): SuiteReading {
|
|
|
136
142
|
}
|
|
137
143
|
|
|
138
144
|
const verdict = raw.verdict;
|
|
145
|
+
// #673 — a reading the caller declared PARTIAL. Its own branch rather than
|
|
146
|
+
// falling through to the clause below, because "nobody recognises this
|
|
147
|
+
// word" and "this run measured some tests and said so" are different
|
|
148
|
+
// things to go and do something about: the first is a corrupt record and
|
|
149
|
+
// the second is a run that has not measured the suite YET.
|
|
150
|
+
if (verdict === "PARTIAL") {
|
|
151
|
+
return unmeasured(
|
|
152
|
+
`the suite measurement for ${measuredSha} was taken over PART of the suite: the caller ` +
|
|
153
|
+
`declared the run it recorded was less than the whole of it, so what it found is not a ` +
|
|
154
|
+
`statement about the suite (#673). ${describe(raw.tests)} test(s) collected is not a ` +
|
|
155
|
+
`reading a gate can act on, whatever the count`,
|
|
156
|
+
);
|
|
157
|
+
}
|
|
139
158
|
if (verdict !== "PASS" && verdict !== "FAIL") {
|
|
140
159
|
return unmeasured(
|
|
141
160
|
`the suite measurement for ${measuredSha} carries no usable verdict (got ${describe(verdict)}) — ` +
|
package/lib/suite-run-reading.ts
CHANGED
|
@@ -53,6 +53,85 @@ const FAIL_LINE = /^\s*(\d+)\s+fail\s*$/gm;
|
|
|
53
53
|
/** `Ran 11521 tests across 324 files.` — and `1 test across 1 file.` */
|
|
54
54
|
const RAN_LINE = /^Ran\s+(\d+)\s+tests?\s+across\s+(\d+)\s+files?\./gm;
|
|
55
55
|
|
|
56
|
+
/**
|
|
57
|
+
* `13 skip` and `818 todo` — bun prints these only when they are non-zero.
|
|
58
|
+
*
|
|
59
|
+
* Read so that `pass + fail` and `tests` stop being two unexplained numbers
|
|
60
|
+
* for one event (#673). On run `wf_62d68d55-81c` the changelog said 13084
|
|
61
|
+
* and the structured field said 13915; the difference was exactly this
|
|
62
|
+
* run's `13 skip + 818 todo`, and nothing in the record said so. A reader
|
|
63
|
+
* comparing either against a previous run's number may be comparing
|
|
64
|
+
* different quantities.
|
|
65
|
+
*
|
|
66
|
+
* **Searched in the SUMMARY BLOCK, not the document.** `pass` and `fail` get
|
|
67
|
+
* away with `lastMatch` over the whole output because the outer run's
|
|
68
|
+
* summary is the last thing printed. These two cannot: bun OMITS them when
|
|
69
|
+
* the count is zero, so a last-match over the document walks BACKWARD past
|
|
70
|
+
* the summary into whatever the suite captured. That is not hypothetical —
|
|
71
|
+
* `~/.rungate/pai-harness-669/suite-output.txt` carries a nested
|
|
72
|
+
* `1 pass / 0 fail` at line 2600 of 2688, from a test that drives bun
|
|
73
|
+
* itself. A nested run's skips would have been recorded as the outer run's.
|
|
74
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`: search the slice.
|
|
75
|
+
*
|
|
76
|
+
* **Absent means zero only when the producer's own total says so.** bun
|
|
77
|
+
* omits the line at zero, so for bun absent and zero are the same statement
|
|
78
|
+
* — but that is a fact about one runner, and a reading that asserts `skip:
|
|
79
|
+
* 0` over a runner with no concept of skipping is a number nobody measured.
|
|
80
|
+
* `pass + fail + skip + todo === tests` is the producer's own arithmetic,
|
|
81
|
+
* and it is what resolves the absence: when it closes, the missing counts
|
|
82
|
+
* really are zero; when it does not, they are `null` — not reported. Both
|
|
83
|
+
* findings were raised by the security review of this change.
|
|
84
|
+
*/
|
|
85
|
+
const SKIP_LINE = /^\s*(\d+)\s+skip\s*$/gm;
|
|
86
|
+
const TODO_LINE = /^\s*(\d+)\s+todo\s*$/gm;
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* `1 error` — unhandled errors between tests (#323).
|
|
90
|
+
*
|
|
91
|
+
* In the summary block beside `skip` and `todo`, and omitted at zero like
|
|
92
|
+
* they are. It is read for one reason: **it is how a failure loses its
|
|
93
|
+
* name.** A module-scope throw counts toward `fail` and prints no `(fail)`
|
|
94
|
+
* line, because the test that would have carried the name never ran.
|
|
95
|
+
*/
|
|
96
|
+
const ERROR_LINE = /^\s*(\d+)\s+errors?\s*$/gm;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* `13648 expect() calls` — how much work the run actually did (#323 SC-4).
|
|
100
|
+
*
|
|
101
|
+
* Read because it is the only surviving lead from the 2026-10-11 reading
|
|
102
|
+
* this issue is about: three consecutive runs on one tree, one of which
|
|
103
|
+
* reported a failure, and the only difference between the failing run and
|
|
104
|
+
* either clean one is that it executed **384 more assertions** against an
|
|
105
|
+
* almost identical test count. That is not a test flipping its verdict; it
|
|
106
|
+
* is a test executing a substantially different amount of work.
|
|
107
|
+
*
|
|
108
|
+
* Nobody could say so at the time, because the number was in a terminal
|
|
109
|
+
* scrollback and nowhere else. As a field it is a reading, and
|
|
110
|
+
* `lib/suite-flake.ts` compares it across a series.
|
|
111
|
+
*
|
|
112
|
+
* In the summary block beside `skip`, `todo` and `error`, and for the same
|
|
113
|
+
* reason: this repository's suite drives `bun test` in subprocesses, so a
|
|
114
|
+
* last-match over the document walks backward out of the summary into a
|
|
115
|
+
* nested run's count.
|
|
116
|
+
*/
|
|
117
|
+
const EXPECT_LINE = /^\s*(\d+)\s+expect\(\)\s+calls\s*$/gm;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* `(fail) suite > case [0.12ms]` — one line per failing test.
|
|
121
|
+
*
|
|
122
|
+
* Anchored to the whole line, and the timing suffix is optional because bun
|
|
123
|
+
* omits it for a case that never started. The name is everything between.
|
|
124
|
+
*
|
|
125
|
+
* This cannot be cut to the summary block the way `skip` and `todo` are:
|
|
126
|
+
* bun prints these as the run goes, throughout the output. That is a real
|
|
127
|
+
* limitation and it is reported rather than papered over — see
|
|
128
|
+
* `describeSuiteFailures`, which refuses to list names when there are more
|
|
129
|
+
* of them than the summary counts, because this repository's own suite
|
|
130
|
+
* captures nested `bun test` output and a nested run's `(fail)` lines sit at
|
|
131
|
+
* column zero like everybody else's.
|
|
132
|
+
*/
|
|
133
|
+
const FAIL_CASE_LINE = /^\s*\(fail\)\s+(\S.*?)(?:\s+\[[\d.]+\s*m?s\])?\s*$/gm;
|
|
134
|
+
|
|
56
135
|
export interface MeasuredRun {
|
|
57
136
|
status: "MEASURED";
|
|
58
137
|
pass: number;
|
|
@@ -65,6 +144,20 @@ export interface MeasuredRun {
|
|
|
65
144
|
* whole module is that a number nobody measured must not be reportable.
|
|
66
145
|
*/
|
|
67
146
|
tests: number | null;
|
|
147
|
+
/**
|
|
148
|
+
* Tests the runner declined to execute, by kind (#673).
|
|
149
|
+
*
|
|
150
|
+
* These exist so that "tests that RAN" and "tests that EXIST" are both
|
|
151
|
+
* nameable without anybody subtracting to find out which is which.
|
|
152
|
+
*
|
|
153
|
+
* **null, not 0, when the runner did not say.** A missing line resolves to
|
|
154
|
+
* zero only when the runner's own `Ran N tests` total closes without it —
|
|
155
|
+
* see the patterns above. "This runner reports no skip count" and "this
|
|
156
|
+
* runner skipped none" are opposite facts, and `0` is the one value that
|
|
157
|
+
* would let a reader add them together.
|
|
158
|
+
*/
|
|
159
|
+
skip: number | null;
|
|
160
|
+
todo: number | null;
|
|
68
161
|
/**
|
|
69
162
|
* Test files the runner discovered, or null when it did not say.
|
|
70
163
|
*
|
|
@@ -74,6 +167,48 @@ export interface MeasuredRun {
|
|
|
74
167
|
* *Silent Default Substitution*, pointed at the number #490 is about.
|
|
75
168
|
*/
|
|
76
169
|
files: number | null;
|
|
170
|
+
/**
|
|
171
|
+
* The names of failing tests the runner printed, in the order it printed
|
|
172
|
+
* them (#323).
|
|
173
|
+
*
|
|
174
|
+
* Beside `fail` and not instead of it, because the two can disagree and
|
|
175
|
+
* the disagreement is the finding. Measured on bun 1.3.14: two test files,
|
|
176
|
+
* one ordinary failing assertion and one module-scope `throw`, report
|
|
177
|
+
* `2 fail` and **one** `(fail)` line — the throw's test never ran, so
|
|
178
|
+
* there is no name to print. The same shape appeared in this repository's
|
|
179
|
+
* own CI on 2026-10-20: `1 fail`, `1 error`, and `1 tests failed:`
|
|
180
|
+
* followed by an empty list.
|
|
181
|
+
*
|
|
182
|
+
* An empty array therefore does not mean "nothing failed" — `fail` means
|
|
183
|
+
* that. It means the runner named nothing, which over a non-zero `fail` is
|
|
184
|
+
* a fact an operator needs rather than a hole to fill in with silence.
|
|
185
|
+
* `describeSuiteFailures` is the one place that turns the two numbers into
|
|
186
|
+
* a sentence.
|
|
187
|
+
*/
|
|
188
|
+
failingTests: string[];
|
|
189
|
+
/**
|
|
190
|
+
* Unhandled errors between tests, or null when the runner did not say.
|
|
191
|
+
*
|
|
192
|
+
* **null, never 0**, for the reason `skip` and `todo` give one field up: a
|
|
193
|
+
* runner that reports no error count and a runner that hit none are
|
|
194
|
+
* opposite facts. Unlike those two there is no total to resolve an absence
|
|
195
|
+
* against — bun's `Ran N tests` does not include errors, which is itself
|
|
196
|
+
* the point: an error is not a test.
|
|
197
|
+
*/
|
|
198
|
+
errors: number | null;
|
|
199
|
+
/**
|
|
200
|
+
* Assertions the runner executed, or null when it did not say (#323).
|
|
201
|
+
*
|
|
202
|
+
* **null, never 0**, for the reason `skip`, `todo` and `errors` give
|
|
203
|
+
* above. A runner with no concept of assertion counting and a run that
|
|
204
|
+
* executed none are opposite facts, and `0` is the one value that would
|
|
205
|
+
* let a reader compare them.
|
|
206
|
+
*
|
|
207
|
+
* It is not part of any verdict. A run executing more assertions than the
|
|
208
|
+
* one before it is not a failure — see `lib/suite-flake.ts`, which reports
|
|
209
|
+
* the spread beside a verdict rather than inside one.
|
|
210
|
+
*/
|
|
211
|
+
expectCalls: number | null;
|
|
77
212
|
}
|
|
78
213
|
|
|
79
214
|
export interface UnmeasuredRun {
|
|
@@ -151,6 +286,36 @@ export function readSuiteRun(output: unknown): SuiteRun {
|
|
|
151
286
|
const tests = ran ? Number(ran[1]) : null;
|
|
152
287
|
const files = ran ? Number(ran[2]) : null;
|
|
153
288
|
|
|
289
|
+
// The summary block, cut from the two lines that are always in it. bun
|
|
290
|
+
// prints `pass`, then `skip` and `todo` when non-zero, then `fail`, so the
|
|
291
|
+
// span between those two anchors is where a skip count belongs and
|
|
292
|
+
// anywhere else in the output is somebody else's run.
|
|
293
|
+
//
|
|
294
|
+
// #323 extended the END of the span to the `Ran N tests` line when there
|
|
295
|
+
// is one. Measured on bun 1.3.14 rather than assumed: `skip` and `todo`
|
|
296
|
+
// are printed BEFORE `fail` and `error` is printed AFTER it, so a span
|
|
297
|
+
// that stops at the `fail` line cannot see an error count at all — the
|
|
298
|
+
// first draft of this read `null` for a run that plainly said `1 error`.
|
|
299
|
+
// `Ran N tests across M files.` closes the block, so it is the anchor.
|
|
300
|
+
const at = (m: RegExpMatchArray) => m.index ?? 0;
|
|
301
|
+
const summary = output.slice(
|
|
302
|
+
Math.min(at(passMatch), at(failMatch)),
|
|
303
|
+
Math.max(
|
|
304
|
+
at(passMatch) + passMatch[0].length,
|
|
305
|
+
at(failMatch) + failMatch[0].length,
|
|
306
|
+
ran ? at(ran) + ran[0].length : 0,
|
|
307
|
+
),
|
|
308
|
+
);
|
|
309
|
+
const skipMatch = lastMatch(SKIP_LINE, summary);
|
|
310
|
+
const todoMatch = lastMatch(TODO_LINE, summary);
|
|
311
|
+
const skipRead = skipMatch ? Number(skipMatch[1]) : null;
|
|
312
|
+
const todoRead = todoMatch ? Number(todoMatch[1]) : null;
|
|
313
|
+
// The runner's own total is what resolves an absent line. It closes only
|
|
314
|
+
// if the counts it did print account for everything it says it ran.
|
|
315
|
+
const absentIsZero = tests !== null && pass + fail + (skipRead ?? 0) + (todoRead ?? 0) === tests;
|
|
316
|
+
const skip = skipRead ?? (absentIsZero ? 0 : null);
|
|
317
|
+
const todo = todoRead ?? (absentIsZero ? 0 : null);
|
|
318
|
+
|
|
154
319
|
// #490 SC-1, as a contradiction rather than as a second way of asking
|
|
155
320
|
// whether anything ran. A runner claiming results it discovered no files to
|
|
156
321
|
// produce is not reporting a measurement of this project, and it is the one
|
|
@@ -178,7 +343,82 @@ export function readSuiteRun(output: unknown): SuiteRun {
|
|
|
178
343
|
);
|
|
179
344
|
}
|
|
180
345
|
|
|
181
|
-
|
|
346
|
+
// #323. Read AFTER every refusal above, so a run with no result cannot
|
|
347
|
+
// arrive carrying a list of names: an UNMEASURED reading has no failures
|
|
348
|
+
// to name, and offering some would make the absence of a measurement look
|
|
349
|
+
// like a measurement with detail.
|
|
350
|
+
//
|
|
351
|
+
// The error count is read from the summary slice for the reason skip and
|
|
352
|
+
// todo are — a captured nested run's `1 error` is not this run's. The
|
|
353
|
+
// NAMES cannot be, and that asymmetry is reported rather than hidden.
|
|
354
|
+
const errorMatch = lastMatch(ERROR_LINE, summary);
|
|
355
|
+
const expectMatch = lastMatch(EXPECT_LINE, summary);
|
|
356
|
+
const failingTests = [...output.matchAll(FAIL_CASE_LINE)].map(m => m[1].trim()).filter(n => n !== "");
|
|
357
|
+
|
|
358
|
+
return {
|
|
359
|
+
status: "MEASURED",
|
|
360
|
+
pass,
|
|
361
|
+
fail,
|
|
362
|
+
tests,
|
|
363
|
+
files,
|
|
364
|
+
skip,
|
|
365
|
+
todo,
|
|
366
|
+
failingTests,
|
|
367
|
+
errors: errorMatch ? Number(errorMatch[1]) : null,
|
|
368
|
+
expectCalls: expectMatch ? Number(expectMatch[1]) : null,
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* The sentence a caller prints instead of `N fail` (#323).
|
|
374
|
+
*
|
|
375
|
+
* One binding, because the interesting part is not any one of the numbers —
|
|
376
|
+
* it is the relationship between `fail` and the names, and a caller that
|
|
377
|
+
* composed its own sentence would get that relationship right on the runs it
|
|
378
|
+
* had seen and wrong on the one that matters.
|
|
379
|
+
*
|
|
380
|
+
* Four readings, and the last two are the ones this exists for:
|
|
381
|
+
*
|
|
382
|
+
* - nothing failed;
|
|
383
|
+
* - every failure is named;
|
|
384
|
+
* - **the runner counted more failures than it named** — a module-scope
|
|
385
|
+
* error is the usual cause, so the error count travels with the sentence;
|
|
386
|
+
* - **more names were found than the summary counts**, which means the
|
|
387
|
+
* output contains another runner's `(fail)` lines. The names are not this
|
|
388
|
+
* run's to report, so they are withheld and the contradiction is stated.
|
|
389
|
+
* This repository's own suite drives `bun test` in subprocesses, so it is
|
|
390
|
+
* not hypothetical.
|
|
391
|
+
*/
|
|
392
|
+
export function describeSuiteFailures(run: SuiteRun): string {
|
|
393
|
+
if (run.status !== "MEASURED") return run.reason;
|
|
394
|
+
const { fail, failingTests, errors } = run;
|
|
395
|
+
const named = failingTests.length;
|
|
396
|
+
|
|
397
|
+
if (named > fail) {
|
|
398
|
+
return (
|
|
399
|
+
`the output names ${named} failing test(s) but the summary counts ${fail} — it carries another ` +
|
|
400
|
+
"runner's output as well as this one's, so the names cannot be attributed to this run (#323)"
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
if (fail === 0) {
|
|
404
|
+
return named === 0
|
|
405
|
+
? "0 failing tests"
|
|
406
|
+
: `0 failing tests by the summary, yet the output names ${named} — the counts contradict each other (#323)`;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
const unnamed = fail - named;
|
|
410
|
+
const names = named === 0 ? "" : `: ${failingTests.join(", ")}`;
|
|
411
|
+
if (unnamed === 0) return `${fail} failing test(s)${names}`;
|
|
412
|
+
|
|
413
|
+
// The case a bare count cannot express, and the one #323 was filed over.
|
|
414
|
+
const why =
|
|
415
|
+
errors === null
|
|
416
|
+
? "and the runner reported no error count, so what happened to the other name(s) is unrecorded"
|
|
417
|
+
: errors === 0
|
|
418
|
+
? "and the runner reported 0 unhandled errors, so the missing name(s) are unexplained"
|
|
419
|
+
: `and the runner reported ${errors} unhandled error(s) between tests — a module-scope ` +
|
|
420
|
+
"failure has no test name, because the test that would carry it never ran";
|
|
421
|
+
return `${fail} failing test(s), ${unnamed} of them unnamed${names} — ${why} (#323)`;
|
|
182
422
|
}
|
|
183
423
|
|
|
184
424
|
/**
|