rungate 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/affected-tests.ts +398 -0
- package/lib/agent-audit.ts +2 -3
- package/lib/agent-worktree.ts +287 -0
- package/lib/agents-md-sections.ts +209 -0
- package/lib/behavioral-cache.ts +150 -2
- package/lib/branch-diff.ts +139 -0
- package/lib/branch-protection.ts +790 -0
- package/lib/brief-ac-alignment.ts +104 -0
- package/lib/brief-references.ts +205 -0
- package/lib/bun-version.ts +347 -0
- package/lib/check-remedy.ts +140 -0
- package/lib/ci-checkout.ts +18 -1
- package/lib/ci-history-depth.ts +159 -0
- package/lib/ci-setup-step.ts +96 -0
- package/lib/claude-code-action.ts +85 -0
- package/lib/claude-tool-vocabulary.ts +188 -0
- package/lib/cli-commands.ts +15 -0
- package/lib/commit-sha.ts +42 -0
- package/lib/commit-subject.ts +90 -0
- package/lib/compliance-report.ts +377 -45
- package/lib/compliance.ts +196 -20
- package/lib/config-loader.ts +51 -1
- package/lib/config-trackable.ts +391 -0
- package/lib/conformity.ts +1272 -189
- package/lib/containment.ts +238 -0
- package/lib/content-hash.ts +24 -0
- package/lib/context-docs.ts +143 -28
- package/lib/create-brief.ts +2 -1
- package/lib/create-sc.ts +323 -4
- package/lib/dev-liveness.ts +180 -0
- package/lib/discovery-branch-points.ts +432 -0
- package/lib/discovery-stability.ts +481 -0
- package/lib/distribution-currency.ts +186 -0
- package/lib/doc-routing.ts +157 -0
- package/lib/doc-scan-targets.ts +9 -0
- package/lib/evidence-command-transport.ts +160 -0
- package/lib/evidence-prevalidator.ts +246 -2
- package/lib/evidence-satisfiability.ts +534 -12
- package/lib/failure-intake.ts +317 -0
- package/lib/fallow-availability.ts +286 -0
- package/lib/fs-write-capabilities.ts +232 -0
- package/lib/gate-refusal-currency.ts +112 -0
- package/lib/gate-run-split.ts +147 -0
- package/lib/gate-timeouts.ts +82 -0
- package/lib/generators/agent-briefs.ts +4 -1
- package/lib/generators/agents-md.ts +204 -48
- package/lib/generators/code-map.ts +2 -1
- package/lib/generators/types.ts +8 -0
- package/lib/git-dates.ts +55 -0
- package/lib/git-isolation.ts +140 -0
- package/lib/github.ts +15 -0
- package/lib/google-auth-action.ts +93 -0
- package/lib/harness-project.ts +48 -0
- package/lib/harness-provenance.ts +255 -0
- package/lib/hook-evidence.ts +235 -0
- package/lib/hook-scopes.ts +386 -0
- package/lib/hook-target-tree.ts +398 -0
- package/lib/hook-trees.ts +447 -0
- package/lib/js-top-level-code.ts +78 -10
- package/lib/junit-checks.ts +536 -0
- package/lib/launch-base.ts +253 -0
- package/lib/line-citation.ts +372 -0
- package/lib/local-calendar-day.ts +78 -0
- package/lib/orphan-reaper.ts +468 -0
- package/lib/parity-binding.ts +205 -0
- package/lib/per-test-timeouts.ts +219 -0
- package/lib/plan-coverage.ts +151 -0
- package/lib/prior-branch.ts +201 -8
- package/lib/process-table.ts +146 -0
- package/lib/project-state-derivation.ts +219 -0
- package/lib/project-state-root.ts +258 -0
- package/lib/project-state-shape.ts +59 -0
- package/lib/promote-outputs.ts +6 -2
- package/lib/published-artifact.ts +525 -0
- package/lib/readdir-sorted.ts +45 -0
- package/lib/report-provenance.ts +156 -0
- package/lib/review-execution-reading.ts +206 -0
- package/lib/review-tool-agreement.ts +433 -0
- package/lib/rule-promotion.ts +234 -0
- package/lib/rule-retirement.ts +525 -0
- package/lib/run-branch.ts +612 -0
- package/lib/rungate-schema.ts +19 -0
- package/lib/runner-home.ts +274 -0
- package/lib/sc-claim-forms.ts +75 -0
- package/lib/sc-id-cross-tree.ts +323 -0
- package/lib/sc-ids.ts +121 -13
- package/lib/sc-reservations.ts +289 -0
- package/lib/scaffold/ai-review-findings.ts +403 -0
- package/lib/scaffold/ai-review-step.ts +4777 -0
- package/lib/scaffold/managed-workflow.ts +587 -192
- package/lib/scaffold/owned-files.ts +186 -39
- package/lib/scaffold/pre-push-hook.ts +49 -5
- package/lib/scaffold/rehearsal.ts +205 -71
- package/lib/scaffold/steps.ts +1517 -293
- package/lib/scaffold/workflow-census.ts +217 -0
- package/lib/scaffold/workflow-drift.ts +217 -0
- package/lib/scaffold/write-steps.ts +10 -4
- package/lib/scan-budget.ts +247 -0
- package/lib/scanner.ts +80 -62
- package/lib/scratch-roots.ts +50 -5
- package/lib/screenshot-discovery.ts +121 -0
- package/lib/screenshot-upload.ts +41 -1
- package/lib/secret-patterns.ts +511 -8
- package/lib/ship-mode.ts +219 -0
- package/lib/source-slice.ts +294 -0
- package/lib/spec-listing.ts +148 -0
- package/lib/spec-requirements.ts +209 -0
- package/lib/stale-cleanup.ts +37 -3
- package/lib/suite-flake.ts +278 -0
- package/lib/suite-measurement.ts +23 -4
- package/lib/suite-run-reading.ts +411 -1
- package/lib/suite-timeout-margin.ts +193 -0
- package/lib/test-case-bodies.ts +883 -0
- package/lib/test-case-deletion.ts +653 -0
- package/lib/test-census.ts +97 -29
- package/lib/test-command-positions.ts +274 -0
- package/lib/test-command.ts +133 -0
- package/lib/test-suite-lock.ts +325 -26
- package/lib/tool-probe.ts +142 -0
- package/lib/top-level-cd.ts +393 -0
- package/lib/tracked-vs-ignored.ts +234 -0
- package/lib/tree-entries.ts +89 -0
- package/lib/ts-ast.ts +200 -0
- package/lib/validators/spec-validators.ts +3 -2
- package/lib/walk-test-files.ts +97 -0
- package/lib/workflow-actions.ts +263 -0
- package/lib/worktree-collect.ts +177 -5
- package/lib/worktree-repo.ts +36 -1
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +45 -0
- package/scripts/audit-specs.ts +247 -12
- package/scripts/audit-transcript.ts +2 -1
- package/scripts/convert-spec.ts +2 -1
- package/scripts/create-sc.ts +37 -7
- package/scripts/create-spec.ts +2 -1
- package/scripts/create-success.ts +2 -1
- package/scripts/extract-constraints.ts +197 -26
- package/scripts/generate-code-map.ts +62 -28
- package/scripts/scaffold-project.ts +12 -2
- package/scripts/split-spec.ts +2 -1
- package/scripts/sync-sc-status.ts +127 -14
- package/scripts/sync-spec-tests.ts +2 -1
|
@@ -0,0 +1,536 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every check a gate run actually executed, read off bun's JUnit report (#532).
|
|
3
|
+
*
|
|
4
|
+
* ## Why the console output cannot answer this
|
|
5
|
+
*
|
|
6
|
+
* `gates/gate-executor.ts`'s `parseTestResults` scans for `(pass)` and
|
|
7
|
+
* `(fail)` markers. Measured on bun 1.3.14, with stdout piped:
|
|
8
|
+
*
|
|
9
|
+
* ```
|
|
10
|
+
* $ bun test p.test.ts # 2 passing, 1 failing
|
|
11
|
+
* (fail) beta fails [0.18ms]
|
|
12
|
+
* 2 pass
|
|
13
|
+
* 1 fail
|
|
14
|
+
* ```
|
|
15
|
+
*
|
|
16
|
+
* **There are no `(pass)` lines.** The runner prints a marker for a failure
|
|
17
|
+
* and a count for everything else, so the `(pass)` branch of
|
|
18
|
+
* `parseTestResults` matches nothing on the runner this repo uses, and the
|
|
19
|
+
* `results` array it returns is a FAILURE list wearing a neutral name.
|
|
20
|
+
*
|
|
21
|
+
* That is fine for the failure ledger, which is all it was ever read for.
|
|
22
|
+
* It is fatal for #532: the retirement rule needs the names a later gate
|
|
23
|
+
* **ran and did not fail**, and a failure list cannot name a single one of
|
|
24
|
+
* them. Sourcing `checksRun` from `results` produced two entries out of a
|
|
25
|
+
* hundred and seventeen on a real scope run — a positive witness that
|
|
26
|
+
* witnesses nothing, which is this repo's central defect class
|
|
27
|
+
* (.claude/rules/checks-must-be-able-to-fail.md).
|
|
28
|
+
*
|
|
29
|
+
* ## Why the JUnit report can
|
|
30
|
+
*
|
|
31
|
+
* `--reporter=junit --reporter-outfile=<path>` writes every testcase, and it
|
|
32
|
+
* runs ALONGSIDE the console output rather than replacing it — one spawn
|
|
33
|
+
* still prints the `(fail)` markers the detail parser reads.
|
|
34
|
+
*
|
|
35
|
+
* ## The trap in it, and it is the dangerous direction
|
|
36
|
+
*
|
|
37
|
+
* A test excluded by `--test-name-pattern` is **still listed**, as a
|
|
38
|
+
* `<skipped />`. Measured, with `-t "keep me"` over a file of five tests:
|
|
39
|
+
*
|
|
40
|
+
* ```xml
|
|
41
|
+
* <testcase name="keep me one" classname="alpha group" ... />
|
|
42
|
+
* <testcase name="skip me two" classname="alpha group" ...><skipped /></testcase>
|
|
43
|
+
* ```
|
|
44
|
+
*
|
|
45
|
+
* The gate's whole pattern is an exclusion — the scope gate runs
|
|
46
|
+
* `schema|scope` over `gates/`, so most of the file is filtered out on every
|
|
47
|
+
* run. Counting a filtered test as "run" would retire a refusal the gate
|
|
48
|
+
* never re-measured, which is worse than the bug #532 closes: it would erase
|
|
49
|
+
* a true refusal rather than keep a stale one.
|
|
50
|
+
*
|
|
51
|
+
* So a `<skipped />` testcase is NOT a check that ran, and the count of them
|
|
52
|
+
* is returned rather than discarded, because "ran 2, skipped 115" and
|
|
53
|
+
* "ran 117" are different facts about the same report.
|
|
54
|
+
*
|
|
55
|
+
* ## Why the names are rebuilt from the element nesting
|
|
56
|
+
*
|
|
57
|
+
* `classname` looks like the obvious source and is not usable: bun emits the
|
|
58
|
+
* describe chain REVERSED and double-escaped. Measured on a
|
|
59
|
+
* `describe("outer") > describe("inner")`, whose console name is
|
|
60
|
+
* `outer > inner > deep fails`:
|
|
61
|
+
*
|
|
62
|
+
* ```xml
|
|
63
|
+
* <testcase name="deep fails" classname="inner &gt; outer" ... />
|
|
64
|
+
* ```
|
|
65
|
+
*
|
|
66
|
+
* The nested `<testsuite>` elements carry the same chain in the right order,
|
|
67
|
+
* so the name is rebuilt from the stack. The outermost suite is the FILE —
|
|
68
|
+
* its `name` is a path — and it is dropped, which is what makes the rebuilt
|
|
69
|
+
* string match the `(fail)` marker the console printed for the same test.
|
|
70
|
+
* That match is load-bearing: `checksRun` has to be a superset of
|
|
71
|
+
* `failedChecks` or the subtraction in `lib/gate-refusal-currency.ts` reports
|
|
72
|
+
* a still-failing check as resolved.
|
|
73
|
+
*/
|
|
74
|
+
import { readFileSync } from "fs";
|
|
75
|
+
|
|
76
|
+
/** What one JUnit report says about which checks ran. */
|
|
77
|
+
export interface JunitReading {
|
|
78
|
+
/**
|
|
79
|
+
* The full names of the checks that RAN — passed or failed. `null` when
|
|
80
|
+
* the report could not be read, which is a different answer from `[]`
|
|
81
|
+
* ("the report says nothing ran") and has to stay different all the way
|
|
82
|
+
* to the consumer (.claude/rules/config-access.md, Silent Default
|
|
83
|
+
* Substitution).
|
|
84
|
+
*/
|
|
85
|
+
names: string[] | null;
|
|
86
|
+
/** Why `names` is null, or why the reading is partial. */
|
|
87
|
+
reason?: string;
|
|
88
|
+
/** Testcases listed but not run — filtered out, `.skip`, or `.todo`. */
|
|
89
|
+
skipped: number;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const ENTITIES: Array<[RegExp, string]> = [
|
|
93
|
+
[/</g, "<"],
|
|
94
|
+
[/>/g, ">"],
|
|
95
|
+
[/"/g, '"'],
|
|
96
|
+
[/'/g, "'"],
|
|
97
|
+
// Last, so `&gt;` decodes to `>` and not to `>`.
|
|
98
|
+
[/&/g, "&"],
|
|
99
|
+
];
|
|
100
|
+
|
|
101
|
+
function unescapeXml(s: string): string {
|
|
102
|
+
return ENTITIES.reduce((acc, [re, to]) => acc.replace(re, to), s);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function attr(tag: string, name: string): string | null {
|
|
106
|
+
const m = tag.match(new RegExp(`\\b${name}="([^"]*)"`));
|
|
107
|
+
return m ? unescapeXml(m[1]) : null;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** One testcase, as the report describes it. */
|
|
111
|
+
interface ScannedTestcase {
|
|
112
|
+
/** The full name, rebuilt from the `<testsuite>` nesting. */
|
|
113
|
+
name: string;
|
|
114
|
+
/** The `time` attribute in MILLISECONDS, or `NaN` when it carries none. */
|
|
115
|
+
ms: number;
|
|
116
|
+
/** False for a `<skipped />` testcase — filtered out, `.skip` or `.todo`. */
|
|
117
|
+
ran: boolean;
|
|
118
|
+
/**
|
|
119
|
+
* The `type` of the testcase's `<failure>`, or null if it did not fail.
|
|
120
|
+
* bun writes `TimeoutError` for a test the runner killed and
|
|
121
|
+
* `AssertionError` for one that failed on its merits — the distinction
|
|
122
|
+
* #621 is entirely about, and the only place in the run where it survives
|
|
123
|
+
* in machine-readable form.
|
|
124
|
+
*/
|
|
125
|
+
failure: string | null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const NOT_A_REPORT = "no <testsuites> root — this is not a bun JUnit report";
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Every testcase in the report, in document order, or `null` when the input
|
|
132
|
+
* is not a bun JUnit report.
|
|
133
|
+
*
|
|
134
|
+
* Written as a tag scanner rather than with an XML library because the only
|
|
135
|
+
* structure it needs is the `<testsuite>` nesting depth, and a dependency
|
|
136
|
+
* that parses the whole document would be a larger surface for a smaller
|
|
137
|
+
* answer.
|
|
138
|
+
*
|
|
139
|
+
* ONE scanner, two public readings. The names (#532) and the durations
|
|
140
|
+
* (#621) are different questions about the same elements, and a second
|
|
141
|
+
* scanner would be a second place for "a `<skipped />` testcase did not run"
|
|
142
|
+
* to be decided — so a mutation of either would leave the other holding the
|
|
143
|
+
* answer up (.claude/rules/checks-must-be-able-to-fail.md, the fourth shape).
|
|
144
|
+
*/
|
|
145
|
+
function scanJunitTestcases(xml: string): { cases: ScannedTestcase[]; skipped: number } | null {
|
|
146
|
+
if (!xml.includes("<testsuites")) return null;
|
|
147
|
+
|
|
148
|
+
const cases: ScannedTestcase[] = [];
|
|
149
|
+
const suites: string[] = [];
|
|
150
|
+
let skipped = 0;
|
|
151
|
+
|
|
152
|
+
// `pending` is non-null between a `<testcase>` open tag and its close.
|
|
153
|
+
let pending: ScannedTestcase | null = null;
|
|
154
|
+
|
|
155
|
+
const TOKEN = /<(\/?)(testsuite|testcase|skipped|failure|error)\b([^>]*?)(\/?)>/g;
|
|
156
|
+
for (const m of xml.matchAll(TOKEN)) {
|
|
157
|
+
const [, closing, tag, rest, selfClosing] = m;
|
|
158
|
+
if (tag === "testsuite") {
|
|
159
|
+
if (closing) {
|
|
160
|
+
suites.pop();
|
|
161
|
+
} else if (!selfClosing) {
|
|
162
|
+
suites.push(attr(rest, "name") ?? "");
|
|
163
|
+
}
|
|
164
|
+
continue;
|
|
165
|
+
}
|
|
166
|
+
if (tag === "testcase") {
|
|
167
|
+
if (closing) {
|
|
168
|
+
if (pending !== null) cases.push(pending);
|
|
169
|
+
pending = null;
|
|
170
|
+
continue;
|
|
171
|
+
}
|
|
172
|
+
// The first suite on the stack is the FILE, whose name is a path.
|
|
173
|
+
// Everything above it is a `describe`.
|
|
174
|
+
const testcase: ScannedTestcase = {
|
|
175
|
+
name: [...suites.slice(1), attr(rest, "name") ?? ""].join(" > "),
|
|
176
|
+
ms: durationMs(rest),
|
|
177
|
+
ran: true,
|
|
178
|
+
failure: null,
|
|
179
|
+
};
|
|
180
|
+
if (selfClosing) {
|
|
181
|
+
cases.push(testcase);
|
|
182
|
+
} else {
|
|
183
|
+
pending = testcase;
|
|
184
|
+
}
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
if (tag === "skipped" && !closing && pending !== null) {
|
|
188
|
+
pending.ran = false;
|
|
189
|
+
skipped++;
|
|
190
|
+
}
|
|
191
|
+
if (tag === "failure" && !closing && pending !== null) {
|
|
192
|
+
pending.failure = attr(rest, "type") ?? "";
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
return { cases, skipped };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* The `time` attribute in milliseconds.
|
|
201
|
+
*
|
|
202
|
+
* bun writes it in fractional SECONDS — `time="1.201579"` for a test the
|
|
203
|
+
* console printed as `[1201.58ms]`. Reading it as milliseconds would put
|
|
204
|
+
* every duration a thousand times below any budget it is compared against,
|
|
205
|
+
* and a proximity check over it would be green for every suite that has ever
|
|
206
|
+
* run (#621).
|
|
207
|
+
*/
|
|
208
|
+
function durationMs(tag: string): number {
|
|
209
|
+
const raw = attr(tag, "time");
|
|
210
|
+
if (raw === null || raw.trim() === "") return NaN;
|
|
211
|
+
const seconds = Number(raw);
|
|
212
|
+
return Number.isFinite(seconds) ? seconds * 1000 : NaN;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** The names of every testcase the report says RAN, in document order. */
|
|
216
|
+
export function parseJunitCheckNames(xml: string): JunitReading {
|
|
217
|
+
const scan = scanJunitTestcases(xml);
|
|
218
|
+
if (scan === null) return { names: null, skipped: 0, reason: NOT_A_REPORT };
|
|
219
|
+
return { names: scan.cases.filter(c => c.ran).map(c => c.name), skipped: scan.skipped };
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** `parseJunitCheckNames` over a file, with the read failure as a reading. */
|
|
223
|
+
export function readJunitCheckNames(path: string): JunitReading {
|
|
224
|
+
let xml: string;
|
|
225
|
+
try {
|
|
226
|
+
xml = readFileSync(path, "utf-8");
|
|
227
|
+
} catch (e: any) {
|
|
228
|
+
// The name, not the message: a reader error quotes the path and this
|
|
229
|
+
// string is written into workflow-state.json.
|
|
230
|
+
return { names: null, skipped: 0, reason: `the JUnit report could not be read (${e?.code || e?.name || "unknown"})` };
|
|
231
|
+
}
|
|
232
|
+
return parseJunitCheckNames(xml);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** What one JUnit report says about which checks PASSED (#659). */
|
|
236
|
+
export interface JunitOutcomeReading {
|
|
237
|
+
/**
|
|
238
|
+
* Full names of the testcases that ran and did not fail. `null` when the
|
|
239
|
+
* report could not be read — a different answer from `[]`, which says the
|
|
240
|
+
* report ran nothing, and the two must stay different all the way to the
|
|
241
|
+
* consumer.
|
|
242
|
+
*/
|
|
243
|
+
passed: string[] | null;
|
|
244
|
+
/** Full names of the testcases that ran and failed. Empty when unreadable. */
|
|
245
|
+
failed: string[];
|
|
246
|
+
/** Why `passed` is null. */
|
|
247
|
+
reason?: string;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Pass and fail, off the SAME scan the names reading uses (#659).
|
|
252
|
+
*
|
|
253
|
+
* `lib/test-case-deletion.ts` needs to prove a deleted case was GREEN before
|
|
254
|
+
* it was deleted, and `parseJunitCheckNames` answers "did it run", which is
|
|
255
|
+
* satisfied by a case that ran and failed. A second scanner for the second
|
|
256
|
+
* question would be two deciders for one document: both would agree on every
|
|
257
|
+
* report anyone has, and a mutation of either would leave the other holding
|
|
258
|
+
* the answer up (`.claude/rules/checks-must-be-able-to-fail.md`, the fourth
|
|
259
|
+
* shape). So the answer comes out of `scanJunitTestcases`, which already
|
|
260
|
+
* records each testcase's `<failure>`.
|
|
261
|
+
*
|
|
262
|
+
* A `<skipped />` testcase is in NEITHER list. It did not run, so it did not
|
|
263
|
+
* pass — and reading a filtered-out case as green is exactly the trap #532
|
|
264
|
+
* records about this report format.
|
|
265
|
+
*/
|
|
266
|
+
export function parseJunitOutcomes(xml: string): JunitOutcomeReading {
|
|
267
|
+
const scan = scanJunitTestcases(xml);
|
|
268
|
+
if (scan === null) return { passed: null, failed: [], reason: NOT_A_REPORT };
|
|
269
|
+
const ran = scan.cases.filter(c => c.ran);
|
|
270
|
+
return {
|
|
271
|
+
passed: ran.filter(c => c.failure === null).map(c => c.name),
|
|
272
|
+
failed: ran.filter(c => c.failure !== null).map(c => c.name),
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** `parseJunitOutcomes` over a file, with the read failure as a reading. */
|
|
277
|
+
export function readJunitOutcomes(path: string): JunitOutcomeReading {
|
|
278
|
+
let xml: string;
|
|
279
|
+
try {
|
|
280
|
+
xml = readFileSync(path, "utf-8");
|
|
281
|
+
} catch (e: any) {
|
|
282
|
+
return { passed: null, failed: [], reason: `the JUnit report could not be read (${e?.code || e?.name || "unknown"})` };
|
|
283
|
+
}
|
|
284
|
+
return parseJunitOutcomes(xml);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* The full witness: every check the gate reported a verdict for.
|
|
289
|
+
*
|
|
290
|
+
* The JUnit report covers the bun tests. `reported` covers everything else
|
|
291
|
+
* the gate decided on its own — `test-census: UNMEASURED`,
|
|
292
|
+
* `ac-verdict-fail: AC-2`, the dev-server liveness probe — which are real
|
|
293
|
+
* verdicts with no testcase behind them, and which appear in the failure
|
|
294
|
+
* ledger. Leaving them out would break the superset property in exactly the
|
|
295
|
+
* places where the names are not bun's.
|
|
296
|
+
*
|
|
297
|
+
* `null` in, `null` out. A gate whose report could not be read says nothing
|
|
298
|
+
* about what it ran, and must retire nothing.
|
|
299
|
+
*/
|
|
300
|
+
export function checksRunFrom(reading: JunitReading, reported: string[]): string[] | null {
|
|
301
|
+
return unionNames(reading.names, reported);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
/**
|
|
305
|
+
* `base` with `extra` appended, de-duplicated, order preserved. `null`
|
|
306
|
+
* propagates.
|
|
307
|
+
*
|
|
308
|
+
* Exported because the union has to be applied twice: once at the gate's
|
|
309
|
+
* call site, and again inside `writeGateResult`, which pushes
|
|
310
|
+
* `ac-verdict-fail:` entries of its own AFTER the caller has read the
|
|
311
|
+
* results. A second union there is not a second decider — it is the same
|
|
312
|
+
* function over a later reading of the same list, and skipping it would
|
|
313
|
+
* leave `checksRun` short of `failures` by exactly the entries that gate
|
|
314
|
+
* adds last.
|
|
315
|
+
*/
|
|
316
|
+
export function unionNames(base: string[] | null, extra: string[]): string[] | null {
|
|
317
|
+
if (base === null) return null;
|
|
318
|
+
const seen = new Set<string>();
|
|
319
|
+
const out: string[] = [];
|
|
320
|
+
for (const name of [...base, ...extra]) {
|
|
321
|
+
if (typeof name !== "string" || name === "" || seen.has(name)) continue;
|
|
322
|
+
seen.add(name);
|
|
323
|
+
out.push(name);
|
|
324
|
+
}
|
|
325
|
+
return out;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/* ───────────────────────────────────────────────────────────────────────────
|
|
329
|
+
* #621: how close the suite ran to the per-test timeout
|
|
330
|
+
*
|
|
331
|
+
* The same report, asked a different question. #621's 122 "failures" were
|
|
332
|
+
* 122 timeouts: every one at or above bun's 5000ms per-test budget, 112
|
|
333
|
+
* carrying `timed out after 5000ms`, and not one an assertion failure. A
|
|
334
|
+
* verdict like that is a fact about machine load, not about the code.
|
|
335
|
+
*
|
|
336
|
+
* The warning is in the durations. A test at 99% of the budget passes today
|
|
337
|
+
* and is killed the first time it runs beside 27 agents, and the console
|
|
338
|
+
* output says nothing about it — only the JUnit report carries a number for
|
|
339
|
+
* every testcase.
|
|
340
|
+
*
|
|
341
|
+
* TWO READINGS, NOT ONE VERDICT. `durationCoverageViolations` answers "did
|
|
342
|
+
* this reading read anything?" and `timeoutProximityViolations` answers "is
|
|
343
|
+
* anything it read near the budget?". An empty report satisfies the second
|
|
344
|
+
* perfectly, which is why the first exists and why the proximity check stays
|
|
345
|
+
* SILENT on an empty reading instead of refusing it: if both refused, they
|
|
346
|
+
* would be one question with two names, and nothing could tell a report that
|
|
347
|
+
* is clean from a reader that stopped reading (AC-7).
|
|
348
|
+
* ──────────────────────────────────────────────────────────────────────── */
|
|
349
|
+
|
|
350
|
+
/** One testcase's measured duration. */
|
|
351
|
+
export interface TestcaseDuration {
|
|
352
|
+
/** The full name, as the console prints it. */
|
|
353
|
+
name: string;
|
|
354
|
+
/** Milliseconds. */
|
|
355
|
+
ms: number;
|
|
356
|
+
/** The runner killed it: bun wrote `<failure type="TimeoutError" />`. */
|
|
357
|
+
timedOut: boolean;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/** What one JUnit report says about how long its testcases took. */
|
|
361
|
+
export interface JunitDurationReading {
|
|
362
|
+
/**
|
|
363
|
+
* Every testcase that RAN and carried a readable duration. `null` when the
|
|
364
|
+
* report could not be read at all — a different answer from `[]` ("the
|
|
365
|
+
* report lists nothing"), and the two have to stay different all the way to
|
|
366
|
+
* the consumer (.claude/rules/config-access.md, Silent Default
|
|
367
|
+
* Substitution).
|
|
368
|
+
*/
|
|
369
|
+
durations: TestcaseDuration[] | null;
|
|
370
|
+
/** How many durations were read. The reading's own coverage (AC-7a). */
|
|
371
|
+
count: number;
|
|
372
|
+
/** Testcases listed but not run — filtered out, `.skip`, or `.todo`. */
|
|
373
|
+
skipped: number;
|
|
374
|
+
/**
|
|
375
|
+
* Testcases that ran and carried no readable `time`. Counted rather than
|
|
376
|
+
* dropped: a reader that silently shrinks its own population reports a
|
|
377
|
+
* clean proximity check over the half it managed to parse.
|
|
378
|
+
*/
|
|
379
|
+
withoutDuration: number;
|
|
380
|
+
/** Why `durations` is null. */
|
|
381
|
+
reason?: string;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** Every testcase duration the report carries, in document order. */
|
|
385
|
+
export function parseJunitDurations(xml: string): JunitDurationReading {
|
|
386
|
+
const scan = scanJunitTestcases(xml);
|
|
387
|
+
if (scan === null) {
|
|
388
|
+
return { durations: null, count: 0, skipped: 0, withoutDuration: 0, reason: NOT_A_REPORT };
|
|
389
|
+
}
|
|
390
|
+
const ran = scan.cases.filter(c => c.ran);
|
|
391
|
+
const durations = ran
|
|
392
|
+
.filter(c => Number.isFinite(c.ms))
|
|
393
|
+
.map(c => ({ name: c.name, ms: c.ms, timedOut: c.failure === "TimeoutError" }));
|
|
394
|
+
return {
|
|
395
|
+
durations,
|
|
396
|
+
count: durations.length,
|
|
397
|
+
skipped: scan.skipped,
|
|
398
|
+
withoutDuration: ran.length - durations.length,
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** `parseJunitDurations` over a file, with the read failure as a reading. */
|
|
403
|
+
export function readJunitDurations(path: string): JunitDurationReading {
|
|
404
|
+
let xml: string;
|
|
405
|
+
try {
|
|
406
|
+
xml = readFileSync(path, "utf-8");
|
|
407
|
+
} catch (e: any) {
|
|
408
|
+
return {
|
|
409
|
+
durations: null,
|
|
410
|
+
count: 0,
|
|
411
|
+
skipped: 0,
|
|
412
|
+
withoutDuration: 0,
|
|
413
|
+
// The name, not the message: a reader error quotes the path.
|
|
414
|
+
reason: `the JUnit report could not be read (${e?.code || e?.name || "unknown"})`,
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
return parseJunitDurations(xml);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/**
|
|
421
|
+
* The per-test timeout a `bunfig.toml` declares, or `null` if it declares
|
|
422
|
+
* none.
|
|
423
|
+
*
|
|
424
|
+
* Scoped to the `[test]` table and nothing else. A `timeout` under
|
|
425
|
+
* `[install]` is a different setting entirely, and a reader that matched the
|
|
426
|
+
* key anywhere in the document would measure the suite against a number
|
|
427
|
+
* about package downloads — searching the document instead of the slice is
|
|
428
|
+
* how three checks in this repo went green over text they were not about
|
|
429
|
+
* (.claude/rules/checks-must-be-able-to-fail.md).
|
|
430
|
+
*/
|
|
431
|
+
export function bunfigTestTimeoutMs(toml: string): number | null {
|
|
432
|
+
const lines = toml.split("\n");
|
|
433
|
+
const start = lines.findIndex(line => /^\s*\[test\]\s*(#.*)?$/.test(line));
|
|
434
|
+
if (start === -1) return null;
|
|
435
|
+
const rest = lines.slice(start + 1);
|
|
436
|
+
const end = rest.findIndex(line => /^\s*\[/.test(line));
|
|
437
|
+
const section = (end === -1 ? rest : rest.slice(0, end)).join("\n");
|
|
438
|
+
const m = /^\s*timeout\s*=\s*(\d+)\s*(#.*)?$/m.exec(section);
|
|
439
|
+
return m ? Number(m[1]) : null;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/**
|
|
443
|
+
* What is wrong with the READING, before anything is concluded from it
|
|
444
|
+
* (AC-7a).
|
|
445
|
+
*
|
|
446
|
+
* A scan that reports a verdict should report its own coverage: "0 of 0 are
|
|
447
|
+
* near the budget" and "0 of 13358 are near the budget" are the same PASS
|
|
448
|
+
* today, and no reader can tell them apart from the output.
|
|
449
|
+
*/
|
|
450
|
+
export function durationCoverageViolations(reading: JunitDurationReading, source: string): string[] {
|
|
451
|
+
if (reading.durations === null) {
|
|
452
|
+
return [
|
|
453
|
+
`the JUnit report at ${source} could not be read (${reading.reason ?? "no reason recorded"}), ` +
|
|
454
|
+
`so nothing is known about how long anything took (#621)`,
|
|
455
|
+
];
|
|
456
|
+
}
|
|
457
|
+
const out: string[] = [];
|
|
458
|
+
if (reading.count === 0) {
|
|
459
|
+
out.push(
|
|
460
|
+
`the JUnit report at ${source} carries no testcase durations (0 read, ${reading.skipped} skipped) — ` +
|
|
461
|
+
`an empty report satisfies a proximity check by having nothing in it, which is not a measurement (#621)`,
|
|
462
|
+
);
|
|
463
|
+
}
|
|
464
|
+
if (reading.withoutDuration > 0) {
|
|
465
|
+
out.push(
|
|
466
|
+
`${reading.withoutDuration} of ${reading.withoutDuration + reading.count} testcases that ran in ${source} ` +
|
|
467
|
+
`carry no readable time attribute — the reading is partial, and the missing ones are exactly the ` +
|
|
468
|
+
`population a proximity check cannot see (#621)`,
|
|
469
|
+
);
|
|
470
|
+
}
|
|
471
|
+
return out;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* Every testcase the per-test budget is about to kill, or already killed —
|
|
476
|
+
* named, with its duration (AC-3b).
|
|
477
|
+
*
|
|
478
|
+
* ## Three classes, and only two of them are findings
|
|
479
|
+
*
|
|
480
|
+
* | What the report shows | What it means | Reported |
|
|
481
|
+
* |---|---|---|
|
|
482
|
+
* | `<failure type="TimeoutError" />` | the runner killed it | **yes** — this is #621's 122 |
|
|
483
|
+
* | ran, and finished inside the band `[timeout·(1−f), timeout)` | it is next | **yes** — the warning |
|
|
484
|
+
* | ran, and finished at or past `timeout` without timing out | it was given a BIGGER budget, or it could not have got there | no |
|
|
485
|
+
*
|
|
486
|
+
* The third row is the one that keeps this check usable. A test carrying
|
|
487
|
+
* `}, 240000)` — `gates/workflow.test.ts`'s typecheck check, measured at
|
|
488
|
+
* 16770ms — is not at risk from a 5000ms default, and reporting it would
|
|
489
|
+
* make the check red on every run forever. A gate that cries wolf trains its
|
|
490
|
+
* reader to ignore it, which is the cost #621 is filed about; so the rule is
|
|
491
|
+
* that **a duration past the budget is itself the evidence that the budget
|
|
492
|
+
* is not this test's**.
|
|
493
|
+
*
|
|
494
|
+
* It follows that declaring a per-test timeout silences a row here, and that
|
|
495
|
+
* is deliberate: an 8-second test that says it needs 8 seconds is a decision
|
|
496
|
+
* somebody made, and an 8-second test that nobody budgeted for is a timeout
|
|
497
|
+
* waiting for a loaded machine. Raising the FILE-wide default to silence a
|
|
498
|
+
* row is the repair #621 refuses, and it is refused where the default is
|
|
499
|
+
* declared rather than here.
|
|
500
|
+
*
|
|
501
|
+
* SILENT on a reading of nothing, deliberately — see the header above.
|
|
502
|
+
* `durationCoverageViolations` is the question about an empty reading, and
|
|
503
|
+
* keeping them apart is what makes a red row say which one broke.
|
|
504
|
+
*/
|
|
505
|
+
export function timeoutProximityViolations(
|
|
506
|
+
reading: JunitDurationReading,
|
|
507
|
+
timeoutMs: number,
|
|
508
|
+
withinFraction = 0.01,
|
|
509
|
+
): string[] {
|
|
510
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
511
|
+
return [
|
|
512
|
+
`no usable per-test timeout to compare against (${timeoutMs}) — a budget of zero or NaN makes every ` +
|
|
513
|
+
`comparison below meaningless rather than clean (#621)`,
|
|
514
|
+
];
|
|
515
|
+
}
|
|
516
|
+
if (reading.durations === null) return [];
|
|
517
|
+
const floor = timeoutMs * (1 - withinFraction);
|
|
518
|
+
const out: string[] = [];
|
|
519
|
+
for (const d of reading.durations) {
|
|
520
|
+
if (d.timedOut) {
|
|
521
|
+
out.push(
|
|
522
|
+
`${d.name} was KILLED after ${d.ms.toFixed(2)}ms by a per-test timeout — it did not fail on its ` +
|
|
523
|
+
`merits, and on a less loaded machine it would not have failed at all (#621)`,
|
|
524
|
+
);
|
|
525
|
+
continue;
|
|
526
|
+
}
|
|
527
|
+
if (d.ms >= floor && d.ms < timeoutMs) {
|
|
528
|
+
out.push(
|
|
529
|
+
`${d.name} took ${d.ms.toFixed(2)}ms, ${((d.ms / timeoutMs) * 100).toFixed(1)}% of the ${timeoutMs}ms ` +
|
|
530
|
+
`per-test timeout — within ${withinFraction * 100}% of being killed instead of failing on its merits. ` +
|
|
531
|
+
`Make it faster, or give it a budget of its own with a per-test timeout argument (#621)`,
|
|
532
|
+
);
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
return out;
|
|
536
|
+
}
|