rungate 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (142) hide show
  1. package/lib/affected-tests.ts +398 -0
  2. package/lib/agent-audit.ts +2 -3
  3. package/lib/agent-worktree.ts +287 -0
  4. package/lib/agents-md-sections.ts +209 -0
  5. package/lib/behavioral-cache.ts +150 -2
  6. package/lib/branch-diff.ts +139 -0
  7. package/lib/branch-protection.ts +790 -0
  8. package/lib/brief-ac-alignment.ts +104 -0
  9. package/lib/brief-references.ts +205 -0
  10. package/lib/bun-version.ts +347 -0
  11. package/lib/check-remedy.ts +140 -0
  12. package/lib/ci-checkout.ts +18 -1
  13. package/lib/ci-history-depth.ts +159 -0
  14. package/lib/ci-setup-step.ts +96 -0
  15. package/lib/claude-code-action.ts +85 -0
  16. package/lib/claude-tool-vocabulary.ts +188 -0
  17. package/lib/cli-commands.ts +15 -0
  18. package/lib/commit-sha.ts +42 -0
  19. package/lib/commit-subject.ts +90 -0
  20. package/lib/compliance-report.ts +377 -45
  21. package/lib/compliance.ts +196 -20
  22. package/lib/config-loader.ts +51 -1
  23. package/lib/config-trackable.ts +391 -0
  24. package/lib/conformity.ts +1272 -189
  25. package/lib/containment.ts +238 -0
  26. package/lib/content-hash.ts +24 -0
  27. package/lib/context-docs.ts +143 -28
  28. package/lib/create-brief.ts +2 -1
  29. package/lib/create-sc.ts +323 -4
  30. package/lib/dev-liveness.ts +180 -0
  31. package/lib/discovery-branch-points.ts +432 -0
  32. package/lib/discovery-stability.ts +481 -0
  33. package/lib/distribution-currency.ts +186 -0
  34. package/lib/doc-routing.ts +157 -0
  35. package/lib/doc-scan-targets.ts +9 -0
  36. package/lib/evidence-command-transport.ts +160 -0
  37. package/lib/evidence-prevalidator.ts +246 -2
  38. package/lib/evidence-satisfiability.ts +534 -12
  39. package/lib/failure-intake.ts +317 -0
  40. package/lib/fallow-availability.ts +286 -0
  41. package/lib/fs-write-capabilities.ts +232 -0
  42. package/lib/gate-refusal-currency.ts +112 -0
  43. package/lib/gate-run-split.ts +147 -0
  44. package/lib/gate-timeouts.ts +82 -0
  45. package/lib/generators/agent-briefs.ts +4 -1
  46. package/lib/generators/agents-md.ts +204 -48
  47. package/lib/generators/code-map.ts +2 -1
  48. package/lib/generators/types.ts +8 -0
  49. package/lib/git-dates.ts +55 -0
  50. package/lib/git-isolation.ts +140 -0
  51. package/lib/github.ts +15 -0
  52. package/lib/google-auth-action.ts +93 -0
  53. package/lib/harness-project.ts +48 -0
  54. package/lib/harness-provenance.ts +255 -0
  55. package/lib/hook-evidence.ts +235 -0
  56. package/lib/hook-scopes.ts +386 -0
  57. package/lib/hook-target-tree.ts +398 -0
  58. package/lib/hook-trees.ts +447 -0
  59. package/lib/js-top-level-code.ts +78 -10
  60. package/lib/junit-checks.ts +536 -0
  61. package/lib/launch-base.ts +253 -0
  62. package/lib/line-citation.ts +372 -0
  63. package/lib/local-calendar-day.ts +78 -0
  64. package/lib/orphan-reaper.ts +468 -0
  65. package/lib/parity-binding.ts +205 -0
  66. package/lib/per-test-timeouts.ts +219 -0
  67. package/lib/plan-coverage.ts +151 -0
  68. package/lib/prior-branch.ts +201 -8
  69. package/lib/process-table.ts +146 -0
  70. package/lib/project-state-derivation.ts +219 -0
  71. package/lib/project-state-root.ts +258 -0
  72. package/lib/project-state-shape.ts +59 -0
  73. package/lib/promote-outputs.ts +6 -2
  74. package/lib/published-artifact.ts +525 -0
  75. package/lib/readdir-sorted.ts +45 -0
  76. package/lib/report-provenance.ts +156 -0
  77. package/lib/review-execution-reading.ts +206 -0
  78. package/lib/review-tool-agreement.ts +433 -0
  79. package/lib/rule-promotion.ts +234 -0
  80. package/lib/rule-retirement.ts +525 -0
  81. package/lib/run-branch.ts +612 -0
  82. package/lib/rungate-schema.ts +19 -0
  83. package/lib/runner-home.ts +274 -0
  84. package/lib/sc-claim-forms.ts +75 -0
  85. package/lib/sc-id-cross-tree.ts +323 -0
  86. package/lib/sc-ids.ts +121 -13
  87. package/lib/sc-reservations.ts +289 -0
  88. package/lib/scaffold/ai-review-findings.ts +403 -0
  89. package/lib/scaffold/ai-review-step.ts +4777 -0
  90. package/lib/scaffold/managed-workflow.ts +587 -192
  91. package/lib/scaffold/owned-files.ts +186 -39
  92. package/lib/scaffold/pre-push-hook.ts +49 -5
  93. package/lib/scaffold/rehearsal.ts +205 -71
  94. package/lib/scaffold/steps.ts +1517 -293
  95. package/lib/scaffold/workflow-census.ts +217 -0
  96. package/lib/scaffold/workflow-drift.ts +217 -0
  97. package/lib/scaffold/write-steps.ts +10 -4
  98. package/lib/scan-budget.ts +247 -0
  99. package/lib/scanner.ts +80 -62
  100. package/lib/scratch-roots.ts +50 -5
  101. package/lib/screenshot-discovery.ts +121 -0
  102. package/lib/screenshot-upload.ts +41 -1
  103. package/lib/secret-patterns.ts +511 -8
  104. package/lib/ship-mode.ts +219 -0
  105. package/lib/source-slice.ts +294 -0
  106. package/lib/spec-listing.ts +148 -0
  107. package/lib/spec-requirements.ts +209 -0
  108. package/lib/stale-cleanup.ts +37 -3
  109. package/lib/suite-flake.ts +278 -0
  110. package/lib/suite-measurement.ts +23 -4
  111. package/lib/suite-run-reading.ts +411 -1
  112. package/lib/suite-timeout-margin.ts +193 -0
  113. package/lib/test-case-bodies.ts +883 -0
  114. package/lib/test-case-deletion.ts +653 -0
  115. package/lib/test-census.ts +97 -29
  116. package/lib/test-command-positions.ts +274 -0
  117. package/lib/test-command.ts +133 -0
  118. package/lib/test-suite-lock.ts +325 -26
  119. package/lib/tool-probe.ts +142 -0
  120. package/lib/top-level-cd.ts +393 -0
  121. package/lib/tracked-vs-ignored.ts +234 -0
  122. package/lib/tree-entries.ts +89 -0
  123. package/lib/ts-ast.ts +200 -0
  124. package/lib/validators/spec-validators.ts +3 -2
  125. package/lib/walk-test-files.ts +97 -0
  126. package/lib/workflow-actions.ts +263 -0
  127. package/lib/worktree-collect.ts +177 -5
  128. package/lib/worktree-repo.ts +36 -1
  129. package/package.json +2 -1
  130. package/scripts/ai-review-findings.ts +45 -0
  131. package/scripts/audit-specs.ts +247 -12
  132. package/scripts/audit-transcript.ts +2 -1
  133. package/scripts/convert-spec.ts +2 -1
  134. package/scripts/create-sc.ts +37 -7
  135. package/scripts/create-spec.ts +2 -1
  136. package/scripts/create-success.ts +2 -1
  137. package/scripts/extract-constraints.ts +197 -26
  138. package/scripts/generate-code-map.ts +62 -28
  139. package/scripts/scaffold-project.ts +12 -2
  140. package/scripts/split-spec.ts +2 -1
  141. package/scripts/sync-sc-status.ts +127 -14
  142. package/scripts/sync-spec-tests.ts +2 -1
@@ -0,0 +1,209 @@
1
+ /**
2
+ * Does a document the routing table calls a spec actually state anything? (#579)
3
+ *
4
+ * `.claude/rules/specs-routing.md` is the table every agent is told to consult
5
+ * before changing an area. It is generated from `specs/`, one row per file,
6
+ * and it promises each row GOVERNS something. Nothing checked that the
7
+ * document on the other end states a requirement.
8
+ *
9
+ * `specs/PROJECT-STATE.md` was a **rendered PROJECT-STATE.md snapshot from
10
+ * 2026-09-22 with spec frontmatter pasted on top** — a "Current phase" line, a
11
+ * stale priorities list, and status tables citing criteria it did not declare.
12
+ * It sat in the routing table as the governing spec for project state
13
+ * management for two months. An agent sent there before touching
14
+ * `scripts/update-project-state.ts` got no criteria at all.
15
+ *
16
+ * **A gap in a routing table is visible; this is not.** The reader believes
17
+ * they have arrived. That is the same harm as `.claude/rules/config-access.md`'s
18
+ * Silent Default Substitution with a document as the victim: an empty answer
19
+ * and a correct answer are the same shape to the consumer.
20
+ *
21
+ * ## Why the exemption is frontmatter and not an allowlist
22
+ *
23
+ * Some documents in `specs/` legitimately state no criteria —
24
+ * `BOOTSTRAP-DATA-FLOW-SPEC.md` is a redirect stub whose whole job is to point
25
+ * readers at the directory it was split into. An allowlist would cover that,
26
+ * and two things are wrong with one here.
27
+ *
28
+ * DRIFT-3 refuses any entry ADDED to `.claude/conformity-allowlists.json`
29
+ * relative to `origin/main`, because an agent must not pass a gate by editing
30
+ * the gate. Shipping this check with its current violators pre-listed is
31
+ * exactly that move, and the ratchet is right to refuse it.
32
+ *
33
+ * The better reason is that **an allowlist is invisible to the person being
34
+ * harmed.** The reader harmed by a criteria-free spec is the one who followed
35
+ * the routing table to it; a JSON file in `.claude/` is not on that path. A
36
+ * declaration in the document's own frontmatter is, and the generated routing
37
+ * table can render it, so the warning reaches the reader before they click
38
+ * rather than after they have read 300 lines looking for requirements.
39
+ *
40
+ * So: state criteria, or say `criteria: none` and say why. Both are honest.
41
+ * Silence is the thing that is not.
42
+ */
43
+ import { existsSync, readFileSync, readdirSync, statSync } from "fs";
44
+ import { basename, join } from "path";
45
+
46
+ /** A checkbox criterion line, whatever id prefix it uses. */
47
+ const CRITERION_LINE = /^\s*- \[[ xX]\]/;
48
+
49
+ /**
50
+ * Counted on any checkbox line rather than on `- [ ] SC-`.
51
+ *
52
+ * `BOOTSTRAP-TEST-PLAN.md` numbers its criteria `TP-1`, `TP-2`, and the first
53
+ * census written for this issue counted `- [ ] SC-` and reported it as
54
+ * stating nothing. It states 41 criteria. A detector narrower than what it
55
+ * detects, in the direction that accuses a correct document —
56
+ * `.claude/rules/checks-must-be-able-to-fail.md`, second shape.
57
+ */
58
+ export function countCriteria(content: string): number {
59
+ return content.split("\n").filter(l => CRITERION_LINE.test(l)).length;
60
+ }
61
+
62
+ export interface SpecRequirementReading {
63
+ /** Path relative to `specs/`, so a report can name it. */
64
+ spec: string;
65
+ criteria: number;
66
+ /** The document says it states none, in its own frontmatter. */
67
+ declaredNone: boolean;
68
+ /** Why it states none. A declaration with no reason is a mute. */
69
+ reason: string | null;
70
+ }
71
+
72
+ function frontmatterField(content: string, key: string): string | null {
73
+ const fm = content.match(/^---\n([\s\S]*?)\n---/);
74
+ if (!fm) return null;
75
+ const m = fm[1].match(new RegExp(`^${key}:\\s*(.+)$`, "m"));
76
+ return m ? m[1].trim() : null;
77
+ }
78
+
79
+ export function readSpecRequirements(specsDir: string, rel: string): SpecRequirementReading {
80
+ const content = readFileSync(join(specsDir, rel), "utf-8");
81
+ const declared = frontmatterField(content, "criteria");
82
+ const reason = frontmatterField(content, "criteria-reason");
83
+ return {
84
+ spec: rel,
85
+ criteria: countCriteria(content),
86
+ declaredNone: declared === "none",
87
+ reason: reason && reason.length > 0 ? reason : null,
88
+ };
89
+ }
90
+
91
+ /**
92
+ * Refusals, worded after the property rather than after the proxy.
93
+ *
94
+ * Three of them, and the second and third exist because the first alone is
95
+ * satisfiable by something other than fixing the problem — #492's trap. A
96
+ * check that only refused "no criteria and no declaration" is cleared by
97
+ * pasting `criteria: none` onto a spec that has requirements and has simply
98
+ * not written them down, which is the state this check exists to surface.
99
+ */
100
+ export function requirementViolations(readings: SpecRequirementReading[]): string[] {
101
+ const out: string[] = [];
102
+ for (const r of readings) {
103
+ if (r.criteria === 0 && !r.declaredNone) {
104
+ out.push(
105
+ `${r.spec} states no criteria and does not say so. The routing table presents it as a ` +
106
+ `governing spec, so a reader is sent there expecting requirements. Add criteria, or ` +
107
+ `declare "criteria: none" with a "criteria-reason:" in its frontmatter.`,
108
+ );
109
+ continue;
110
+ }
111
+ // Both directions, like every other ratchet here. A spec that gains
112
+ // criteria must withdraw the declaration, or the declaration rots into a
113
+ // line that certifies nothing — and the routing table keeps warning
114
+ // readers away from a document that now has what they came for.
115
+ if (r.declaredNone && r.criteria > 0) {
116
+ out.push(
117
+ `${r.spec} declares "criteria: none" and states ${r.criteria}. Remove the declaration.`,
118
+ );
119
+ continue;
120
+ }
121
+ if (r.declaredNone && r.reason === null) {
122
+ out.push(
123
+ `${r.spec} declares "criteria: none" with no "criteria-reason:". A reader routed here ` +
124
+ `needs to know whether this is a redirect stub or an unwritten spec.`,
125
+ );
126
+ }
127
+ }
128
+ return out;
129
+ }
130
+
131
+ /**
132
+ * `testable:` read as a value rather than as `=== "true"`.
133
+ *
134
+ * DRIFT-2 and SPEC-DISCOVERY both ask `fm.testable === "true"`, so every
135
+ * other spelling is silently FALSE. `specs/DA-COMPLIANCE-SPEC.md` carries
136
+ * `testable: no`, which happens to mean what its author intended; `testable:
137
+ * yes` would mean the opposite of what its author intended and nothing would
138
+ * say so. The spec would drop out of the conformity surface while its
139
+ * frontmatter, the routing table and any reader all said it was in.
140
+ *
141
+ * A fail-open with no amplifier today is still a fail-open. One line.
142
+ */
143
+ export function testableViolations(specsDir: string, specs: string[]): string[] {
144
+ const out: string[] = [];
145
+ for (const rel of specs) {
146
+ const raw = frontmatterField(readFileSync(join(specsDir, rel), "utf-8"), "testable");
147
+ if (raw === null) continue; // absence is a separate check's question
148
+ if (raw !== "true" && raw !== "false") {
149
+ out.push(
150
+ `${rel} has "testable: ${raw}". Every reader compares against the string "true", so ` +
151
+ `anything else means false — including "yes". Write true or false.`,
152
+ );
153
+ }
154
+ }
155
+ return out;
156
+ }
157
+
158
+ /**
159
+ * Spec basenames that also name a file somewhere else in the project.
160
+ *
161
+ * DRIFT-2 credits a testable spec when any test file's text contains its
162
+ * name, and for a top-level spec that name is a bare basename. On this repo
163
+ * `specs/PROJECT-STATE.md` and the generated root `PROJECT-STATE.md` share
164
+ * one, and **ten test files mention the root one** — so the spec was
165
+ * referenced, by DRIFT-2's reading, by tests that have nothing to do with it.
166
+ *
167
+ * This is the fifth shape and it was aimed straight at #579's own fix: the
168
+ * issue's prescribed remedy is `testable: true` plus "a test naming it, which
169
+ * test/update-project-state.test.ts can do in one line". Flipping the flag
170
+ * alone would have turned DRIFT-2 green for this spec on the strength of
171
+ * `test/organize-project.test.ts` talking about a status file.
172
+ *
173
+ * Measured rather than assumed: exactly one spec basename collides on this
174
+ * tree. Requiring the `specs/`-qualified path from EVERY test would orphan 15
175
+ * of 22 testable specs, so the rule is narrowed to the ambiguous ones — where
176
+ * a bare name genuinely does not say which file is meant.
177
+ */
178
+ export function ambiguousSpecNames(root: string, specs: string[]): Set<string> {
179
+ const names = new Map<string, string>();
180
+ for (const rel of specs) names.set(basename(rel), rel);
181
+ const found = new Set<string>();
182
+ const SKIP = new Set(["node_modules", ".git", "specs", "dist", "build", ".fallow", ".next"]);
183
+ const walk = (dir: string): void => {
184
+ let entries: string[];
185
+ try { entries = readdirSync(dir); } catch { return; }
186
+ for (const n of entries) {
187
+ if (SKIP.has(n)) continue;
188
+ const p = join(dir, n);
189
+ let st;
190
+ try { st = statSync(p); } catch { continue; }
191
+ if (st.isDirectory()) walk(p);
192
+ else if (names.has(n)) found.add(names.get(n)!);
193
+ }
194
+ };
195
+ if (existsSync(root)) walk(root);
196
+ return found;
197
+ }
198
+
199
+ /**
200
+ * Whether a test names THIS spec.
201
+ *
202
+ * Unambiguous names are matched bare, which is how every existing credit in
203
+ * this repo is earned. An ambiguous one must be written `specs/<path>`, which
204
+ * is the only form that says which file is meant.
205
+ */
206
+ export function specIsReferenced(testContent: string, rel: string, ambiguous: Set<string>): boolean {
207
+ if (ambiguous.has(rel)) return testContent.includes(`specs/${rel}`);
208
+ return testContent.includes(rel);
209
+ }
@@ -9,6 +9,7 @@
9
9
 
10
10
  import { readdirSync, readFileSync, statSync, unlinkSync, existsSync, rmdirSync, renameSync, mkdirSync } from 'fs';
11
11
  import { join, dirname } from 'path';
12
+ import { workRootOwnership, workRootRefusalLine, type WorkRootVerdict } from './runner-home';
12
13
 
13
14
  /**
14
15
  * Recursively find files by name under a directory.
@@ -32,6 +33,13 @@ function findFiles(dir: string, name: string): string[] {
32
33
  export interface CleanupResult {
33
34
  deleted: number;
34
35
  logs: string[];
36
+ /**
37
+ * Whether the sweep declined to touch this root (#904).
38
+ *
39
+ * `deleted: 0` is what a refusal and a clean root both produce, so the
40
+ * counts cannot carry this. See `lib/runner-home.ts`.
41
+ */
42
+ refused: boolean;
35
43
  }
36
44
 
37
45
  export interface ArchiveResult {
@@ -39,6 +47,8 @@ export interface ArchiveResult {
39
47
  purged: number;
40
48
  skipped: number;
41
49
  logs: string[];
50
+ /** Whether the sweep declined to touch this root (#904). */
51
+ refused: boolean;
42
52
  }
43
53
 
44
54
  /**
@@ -52,7 +62,22 @@ export function archiveAndPurge(
52
62
  workDir: string,
53
63
  archiveTtlMs: number,
54
64
  purgeTtlMs: number,
65
+ env: NodeJS.ProcessEnv = process.env,
55
66
  ): ArchiveResult {
67
+ // Guarded here as well as in `cleanStaleFiles` because this is exported and
68
+ // destructive. A refusal reachable only through the caller is a side door,
69
+ // and the next caller walks through it (#904).
70
+ const verdict = workRootOwnership(workDir, env);
71
+ if (!verdict.sweep) {
72
+ return {
73
+ archived: 0,
74
+ purged: 0,
75
+ skipped: 0,
76
+ logs: [workRootRefusalLine(workDir, verdict)],
77
+ refused: true,
78
+ };
79
+ }
80
+
56
81
  const now = Date.now();
57
82
  let archived = 0;
58
83
  let purged = 0;
@@ -185,7 +210,7 @@ export function archiveAndPurge(
185
210
  } catch {}
186
211
  }
187
212
 
188
- return { archived, purged, skipped, logs };
213
+ return { archived, purged, skipped, logs, refused: false };
189
214
  }
190
215
 
191
216
  /**
@@ -196,7 +221,16 @@ export function cleanStaleFiles(
196
221
  workDir: string,
197
222
  fourHoursMs: number,
198
223
  sevenDaysMs: number,
224
+ env: NodeJS.ProcessEnv = process.env,
199
225
  ): CleanupResult {
226
+ // Decided before the `.ship-active` walk, not just before the archive pass:
227
+ // a sibling runner's `.ship-active` is state it is relying on, and deleting
228
+ // it is the same harm one function lower down (#904).
229
+ const verdict: WorkRootVerdict = workRootOwnership(workDir, env);
230
+ if (!verdict.sweep) {
231
+ return { deleted: 0, logs: [workRootRefusalLine(workDir, verdict)], refused: true };
232
+ }
233
+
200
234
  const now = Date.now();
201
235
  let deleted = 0;
202
236
  const logs: string[] = [];
@@ -215,10 +249,10 @@ export function cleanStaleFiles(
215
249
 
216
250
  // Archive and purge work directories
217
251
  const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000;
218
- const archiveResult = archiveAndPurge(workDir, fourHoursMs, thirtyDaysMs);
252
+ const archiveResult = archiveAndPurge(workDir, fourHoursMs, thirtyDaysMs, env);
219
253
 
220
254
  // Merge logs
221
255
  logs.push(...archiveResult.logs);
222
256
 
223
- return { deleted, logs };
257
+ return { deleted, logs, refused: archiveResult.refused };
224
258
  }
@@ -0,0 +1,278 @@
1
+ /**
2
+ * #323 SC-5 — whether a series of suite runs says anything.
3
+ *
4
+ * `lib/suite-run-reading.ts` reads ONE run. This reads a SERIES, and the two
5
+ * questions are genuinely different: a run is clean or it is not, and a
6
+ * series is consistent, flaky, failing, or not a series at all.
7
+ *
8
+ * One binding, because the interesting part is not any single run — it is
9
+ * the relationship between them, and a caller composing its own verdict
10
+ * would get that relationship right on the series it had seen and wrong on
11
+ * the one that matters. `scripts/suite-flake-run.ts` executes the runs and
12
+ * decides nothing.
13
+ */
14
+ import { describeSuiteFailures, type SuiteRun } from "./suite-run-reading";
15
+
16
+ /**
17
+ * What tree a run was taken against.
18
+ *
19
+ * `head` alone is not enough: the suites #323 suspects write scratch
20
+ * directories under the repository, and a run that dirties the tree it is
21
+ * measuring has changed the input to the next run without moving HEAD. The
22
+ * dirty list is `git status --porcelain` paths.
23
+ */
24
+ export interface TreeSignature {
25
+ head: string;
26
+ dirty: string[];
27
+ }
28
+
29
+ /**
30
+ * How the process ended, which no reading of its TEXT can establish.
31
+ *
32
+ * `readSuiteRun` deliberately takes no exit code: #469's run exited 0 having
33
+ * measured nothing and run `39405680063` exited `null` having measured
34
+ * nothing, and nothing in either output says which way to resolve that. A
35
+ * SIGNAL is not ambiguous — but only the process's caller ever sees one, so
36
+ * it is carried here and read by the series.
37
+ */
38
+ export interface RunExit {
39
+ status: number | null;
40
+ signal: string | null;
41
+ /** The spawn's own error, e.g. `ENOBUFS`, which names the cause. */
42
+ error: string | null;
43
+ }
44
+
45
+ export interface FlakeRun {
46
+ /** 1-based, so a reason can name the run an operator will look for. */
47
+ index: number;
48
+ signature: TreeSignature;
49
+ run: SuiteRun;
50
+ /** Where the full output was kept. A failure that is not kept is #323. */
51
+ log: string;
52
+ seconds: number;
53
+ exit: RunExit;
54
+ }
55
+
56
+ /**
57
+ * Why a run's ENDING disqualifies it, or null.
58
+ *
59
+ * `bun test` exits 0 for a clean suite and 1 for a failing one. **1 must not
60
+ * be refused**: refusing every non-zero exit would turn every FLAKY and
61
+ * FAILING series into UNMEASURED and leave the instrument unable to report
62
+ * its own subject.
63
+ */
64
+ export function exitRefusal(exit: RunExit): string | null {
65
+ if (exit.error !== null && exit.error !== "") {
66
+ return `the spawn failed: ${exit.error}`;
67
+ }
68
+ if (exit.signal !== null) {
69
+ return (
70
+ `it was killed by ${exit.signal}, so its log ends wherever the process did — and a truncated log ` +
71
+ "still parses as a clean reading when the output it captured contains a nested suite's summary, " +
72
+ "which is how run 39405680063 reported CONSISTENT over ten killed runs (#323)"
73
+ );
74
+ }
75
+ if (exit.status === null) {
76
+ return "the spawn reported no exit status and no signal, so nothing says how it ended";
77
+ }
78
+ if (exit.status !== 0 && exit.status !== 1) {
79
+ return `it exited ${exit.status}, which is not a verdict about the tests — bun uses 0 for a clean suite and 1 for a failing one`;
80
+ }
81
+ return null;
82
+ }
83
+
84
+ /**
85
+ * CONSISTENT, FLAKY and FAILING are three facts, not two.
86
+ *
87
+ * A suite that fails every run is red — an ordinary thing to be, and
88
+ * somebody's build to fix. A suite that fails SOME runs is the defect #323
89
+ * is about. One word for both would leave the series unable to report its
90
+ * own subject.
91
+ */
92
+ export type FlakeVerdict = "CONSISTENT" | "FLAKY" | "FAILING" | "UNMEASURED";
93
+
94
+ /**
95
+ * Whether the runs agreed about how much work they did.
96
+ *
97
+ * Reported BESIDE the verdict and never folded into it. The 2026-10-11
98
+ * reading's only surviving lead is that the failing run executed 384 more
99
+ * `expect()` calls than either clean one — so a varying count is a finding.
100
+ * It is not, on its own, evidence that anything is wrong: a case enumerating
101
+ * a directory legitimately varies. Folding it into the verdict would make
102
+ * one of those two readings unavailable.
103
+ *
104
+ * `UNREPORTED` is first-class. A runner that printed no count and a runner
105
+ * whose count held steady are opposite facts, and `STABLE` is the value that
106
+ * would let a reader treat the first as the second.
107
+ */
108
+ export type AssertionStability = "STABLE" | "VARIED" | "UNREPORTED";
109
+
110
+ /**
111
+ * What `treeSignatureKey` can and cannot see, carried in the reading itself.
112
+ *
113
+ * A `CONSISTENT` verdict says the runs agreed over one tree, and a reader
114
+ * naturally hears "one set of inputs". It is not: the signature is HEAD plus
115
+ * `git status --porcelain`, so a gitignored path, anything under `/tmp` or
116
+ * `~/.rungate`, and `node_modules` can all change between runs with the
117
+ * signature unmoved.
118
+ *
119
+ * #799 is the measured instance rather than a worry. Series `39412289321`
120
+ * reported ten runs over one tree and was right to — while `test/phase-0.test.ts`
121
+ * collected its test list from a `/tmp` fixture the PREVIOUS run had left, so
122
+ * run 1 executed 15,194 tests and runs 2 through 10 executed 15,250.
123
+ *
124
+ * In the reading and not only in the printed sentence, because `flake.json`
125
+ * is what anybody reads months later, and a caveat that exists only in a
126
+ * console line is a caveat the artefact does not carry.
127
+ */
128
+ export const SIGNATURE_SCOPE =
129
+ "the tree signature is HEAD plus `git status --porcelain`, so it covers the working tree and nothing " +
130
+ "else — a gitignored path, anything under /tmp or ~/.rungate, and node_modules can all differ between " +
131
+ "runs while this series reports one tree (#799)";
132
+
133
+ export interface FlakeReading {
134
+ verdict: FlakeVerdict;
135
+ /** Why it is not CONSISTENT. Empty exactly when it is. */
136
+ reasons: string[];
137
+ assertions: AssertionStability;
138
+ /** One entry per run, in order, `null` where the runner did not say. */
139
+ assertionCounts: Array<number | null>;
140
+ runs: number;
141
+ /** What the tree comparison covered. Always present; see `SIGNATURE_SCOPE`. */
142
+ signatureScope: string;
143
+ }
144
+
145
+ /**
146
+ * One string per tree, so two readings can be compared without anybody
147
+ * deciding field by field what counts as the same tree.
148
+ *
149
+ * The dirty list is sorted: `git status --porcelain` order is not a property
150
+ * of the tree, and two runs differing only in it are one tree.
151
+ */
152
+ export function treeSignatureKey(signature: TreeSignature): string {
153
+ return `${signature.head}\n${[...signature.dirty].sort().join("\n")}`;
154
+ }
155
+
156
+ /** The runs that could be read at all. */
157
+ function measured(runs: FlakeRun[]) {
158
+ return runs.flatMap(r => (r.run.status === "MEASURED" ? [{ entry: r, run: r.run }] : []));
159
+ }
160
+
161
+ /**
162
+ * Read a series.
163
+ *
164
+ * FAILS CLOSED, like the single-run reader it sits on. Every path that
165
+ * cannot positively establish that `minimum` readable runs were taken over
166
+ * one tree returns UNMEASURED with reasons. There is no path that returns
167
+ * CONSISTENT from a series that does not say so.
168
+ */
169
+ export function flakeReading(runs: FlakeRun[], minimum: number): FlakeReading {
170
+ const assertionCounts = runs.map(r => (r.run.status === "MEASURED" ? r.run.expectCalls : null));
171
+ const reasons: string[] = [];
172
+
173
+ // A short series first, because it explains every other absence. A job
174
+ // that died after three runs has measured three runs, and ten is what was
175
+ // asked for — the instrument failing is not the subject passing (#60).
176
+ if (runs.length < minimum) {
177
+ reasons.push(
178
+ `only ${runs.length} of ${minimum} run(s) completed, so the series was cut short — ` +
179
+ "a short series is not a clean series, however green the runs in it were (#323 SC-5)",
180
+ );
181
+ }
182
+
183
+ // Every run that did not FINISH, before anything is read off its text.
184
+ // This is the guard run 39405680063 did not have: ten children killed by
185
+ // SIGTERM on a 1 MiB `maxBuffer` overflow, ten truncated logs whose only
186
+ // surviving summary belonged to a nested suite, and a CONSISTENT verdict
187
+ // over a suite of 5,500 tests reporting one pass.
188
+ for (const r of runs) {
189
+ const why = exitRefusal(r.exit);
190
+ if (why !== null) reasons.push(`run ${r.index} did not finish (log ${r.log}): ${why}`);
191
+ }
192
+
193
+ // Every unreadable run, by index. An UNMEASURED run is not a clean run:
194
+ // `readSuiteRun` returns it for a log that printed no summary, and this
195
+ // repository's logs carry nested suites' summaries, so such a log still
196
+ // contains the text a `grep "0 fail"` would match.
197
+ for (const r of runs) {
198
+ if (r.run.status !== "MEASURED") {
199
+ reasons.push(`run ${r.index} could not be read (log ${r.log}): ${r.run.reason}`);
200
+ }
201
+ }
202
+
203
+ // One tree, or it is not a series. Named by what differs, because
204
+ // "the tree moved" tells nobody where to look and a scratch directory
205
+ // appearing under the repository is SC-6's whole hypothesis.
206
+ const trees = new Map<string, FlakeRun[]>();
207
+ for (const r of runs) {
208
+ const key = treeSignatureKey(r.signature);
209
+ trees.set(key, [...(trees.get(key) ?? []), r]);
210
+ }
211
+ if (trees.size > 1) {
212
+ const described = [...trees.entries()].map(([key, group]) => {
213
+ const where = group.map(g => g.index).join(", ");
214
+ const [head, ...dirty] = key.split("\n");
215
+ const paths = dirty.filter(d => d !== "");
216
+ return `run(s) ${where} at ${head}${paths.length === 0 ? " with a clean tree" : ` with ${paths.join(", ")}`}`;
217
+ });
218
+ reasons.push(
219
+ `the tree changed during the series, so these runs did not measure one thing: ${described.join("; ")} ` +
220
+ "— ten green runs over three trees is not ten green runs (#323 SC-5)",
221
+ );
222
+ }
223
+
224
+ const counts = assertionCounts.filter((c): c is number => c !== null);
225
+ const assertions: AssertionStability =
226
+ counts.length !== runs.length || runs.length === 0
227
+ ? "UNREPORTED"
228
+ : new Set(counts).size === 1
229
+ ? "STABLE"
230
+ : "VARIED";
231
+
232
+ if (reasons.length > 0) {
233
+ return { verdict: "UNMEASURED", reasons, assertions, assertionCounts, runs: runs.length, signatureScope: SIGNATURE_SCOPE };
234
+ }
235
+
236
+ const readable = measured(runs);
237
+ const failing = readable.filter(r => r.run.fail > 0);
238
+ if (failing.length > 0) {
239
+ for (const f of failing) {
240
+ reasons.push(`run ${f.entry.index} (log ${f.entry.log}): ${describeSuiteFailures(f.run)}`);
241
+ }
242
+ }
243
+
244
+ const verdict: FlakeVerdict =
245
+ failing.length === 0 ? "CONSISTENT" : failing.length === readable.length ? "FAILING" : "FLAKY";
246
+
247
+ return { verdict, reasons, assertions, assertionCounts, runs: runs.length, signatureScope: SIGNATURE_SCOPE };
248
+ }
249
+
250
+ /** The sentence a caller prints instead of the verdict on its own. */
251
+ export function describeFlakeReading(reading: FlakeReading): string {
252
+ const { verdict, runs, assertions, assertionCounts, reasons } = reading;
253
+ const counts = assertionCounts.filter((c): c is number => c !== null);
254
+ const spread =
255
+ assertions === "VARIED"
256
+ ? ` The runs did not agree about how much work they did: ${Math.min(...counts)} to ` +
257
+ `${Math.max(...counts)} expect() calls, a spread of ${Math.max(...counts) - Math.min(...counts)}. ` +
258
+ "That is the only surviving lead from the 2026-10-11 reading (#323 SC-4)."
259
+ : assertions === "UNREPORTED"
260
+ ? " The runner did not report an assertion count for every run, so nothing is claimed about one."
261
+ : ` Every run executed ${counts[0]} expect() calls.`;
262
+
263
+ const failed = reasons.length;
264
+ const head =
265
+ verdict === "CONSISTENT"
266
+ ? `CONSISTENT: ${runs} consecutive run(s) on one tree, none failing.`
267
+ : verdict === "FLAKY"
268
+ ? `FLAKY: ${failed} of ${runs} consecutive run(s) on one tree failed — the same tree, different answers.`
269
+ : verdict === "FAILING"
270
+ ? `FAILING: all ${runs} run(s) failed. The suite is red; this is not flake.`
271
+ : `UNMEASURED: the series does not support a verdict.`;
272
+
273
+ // The scope goes on EVERY verdict, not only CONSISTENT. A reader who sees
274
+ // it once beside a clean series and never again would reasonably conclude
275
+ // the other verdicts had looked harder, and they did not — it is the same
276
+ // comparison in all four cases.
277
+ return [head, ...reasons.map(r => ` - ${r}`), spread.trim(), `Scope: ${reading.signatureScope}`].join("\n");
278
+ }
@@ -28,15 +28,21 @@ const SHA = /^[0-9a-f]{7,40}$/i;
28
28
 
29
29
  /**
30
30
  * UNMEASURED is not a result. It is the statement that this run has no suite
31
- * evidence, and it is deliberately not writable — see gates/schema.ts, which
32
- * accepts PASS and FAIL only. A recorded UNMEASURED would be a measurement
33
- * saying there was no measurement.
31
+ * evidence, and it is deliberately not writable — see gates/schema.ts, whose
32
+ * writable verdicts are PASS, FAIL and PARTIAL. A recorded UNMEASURED would
33
+ * be a measurement saying there was no measurement.
34
+ *
35
+ * PARTIAL is writable and is not a verdict THIS type has, which is the whole
36
+ * of #673's first half: a reading taken over less than the suite is a reading
37
+ * that was taken, so the recorder may write it — and it is not a statement
38
+ * about the suite, so every consumer must see UNMEASURED. The conversion
39
+ * happens in `readSuiteMeasurement` below and nowhere else.
34
40
  */
35
41
  export type SuiteVerdict = "PASS" | "FAIL" | "UNMEASURED";
36
42
 
37
43
  /** What a recorder persists at `suiteMeasurement` in workflow-state.json. */
38
44
  export interface SuiteMeasurement {
39
- verdict: "PASS" | "FAIL";
45
+ verdict: "PASS" | "FAIL" | "PARTIAL";
40
46
  /** The commit the suite actually ran against. */
41
47
  measuredSha: string;
42
48
  /** Failing tests. 0 for a PASS, at least 1 for a FAIL. */
@@ -136,6 +142,19 @@ export function readSuiteMeasurement(state: unknown): SuiteReading {
136
142
  }
137
143
 
138
144
  const verdict = raw.verdict;
145
+ // #673 — a reading the caller declared PARTIAL. Its own branch rather than
146
+ // falling through to the clause below, because "nobody recognises this
147
+ // word" and "this run measured some tests and said so" are different
148
+ // things to go and do something about: the first is a corrupt record and
149
+ // the second is a run that has not measured the suite YET.
150
+ if (verdict === "PARTIAL") {
151
+ return unmeasured(
152
+ `the suite measurement for ${measuredSha} was taken over PART of the suite: the caller ` +
153
+ `declared the run it recorded was less than the whole of it, so what it found is not a ` +
154
+ `statement about the suite (#673). ${describe(raw.tests)} test(s) collected is not a ` +
155
+ `reading a gate can act on, whatever the count`,
156
+ );
157
+ }
139
158
  if (verdict !== "PASS" && verdict !== "FAIL") {
140
159
  return unmeasured(
141
160
  `the suite measurement for ${measuredSha} carries no usable verdict (got ${describe(verdict)}) — ` +