rungate 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/affected-tests.ts +398 -0
- package/lib/agent-audit.ts +2 -3
- package/lib/agent-worktree.ts +287 -0
- package/lib/agents-md-sections.ts +209 -0
- package/lib/behavioral-cache.ts +150 -2
- package/lib/branch-diff.ts +139 -0
- package/lib/branch-protection.ts +790 -0
- package/lib/brief-ac-alignment.ts +104 -0
- package/lib/brief-references.ts +205 -0
- package/lib/bun-version.ts +347 -0
- package/lib/check-remedy.ts +140 -0
- package/lib/ci-checkout.ts +18 -1
- package/lib/ci-history-depth.ts +159 -0
- package/lib/ci-setup-step.ts +96 -0
- package/lib/claude-code-action.ts +85 -0
- package/lib/claude-tool-vocabulary.ts +188 -0
- package/lib/cli-commands.ts +15 -0
- package/lib/commit-sha.ts +42 -0
- package/lib/commit-subject.ts +90 -0
- package/lib/compliance-report.ts +377 -45
- package/lib/compliance.ts +196 -20
- package/lib/config-loader.ts +51 -1
- package/lib/config-trackable.ts +391 -0
- package/lib/conformity.ts +1272 -189
- package/lib/containment.ts +238 -0
- package/lib/content-hash.ts +24 -0
- package/lib/context-docs.ts +143 -28
- package/lib/create-brief.ts +2 -1
- package/lib/create-sc.ts +323 -4
- package/lib/dev-liveness.ts +180 -0
- package/lib/discovery-branch-points.ts +432 -0
- package/lib/discovery-stability.ts +481 -0
- package/lib/distribution-currency.ts +186 -0
- package/lib/doc-routing.ts +157 -0
- package/lib/doc-scan-targets.ts +9 -0
- package/lib/evidence-command-transport.ts +160 -0
- package/lib/evidence-prevalidator.ts +246 -2
- package/lib/evidence-satisfiability.ts +534 -12
- package/lib/failure-intake.ts +317 -0
- package/lib/fallow-availability.ts +286 -0
- package/lib/fs-write-capabilities.ts +232 -0
- package/lib/gate-refusal-currency.ts +112 -0
- package/lib/gate-run-split.ts +147 -0
- package/lib/gate-timeouts.ts +82 -0
- package/lib/generators/agent-briefs.ts +4 -1
- package/lib/generators/agents-md.ts +204 -48
- package/lib/generators/code-map.ts +2 -1
- package/lib/generators/types.ts +8 -0
- package/lib/git-dates.ts +55 -0
- package/lib/git-isolation.ts +140 -0
- package/lib/github.ts +15 -0
- package/lib/google-auth-action.ts +93 -0
- package/lib/harness-project.ts +48 -0
- package/lib/harness-provenance.ts +255 -0
- package/lib/hook-evidence.ts +235 -0
- package/lib/hook-scopes.ts +386 -0
- package/lib/hook-target-tree.ts +398 -0
- package/lib/hook-trees.ts +447 -0
- package/lib/js-top-level-code.ts +78 -10
- package/lib/junit-checks.ts +536 -0
- package/lib/launch-base.ts +253 -0
- package/lib/line-citation.ts +372 -0
- package/lib/local-calendar-day.ts +78 -0
- package/lib/orphan-reaper.ts +468 -0
- package/lib/parity-binding.ts +205 -0
- package/lib/per-test-timeouts.ts +219 -0
- package/lib/plan-coverage.ts +151 -0
- package/lib/prior-branch.ts +201 -8
- package/lib/process-table.ts +146 -0
- package/lib/project-state-derivation.ts +219 -0
- package/lib/project-state-root.ts +258 -0
- package/lib/project-state-shape.ts +59 -0
- package/lib/promote-outputs.ts +6 -2
- package/lib/published-artifact.ts +525 -0
- package/lib/readdir-sorted.ts +45 -0
- package/lib/report-provenance.ts +156 -0
- package/lib/review-execution-reading.ts +206 -0
- package/lib/review-tool-agreement.ts +433 -0
- package/lib/rule-promotion.ts +234 -0
- package/lib/rule-retirement.ts +525 -0
- package/lib/run-branch.ts +612 -0
- package/lib/rungate-schema.ts +19 -0
- package/lib/runner-home.ts +274 -0
- package/lib/sc-claim-forms.ts +75 -0
- package/lib/sc-id-cross-tree.ts +323 -0
- package/lib/sc-ids.ts +121 -13
- package/lib/sc-reservations.ts +289 -0
- package/lib/scaffold/ai-review-findings.ts +403 -0
- package/lib/scaffold/ai-review-step.ts +4777 -0
- package/lib/scaffold/managed-workflow.ts +587 -192
- package/lib/scaffold/owned-files.ts +186 -39
- package/lib/scaffold/pre-push-hook.ts +49 -5
- package/lib/scaffold/rehearsal.ts +205 -71
- package/lib/scaffold/steps.ts +1517 -293
- package/lib/scaffold/workflow-census.ts +217 -0
- package/lib/scaffold/workflow-drift.ts +217 -0
- package/lib/scaffold/write-steps.ts +10 -4
- package/lib/scan-budget.ts +247 -0
- package/lib/scanner.ts +80 -62
- package/lib/scratch-roots.ts +50 -5
- package/lib/screenshot-discovery.ts +121 -0
- package/lib/screenshot-upload.ts +41 -1
- package/lib/secret-patterns.ts +511 -8
- package/lib/ship-mode.ts +219 -0
- package/lib/source-slice.ts +294 -0
- package/lib/spec-listing.ts +148 -0
- package/lib/spec-requirements.ts +209 -0
- package/lib/stale-cleanup.ts +37 -3
- package/lib/suite-flake.ts +278 -0
- package/lib/suite-measurement.ts +23 -4
- package/lib/suite-run-reading.ts +411 -1
- package/lib/suite-timeout-margin.ts +193 -0
- package/lib/test-case-bodies.ts +883 -0
- package/lib/test-case-deletion.ts +653 -0
- package/lib/test-census.ts +97 -29
- package/lib/test-command-positions.ts +274 -0
- package/lib/test-command.ts +133 -0
- package/lib/test-suite-lock.ts +325 -26
- package/lib/tool-probe.ts +142 -0
- package/lib/top-level-cd.ts +393 -0
- package/lib/tracked-vs-ignored.ts +234 -0
- package/lib/tree-entries.ts +89 -0
- package/lib/ts-ast.ts +200 -0
- package/lib/validators/spec-validators.ts +3 -2
- package/lib/walk-test-files.ts +97 -0
- package/lib/workflow-actions.ts +263 -0
- package/lib/worktree-collect.ts +177 -5
- package/lib/worktree-repo.ts +36 -1
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +45 -0
- package/scripts/audit-specs.ts +247 -12
- package/scripts/audit-transcript.ts +2 -1
- package/scripts/convert-spec.ts +2 -1
- package/scripts/create-sc.ts +37 -7
- package/scripts/create-spec.ts +2 -1
- package/scripts/create-success.ts +2 -1
- package/scripts/extract-constraints.ts +197 -26
- package/scripts/generate-code-map.ts +62 -28
- package/scripts/scaffold-project.ts +12 -2
- package/scripts/split-spec.ts +2 -1
- package/scripts/sync-sc-status.ts +127 -14
- package/scripts/sync-spec-tests.ts +2 -1
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does a document the routing table calls a spec actually state anything? (#579)
|
|
3
|
+
*
|
|
4
|
+
* `.claude/rules/specs-routing.md` is the table every agent is told to consult
|
|
5
|
+
* before changing an area. It is generated from `specs/`, one row per file,
|
|
6
|
+
* and it promises each row GOVERNS something. Nothing checked that the
|
|
7
|
+
* document on the other end states a requirement.
|
|
8
|
+
*
|
|
9
|
+
* `specs/PROJECT-STATE.md` was a **rendered PROJECT-STATE.md snapshot from
|
|
10
|
+
* 2026-09-22 with spec frontmatter pasted on top** — a "Current phase" line, a
|
|
11
|
+
* stale priorities list, and status tables citing criteria it did not declare.
|
|
12
|
+
* It sat in the routing table as the governing spec for project state
|
|
13
|
+
* management for two months. An agent sent there before touching
|
|
14
|
+
* `scripts/update-project-state.ts` got no criteria at all.
|
|
15
|
+
*
|
|
16
|
+
* **A gap in a routing table is visible; this is not.** The reader believes
|
|
17
|
+
* they have arrived. That is the same harm as `.claude/rules/config-access.md`'s
|
|
18
|
+
* Silent Default Substitution with a document as the victim: an empty answer
|
|
19
|
+
* and a correct answer are the same shape to the consumer.
|
|
20
|
+
*
|
|
21
|
+
* ## Why the exemption is frontmatter and not an allowlist
|
|
22
|
+
*
|
|
23
|
+
* Some documents in `specs/` legitimately state no criteria —
|
|
24
|
+
* `BOOTSTRAP-DATA-FLOW-SPEC.md` is a redirect stub whose whole job is to point
|
|
25
|
+
* readers at the directory it was split into. An allowlist would cover that,
|
|
26
|
+
* and two things are wrong with one here.
|
|
27
|
+
*
|
|
28
|
+
* DRIFT-3 refuses any entry ADDED to `.claude/conformity-allowlists.json`
|
|
29
|
+
* relative to `origin/main`, because an agent must not pass a gate by editing
|
|
30
|
+
* the gate. Shipping this check with its current violators pre-listed is
|
|
31
|
+
* exactly that move, and the ratchet is right to refuse it.
|
|
32
|
+
*
|
|
33
|
+
* The better reason is that **an allowlist is invisible to the person being
|
|
34
|
+
* harmed.** The reader harmed by a criteria-free spec is the one who followed
|
|
35
|
+
* the routing table to it; a JSON file in `.claude/` is not on that path. A
|
|
36
|
+
* declaration in the document's own frontmatter is, and the generated routing
|
|
37
|
+
* table can render it, so the warning reaches the reader before they click
|
|
38
|
+
* rather than after they have read 300 lines looking for requirements.
|
|
39
|
+
*
|
|
40
|
+
* So: state criteria, or say `criteria: none` and say why. Both are honest.
|
|
41
|
+
* Silence is the thing that is not.
|
|
42
|
+
*/
|
|
43
|
+
import { existsSync, readFileSync, readdirSync, statSync } from "fs";
|
|
44
|
+
import { basename, join } from "path";
|
|
45
|
+
|
|
46
|
+
/** A checkbox criterion line, whatever id prefix it uses. */
|
|
47
|
+
const CRITERION_LINE = /^\s*- \[[ xX]\]/;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Counted on any checkbox line rather than on `- [ ] SC-`.
|
|
51
|
+
*
|
|
52
|
+
* `BOOTSTRAP-TEST-PLAN.md` numbers its criteria `TP-1`, `TP-2`, and the first
|
|
53
|
+
* census written for this issue counted `- [ ] SC-` and reported it as
|
|
54
|
+
* stating nothing. It states 41 criteria. A detector narrower than what it
|
|
55
|
+
* detects, in the direction that accuses a correct document —
|
|
56
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`, second shape.
|
|
57
|
+
*/
|
|
58
|
+
export function countCriteria(content: string): number {
|
|
59
|
+
return content.split("\n").filter(l => CRITERION_LINE.test(l)).length;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface SpecRequirementReading {
|
|
63
|
+
/** Path relative to `specs/`, so a report can name it. */
|
|
64
|
+
spec: string;
|
|
65
|
+
criteria: number;
|
|
66
|
+
/** The document says it states none, in its own frontmatter. */
|
|
67
|
+
declaredNone: boolean;
|
|
68
|
+
/** Why it states none. A declaration with no reason is a mute. */
|
|
69
|
+
reason: string | null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function frontmatterField(content: string, key: string): string | null {
|
|
73
|
+
const fm = content.match(/^---\n([\s\S]*?)\n---/);
|
|
74
|
+
if (!fm) return null;
|
|
75
|
+
const m = fm[1].match(new RegExp(`^${key}:\\s*(.+)$`, "m"));
|
|
76
|
+
return m ? m[1].trim() : null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function readSpecRequirements(specsDir: string, rel: string): SpecRequirementReading {
|
|
80
|
+
const content = readFileSync(join(specsDir, rel), "utf-8");
|
|
81
|
+
const declared = frontmatterField(content, "criteria");
|
|
82
|
+
const reason = frontmatterField(content, "criteria-reason");
|
|
83
|
+
return {
|
|
84
|
+
spec: rel,
|
|
85
|
+
criteria: countCriteria(content),
|
|
86
|
+
declaredNone: declared === "none",
|
|
87
|
+
reason: reason && reason.length > 0 ? reason : null,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Refusals, worded after the property rather than after the proxy.
|
|
93
|
+
*
|
|
94
|
+
* Three of them, and the second and third exist because the first alone is
|
|
95
|
+
* satisfiable by something other than fixing the problem — #492's trap. A
|
|
96
|
+
* check that only refused "no criteria and no declaration" is cleared by
|
|
97
|
+
* pasting `criteria: none` onto a spec that has requirements and has simply
|
|
98
|
+
* not written them down, which is the state this check exists to surface.
|
|
99
|
+
*/
|
|
100
|
+
export function requirementViolations(readings: SpecRequirementReading[]): string[] {
|
|
101
|
+
const out: string[] = [];
|
|
102
|
+
for (const r of readings) {
|
|
103
|
+
if (r.criteria === 0 && !r.declaredNone) {
|
|
104
|
+
out.push(
|
|
105
|
+
`${r.spec} states no criteria and does not say so. The routing table presents it as a ` +
|
|
106
|
+
`governing spec, so a reader is sent there expecting requirements. Add criteria, or ` +
|
|
107
|
+
`declare "criteria: none" with a "criteria-reason:" in its frontmatter.`,
|
|
108
|
+
);
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
// Both directions, like every other ratchet here. A spec that gains
|
|
112
|
+
// criteria must withdraw the declaration, or the declaration rots into a
|
|
113
|
+
// line that certifies nothing — and the routing table keeps warning
|
|
114
|
+
// readers away from a document that now has what they came for.
|
|
115
|
+
if (r.declaredNone && r.criteria > 0) {
|
|
116
|
+
out.push(
|
|
117
|
+
`${r.spec} declares "criteria: none" and states ${r.criteria}. Remove the declaration.`,
|
|
118
|
+
);
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
if (r.declaredNone && r.reason === null) {
|
|
122
|
+
out.push(
|
|
123
|
+
`${r.spec} declares "criteria: none" with no "criteria-reason:". A reader routed here ` +
|
|
124
|
+
`needs to know whether this is a redirect stub or an unwritten spec.`,
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return out;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* `testable:` read as a value rather than as `=== "true"`.
|
|
133
|
+
*
|
|
134
|
+
* DRIFT-2 and SPEC-DISCOVERY both ask `fm.testable === "true"`, so every
|
|
135
|
+
* other spelling is silently FALSE. `specs/DA-COMPLIANCE-SPEC.md` carries
|
|
136
|
+
* `testable: no`, which happens to mean what its author intended; `testable:
|
|
137
|
+
* yes` would mean the opposite of what its author intended and nothing would
|
|
138
|
+
* say so. The spec would drop out of the conformity surface while its
|
|
139
|
+
* frontmatter, the routing table and any reader all said it was in.
|
|
140
|
+
*
|
|
141
|
+
* A fail-open with no amplifier today is still a fail-open. One line.
|
|
142
|
+
*/
|
|
143
|
+
export function testableViolations(specsDir: string, specs: string[]): string[] {
|
|
144
|
+
const out: string[] = [];
|
|
145
|
+
for (const rel of specs) {
|
|
146
|
+
const raw = frontmatterField(readFileSync(join(specsDir, rel), "utf-8"), "testable");
|
|
147
|
+
if (raw === null) continue; // absence is a separate check's question
|
|
148
|
+
if (raw !== "true" && raw !== "false") {
|
|
149
|
+
out.push(
|
|
150
|
+
`${rel} has "testable: ${raw}". Every reader compares against the string "true", so ` +
|
|
151
|
+
`anything else means false — including "yes". Write true or false.`,
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return out;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Spec basenames that also name a file somewhere else in the project.
|
|
160
|
+
*
|
|
161
|
+
* DRIFT-2 credits a testable spec when any test file's text contains its
|
|
162
|
+
* name, and for a top-level spec that name is a bare basename. On this repo
|
|
163
|
+
* `specs/PROJECT-STATE.md` and the generated root `PROJECT-STATE.md` share
|
|
164
|
+
* one, and **ten test files mention the root one** — so the spec was
|
|
165
|
+
* referenced, by DRIFT-2's reading, by tests that have nothing to do with it.
|
|
166
|
+
*
|
|
167
|
+
* This is the fifth shape and it was aimed straight at #579's own fix: the
|
|
168
|
+
* issue's prescribed remedy is `testable: true` plus "a test naming it, which
|
|
169
|
+
* test/update-project-state.test.ts can do in one line". Flipping the flag
|
|
170
|
+
* alone would have turned DRIFT-2 green for this spec on the strength of
|
|
171
|
+
* `test/organize-project.test.ts` talking about a status file.
|
|
172
|
+
*
|
|
173
|
+
* Measured rather than assumed: exactly one spec basename collides on this
|
|
174
|
+
* tree. Requiring the `specs/`-qualified path from EVERY test would orphan 15
|
|
175
|
+
* of 22 testable specs, so the rule is narrowed to the ambiguous ones — where
|
|
176
|
+
* a bare name genuinely does not say which file is meant.
|
|
177
|
+
*/
|
|
178
|
+
export function ambiguousSpecNames(root: string, specs: string[]): Set<string> {
|
|
179
|
+
const names = new Map<string, string>();
|
|
180
|
+
for (const rel of specs) names.set(basename(rel), rel);
|
|
181
|
+
const found = new Set<string>();
|
|
182
|
+
const SKIP = new Set(["node_modules", ".git", "specs", "dist", "build", ".fallow", ".next"]);
|
|
183
|
+
const walk = (dir: string): void => {
|
|
184
|
+
let entries: string[];
|
|
185
|
+
try { entries = readdirSync(dir); } catch { return; }
|
|
186
|
+
for (const n of entries) {
|
|
187
|
+
if (SKIP.has(n)) continue;
|
|
188
|
+
const p = join(dir, n);
|
|
189
|
+
let st;
|
|
190
|
+
try { st = statSync(p); } catch { continue; }
|
|
191
|
+
if (st.isDirectory()) walk(p);
|
|
192
|
+
else if (names.has(n)) found.add(names.get(n)!);
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
if (existsSync(root)) walk(root);
|
|
196
|
+
return found;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Whether a test names THIS spec.
|
|
201
|
+
*
|
|
202
|
+
* Unambiguous names are matched bare, which is how every existing credit in
|
|
203
|
+
* this repo is earned. An ambiguous one must be written `specs/<path>`, which
|
|
204
|
+
* is the only form that says which file is meant.
|
|
205
|
+
*/
|
|
206
|
+
export function specIsReferenced(testContent: string, rel: string, ambiguous: Set<string>): boolean {
|
|
207
|
+
if (ambiguous.has(rel)) return testContent.includes(`specs/${rel}`);
|
|
208
|
+
return testContent.includes(rel);
|
|
209
|
+
}
|
package/lib/stale-cleanup.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import { readdirSync, readFileSync, statSync, unlinkSync, existsSync, rmdirSync, renameSync, mkdirSync } from 'fs';
|
|
11
11
|
import { join, dirname } from 'path';
|
|
12
|
+
import { workRootOwnership, workRootRefusalLine, type WorkRootVerdict } from './runner-home';
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
15
|
* Recursively find files by name under a directory.
|
|
@@ -32,6 +33,13 @@ function findFiles(dir: string, name: string): string[] {
|
|
|
32
33
|
export interface CleanupResult {
|
|
33
34
|
deleted: number;
|
|
34
35
|
logs: string[];
|
|
36
|
+
/**
|
|
37
|
+
* Whether the sweep declined to touch this root (#904).
|
|
38
|
+
*
|
|
39
|
+
* `deleted: 0` is what a refusal and a clean root both produce, so the
|
|
40
|
+
* counts cannot carry this. See `lib/runner-home.ts`.
|
|
41
|
+
*/
|
|
42
|
+
refused: boolean;
|
|
35
43
|
}
|
|
36
44
|
|
|
37
45
|
export interface ArchiveResult {
|
|
@@ -39,6 +47,8 @@ export interface ArchiveResult {
|
|
|
39
47
|
purged: number;
|
|
40
48
|
skipped: number;
|
|
41
49
|
logs: string[];
|
|
50
|
+
/** Whether the sweep declined to touch this root (#904). */
|
|
51
|
+
refused: boolean;
|
|
42
52
|
}
|
|
43
53
|
|
|
44
54
|
/**
|
|
@@ -52,7 +62,22 @@ export function archiveAndPurge(
|
|
|
52
62
|
workDir: string,
|
|
53
63
|
archiveTtlMs: number,
|
|
54
64
|
purgeTtlMs: number,
|
|
65
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
55
66
|
): ArchiveResult {
|
|
67
|
+
// Guarded here as well as in `cleanStaleFiles` because this is exported and
|
|
68
|
+
// destructive. A refusal reachable only through the caller is a side door,
|
|
69
|
+
// and the next caller walks through it (#904).
|
|
70
|
+
const verdict = workRootOwnership(workDir, env);
|
|
71
|
+
if (!verdict.sweep) {
|
|
72
|
+
return {
|
|
73
|
+
archived: 0,
|
|
74
|
+
purged: 0,
|
|
75
|
+
skipped: 0,
|
|
76
|
+
logs: [workRootRefusalLine(workDir, verdict)],
|
|
77
|
+
refused: true,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
56
81
|
const now = Date.now();
|
|
57
82
|
let archived = 0;
|
|
58
83
|
let purged = 0;
|
|
@@ -185,7 +210,7 @@ export function archiveAndPurge(
|
|
|
185
210
|
} catch {}
|
|
186
211
|
}
|
|
187
212
|
|
|
188
|
-
return { archived, purged, skipped, logs };
|
|
213
|
+
return { archived, purged, skipped, logs, refused: false };
|
|
189
214
|
}
|
|
190
215
|
|
|
191
216
|
/**
|
|
@@ -196,7 +221,16 @@ export function cleanStaleFiles(
|
|
|
196
221
|
workDir: string,
|
|
197
222
|
fourHoursMs: number,
|
|
198
223
|
sevenDaysMs: number,
|
|
224
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
199
225
|
): CleanupResult {
|
|
226
|
+
// Decided before the `.ship-active` walk, not just before the archive pass:
|
|
227
|
+
// a sibling runner's `.ship-active` is state it is relying on, and deleting
|
|
228
|
+
// it is the same harm one function lower down (#904).
|
|
229
|
+
const verdict: WorkRootVerdict = workRootOwnership(workDir, env);
|
|
230
|
+
if (!verdict.sweep) {
|
|
231
|
+
return { deleted: 0, logs: [workRootRefusalLine(workDir, verdict)], refused: true };
|
|
232
|
+
}
|
|
233
|
+
|
|
200
234
|
const now = Date.now();
|
|
201
235
|
let deleted = 0;
|
|
202
236
|
const logs: string[] = [];
|
|
@@ -215,10 +249,10 @@ export function cleanStaleFiles(
|
|
|
215
249
|
|
|
216
250
|
// Archive and purge work directories
|
|
217
251
|
const thirtyDaysMs = 30 * 24 * 60 * 60 * 1000;
|
|
218
|
-
const archiveResult = archiveAndPurge(workDir, fourHoursMs, thirtyDaysMs);
|
|
252
|
+
const archiveResult = archiveAndPurge(workDir, fourHoursMs, thirtyDaysMs, env);
|
|
219
253
|
|
|
220
254
|
// Merge logs
|
|
221
255
|
logs.push(...archiveResult.logs);
|
|
222
256
|
|
|
223
|
-
return { deleted, logs };
|
|
257
|
+
return { deleted, logs, refused: archiveResult.refused };
|
|
224
258
|
}
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #323 SC-5 — whether a series of suite runs says anything.
|
|
3
|
+
*
|
|
4
|
+
* `lib/suite-run-reading.ts` reads ONE run. This reads a SERIES, and the two
|
|
5
|
+
* questions are genuinely different: a run is clean or it is not, and a
|
|
6
|
+
* series is consistent, flaky, failing, or not a series at all.
|
|
7
|
+
*
|
|
8
|
+
* One binding, because the interesting part is not any single run — it is
|
|
9
|
+
* the relationship between them, and a caller composing its own verdict
|
|
10
|
+
* would get that relationship right on the series it had seen and wrong on
|
|
11
|
+
* the one that matters. `scripts/suite-flake-run.ts` executes the runs and
|
|
12
|
+
* decides nothing.
|
|
13
|
+
*/
|
|
14
|
+
import { describeSuiteFailures, type SuiteRun } from "./suite-run-reading";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* What tree a run was taken against.
|
|
18
|
+
*
|
|
19
|
+
* `head` alone is not enough: the suites #323 suspects write scratch
|
|
20
|
+
* directories under the repository, and a run that dirties the tree it is
|
|
21
|
+
* measuring has changed the input to the next run without moving HEAD. The
|
|
22
|
+
* dirty list is `git status --porcelain` paths.
|
|
23
|
+
*/
|
|
24
|
+
export interface TreeSignature {
|
|
25
|
+
head: string;
|
|
26
|
+
dirty: string[];
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* How the process ended, which no reading of its TEXT can establish.
|
|
31
|
+
*
|
|
32
|
+
* `readSuiteRun` deliberately takes no exit code: #469's run exited 0 having
|
|
33
|
+
* measured nothing and run `39405680063` exited `null` having measured
|
|
34
|
+
* nothing, and nothing in either output says which way to resolve that. A
|
|
35
|
+
* SIGNAL is not ambiguous — but only the process's caller ever sees one, so
|
|
36
|
+
* it is carried here and read by the series.
|
|
37
|
+
*/
|
|
38
|
+
export interface RunExit {
|
|
39
|
+
status: number | null;
|
|
40
|
+
signal: string | null;
|
|
41
|
+
/** The spawn's own error, e.g. `ENOBUFS`, which names the cause. */
|
|
42
|
+
error: string | null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface FlakeRun {
|
|
46
|
+
/** 1-based, so a reason can name the run an operator will look for. */
|
|
47
|
+
index: number;
|
|
48
|
+
signature: TreeSignature;
|
|
49
|
+
run: SuiteRun;
|
|
50
|
+
/** Where the full output was kept. A failure that is not kept is #323. */
|
|
51
|
+
log: string;
|
|
52
|
+
seconds: number;
|
|
53
|
+
exit: RunExit;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Why a run's ENDING disqualifies it, or null.
|
|
58
|
+
*
|
|
59
|
+
* `bun test` exits 0 for a clean suite and 1 for a failing one. **1 must not
|
|
60
|
+
* be refused**: refusing every non-zero exit would turn every FLAKY and
|
|
61
|
+
* FAILING series into UNMEASURED and leave the instrument unable to report
|
|
62
|
+
* its own subject.
|
|
63
|
+
*/
|
|
64
|
+
export function exitRefusal(exit: RunExit): string | null {
|
|
65
|
+
if (exit.error !== null && exit.error !== "") {
|
|
66
|
+
return `the spawn failed: ${exit.error}`;
|
|
67
|
+
}
|
|
68
|
+
if (exit.signal !== null) {
|
|
69
|
+
return (
|
|
70
|
+
`it was killed by ${exit.signal}, so its log ends wherever the process did — and a truncated log ` +
|
|
71
|
+
"still parses as a clean reading when the output it captured contains a nested suite's summary, " +
|
|
72
|
+
"which is how run 39405680063 reported CONSISTENT over ten killed runs (#323)"
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
if (exit.status === null) {
|
|
76
|
+
return "the spawn reported no exit status and no signal, so nothing says how it ended";
|
|
77
|
+
}
|
|
78
|
+
if (exit.status !== 0 && exit.status !== 1) {
|
|
79
|
+
return `it exited ${exit.status}, which is not a verdict about the tests — bun uses 0 for a clean suite and 1 for a failing one`;
|
|
80
|
+
}
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* CONSISTENT, FLAKY and FAILING are three facts, not two.
|
|
86
|
+
*
|
|
87
|
+
* A suite that fails every run is red — an ordinary thing to be, and
|
|
88
|
+
* somebody's build to fix. A suite that fails SOME runs is the defect #323
|
|
89
|
+
* is about. One word for both would leave the series unable to report its
|
|
90
|
+
* own subject.
|
|
91
|
+
*/
|
|
92
|
+
export type FlakeVerdict = "CONSISTENT" | "FLAKY" | "FAILING" | "UNMEASURED";
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Whether the runs agreed about how much work they did.
|
|
96
|
+
*
|
|
97
|
+
* Reported BESIDE the verdict and never folded into it. The 2026-10-11
|
|
98
|
+
* reading's only surviving lead is that the failing run executed 384 more
|
|
99
|
+
* `expect()` calls than either clean one — so a varying count is a finding.
|
|
100
|
+
* It is not, on its own, evidence that anything is wrong: a case enumerating
|
|
101
|
+
* a directory legitimately varies. Folding it into the verdict would make
|
|
102
|
+
* one of those two readings unavailable.
|
|
103
|
+
*
|
|
104
|
+
* `UNREPORTED` is first-class. A runner that printed no count and a runner
|
|
105
|
+
* whose count held steady are opposite facts, and `STABLE` is the value that
|
|
106
|
+
* would let a reader treat the first as the second.
|
|
107
|
+
*/
|
|
108
|
+
export type AssertionStability = "STABLE" | "VARIED" | "UNREPORTED";
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* What `treeSignatureKey` can and cannot see, carried in the reading itself.
|
|
112
|
+
*
|
|
113
|
+
* A `CONSISTENT` verdict says the runs agreed over one tree, and a reader
|
|
114
|
+
* naturally hears "one set of inputs". It is not: the signature is HEAD plus
|
|
115
|
+
* `git status --porcelain`, so a gitignored path, anything under `/tmp` or
|
|
116
|
+
* `~/.rungate`, and `node_modules` can all change between runs with the
|
|
117
|
+
* signature unmoved.
|
|
118
|
+
*
|
|
119
|
+
* #799 is the measured instance rather than a worry. Series `39412289321`
|
|
120
|
+
* reported ten runs over one tree and was right to — while `test/phase-0.test.ts`
|
|
121
|
+
* collected its test list from a `/tmp` fixture the PREVIOUS run had left, so
|
|
122
|
+
* run 1 executed 15,194 tests and runs 2 through 10 executed 15,250.
|
|
123
|
+
*
|
|
124
|
+
* In the reading and not only in the printed sentence, because `flake.json`
|
|
125
|
+
* is what anybody reads months later, and a caveat that exists only in a
|
|
126
|
+
* console line is a caveat the artefact does not carry.
|
|
127
|
+
*/
|
|
128
|
+
export const SIGNATURE_SCOPE =
|
|
129
|
+
"the tree signature is HEAD plus `git status --porcelain`, so it covers the working tree and nothing " +
|
|
130
|
+
"else — a gitignored path, anything under /tmp or ~/.rungate, and node_modules can all differ between " +
|
|
131
|
+
"runs while this series reports one tree (#799)";
|
|
132
|
+
|
|
133
|
+
export interface FlakeReading {
|
|
134
|
+
verdict: FlakeVerdict;
|
|
135
|
+
/** Why it is not CONSISTENT. Empty exactly when it is. */
|
|
136
|
+
reasons: string[];
|
|
137
|
+
assertions: AssertionStability;
|
|
138
|
+
/** One entry per run, in order, `null` where the runner did not say. */
|
|
139
|
+
assertionCounts: Array<number | null>;
|
|
140
|
+
runs: number;
|
|
141
|
+
/** What the tree comparison covered. Always present; see `SIGNATURE_SCOPE`. */
|
|
142
|
+
signatureScope: string;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* One string per tree, so two readings can be compared without anybody
|
|
147
|
+
* deciding field by field what counts as the same tree.
|
|
148
|
+
*
|
|
149
|
+
* The dirty list is sorted: `git status --porcelain` order is not a property
|
|
150
|
+
* of the tree, and two runs differing only in it are one tree.
|
|
151
|
+
*/
|
|
152
|
+
export function treeSignatureKey(signature: TreeSignature): string {
|
|
153
|
+
return `${signature.head}\n${[...signature.dirty].sort().join("\n")}`;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** The runs that could be read at all. */
|
|
157
|
+
function measured(runs: FlakeRun[]) {
|
|
158
|
+
return runs.flatMap(r => (r.run.status === "MEASURED" ? [{ entry: r, run: r.run }] : []));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Read a series.
|
|
163
|
+
*
|
|
164
|
+
* FAILS CLOSED, like the single-run reader it sits on. Every path that
|
|
165
|
+
* cannot positively establish that `minimum` readable runs were taken over
|
|
166
|
+
* one tree returns UNMEASURED with reasons. There is no path that returns
|
|
167
|
+
* CONSISTENT from a series that does not say so.
|
|
168
|
+
*/
|
|
169
|
+
export function flakeReading(runs: FlakeRun[], minimum: number): FlakeReading {
|
|
170
|
+
const assertionCounts = runs.map(r => (r.run.status === "MEASURED" ? r.run.expectCalls : null));
|
|
171
|
+
const reasons: string[] = [];
|
|
172
|
+
|
|
173
|
+
// A short series first, because it explains every other absence. A job
|
|
174
|
+
// that died after three runs has measured three runs, and ten is what was
|
|
175
|
+
// asked for — the instrument failing is not the subject passing (#60).
|
|
176
|
+
if (runs.length < minimum) {
|
|
177
|
+
reasons.push(
|
|
178
|
+
`only ${runs.length} of ${minimum} run(s) completed, so the series was cut short — ` +
|
|
179
|
+
"a short series is not a clean series, however green the runs in it were (#323 SC-5)",
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Every run that did not FINISH, before anything is read off its text.
|
|
184
|
+
// This is the guard run 39405680063 did not have: ten children killed by
|
|
185
|
+
// SIGTERM on a 1 MiB `maxBuffer` overflow, ten truncated logs whose only
|
|
186
|
+
// surviving summary belonged to a nested suite, and a CONSISTENT verdict
|
|
187
|
+
// over a suite of 5,500 tests reporting one pass.
|
|
188
|
+
for (const r of runs) {
|
|
189
|
+
const why = exitRefusal(r.exit);
|
|
190
|
+
if (why !== null) reasons.push(`run ${r.index} did not finish (log ${r.log}): ${why}`);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// Every unreadable run, by index. An UNMEASURED run is not a clean run:
|
|
194
|
+
// `readSuiteRun` returns it for a log that printed no summary, and this
|
|
195
|
+
// repository's logs carry nested suites' summaries, so such a log still
|
|
196
|
+
// contains the text a `grep "0 fail"` would match.
|
|
197
|
+
for (const r of runs) {
|
|
198
|
+
if (r.run.status !== "MEASURED") {
|
|
199
|
+
reasons.push(`run ${r.index} could not be read (log ${r.log}): ${r.run.reason}`);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// One tree, or it is not a series. Named by what differs, because
|
|
204
|
+
// "the tree moved" tells nobody where to look and a scratch directory
|
|
205
|
+
// appearing under the repository is SC-6's whole hypothesis.
|
|
206
|
+
const trees = new Map<string, FlakeRun[]>();
|
|
207
|
+
for (const r of runs) {
|
|
208
|
+
const key = treeSignatureKey(r.signature);
|
|
209
|
+
trees.set(key, [...(trees.get(key) ?? []), r]);
|
|
210
|
+
}
|
|
211
|
+
if (trees.size > 1) {
|
|
212
|
+
const described = [...trees.entries()].map(([key, group]) => {
|
|
213
|
+
const where = group.map(g => g.index).join(", ");
|
|
214
|
+
const [head, ...dirty] = key.split("\n");
|
|
215
|
+
const paths = dirty.filter(d => d !== "");
|
|
216
|
+
return `run(s) ${where} at ${head}${paths.length === 0 ? " with a clean tree" : ` with ${paths.join(", ")}`}`;
|
|
217
|
+
});
|
|
218
|
+
reasons.push(
|
|
219
|
+
`the tree changed during the series, so these runs did not measure one thing: ${described.join("; ")} ` +
|
|
220
|
+
"— ten green runs over three trees is not ten green runs (#323 SC-5)",
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const counts = assertionCounts.filter((c): c is number => c !== null);
|
|
225
|
+
const assertions: AssertionStability =
|
|
226
|
+
counts.length !== runs.length || runs.length === 0
|
|
227
|
+
? "UNREPORTED"
|
|
228
|
+
: new Set(counts).size === 1
|
|
229
|
+
? "STABLE"
|
|
230
|
+
: "VARIED";
|
|
231
|
+
|
|
232
|
+
if (reasons.length > 0) {
|
|
233
|
+
return { verdict: "UNMEASURED", reasons, assertions, assertionCounts, runs: runs.length, signatureScope: SIGNATURE_SCOPE };
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const readable = measured(runs);
|
|
237
|
+
const failing = readable.filter(r => r.run.fail > 0);
|
|
238
|
+
if (failing.length > 0) {
|
|
239
|
+
for (const f of failing) {
|
|
240
|
+
reasons.push(`run ${f.entry.index} (log ${f.entry.log}): ${describeSuiteFailures(f.run)}`);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const verdict: FlakeVerdict =
|
|
245
|
+
failing.length === 0 ? "CONSISTENT" : failing.length === readable.length ? "FAILING" : "FLAKY";
|
|
246
|
+
|
|
247
|
+
return { verdict, reasons, assertions, assertionCounts, runs: runs.length, signatureScope: SIGNATURE_SCOPE };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** The sentence a caller prints instead of the verdict on its own. */
|
|
251
|
+
export function describeFlakeReading(reading: FlakeReading): string {
|
|
252
|
+
const { verdict, runs, assertions, assertionCounts, reasons } = reading;
|
|
253
|
+
const counts = assertionCounts.filter((c): c is number => c !== null);
|
|
254
|
+
const spread =
|
|
255
|
+
assertions === "VARIED"
|
|
256
|
+
? ` The runs did not agree about how much work they did: ${Math.min(...counts)} to ` +
|
|
257
|
+
`${Math.max(...counts)} expect() calls, a spread of ${Math.max(...counts) - Math.min(...counts)}. ` +
|
|
258
|
+
"That is the only surviving lead from the 2026-10-11 reading (#323 SC-4)."
|
|
259
|
+
: assertions === "UNREPORTED"
|
|
260
|
+
? " The runner did not report an assertion count for every run, so nothing is claimed about one."
|
|
261
|
+
: ` Every run executed ${counts[0]} expect() calls.`;
|
|
262
|
+
|
|
263
|
+
const failed = reasons.length;
|
|
264
|
+
const head =
|
|
265
|
+
verdict === "CONSISTENT"
|
|
266
|
+
? `CONSISTENT: ${runs} consecutive run(s) on one tree, none failing.`
|
|
267
|
+
: verdict === "FLAKY"
|
|
268
|
+
? `FLAKY: ${failed} of ${runs} consecutive run(s) on one tree failed — the same tree, different answers.`
|
|
269
|
+
: verdict === "FAILING"
|
|
270
|
+
? `FAILING: all ${runs} run(s) failed. The suite is red; this is not flake.`
|
|
271
|
+
: `UNMEASURED: the series does not support a verdict.`;
|
|
272
|
+
|
|
273
|
+
// The scope goes on EVERY verdict, not only CONSISTENT. A reader who sees
|
|
274
|
+
// it once beside a clean series and never again would reasonably conclude
|
|
275
|
+
// the other verdicts had looked harder, and they did not — it is the same
|
|
276
|
+
// comparison in all four cases.
|
|
277
|
+
return [head, ...reasons.map(r => ` - ${r}`), spread.trim(), `Scope: ${reading.signatureScope}`].join("\n");
|
|
278
|
+
}
|
package/lib/suite-measurement.ts
CHANGED
|
@@ -28,15 +28,21 @@ const SHA = /^[0-9a-f]{7,40}$/i;
|
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
30
|
* UNMEASURED is not a result. It is the statement that this run has no suite
|
|
31
|
-
* evidence, and it is deliberately not writable — see gates/schema.ts,
|
|
32
|
-
*
|
|
33
|
-
* saying there was no measurement.
|
|
31
|
+
* evidence, and it is deliberately not writable — see gates/schema.ts, whose
|
|
32
|
+
* writable verdicts are PASS, FAIL and PARTIAL. A recorded UNMEASURED would
|
|
33
|
+
* be a measurement saying there was no measurement.
|
|
34
|
+
*
|
|
35
|
+
* PARTIAL is writable and is not a verdict THIS type has, which is the whole
|
|
36
|
+
* of #673's first half: a reading taken over less than the suite is a reading
|
|
37
|
+
* that was taken, so the recorder may write it — and it is not a statement
|
|
38
|
+
* about the suite, so every consumer must see UNMEASURED. The conversion
|
|
39
|
+
* happens in `readSuiteMeasurement` below and nowhere else.
|
|
34
40
|
*/
|
|
35
41
|
export type SuiteVerdict = "PASS" | "FAIL" | "UNMEASURED";
|
|
36
42
|
|
|
37
43
|
/** What a recorder persists at `suiteMeasurement` in workflow-state.json. */
|
|
38
44
|
export interface SuiteMeasurement {
|
|
39
|
-
verdict: "PASS" | "FAIL";
|
|
45
|
+
verdict: "PASS" | "FAIL" | "PARTIAL";
|
|
40
46
|
/** The commit the suite actually ran against. */
|
|
41
47
|
measuredSha: string;
|
|
42
48
|
/** Failing tests. 0 for a PASS, at least 1 for a FAIL. */
|
|
@@ -136,6 +142,19 @@ export function readSuiteMeasurement(state: unknown): SuiteReading {
|
|
|
136
142
|
}
|
|
137
143
|
|
|
138
144
|
const verdict = raw.verdict;
|
|
145
|
+
// #673 — a reading the caller declared PARTIAL. Its own branch rather than
|
|
146
|
+
// falling through to the clause below, because "nobody recognises this
|
|
147
|
+
// word" and "this run measured some tests and said so" are different
|
|
148
|
+
// things to go and do something about: the first is a corrupt record and
|
|
149
|
+
// the second is a run that has not measured the suite YET.
|
|
150
|
+
if (verdict === "PARTIAL") {
|
|
151
|
+
return unmeasured(
|
|
152
|
+
`the suite measurement for ${measuredSha} was taken over PART of the suite: the caller ` +
|
|
153
|
+
`declared the run it recorded was less than the whole of it, so what it found is not a ` +
|
|
154
|
+
`statement about the suite (#673). ${describe(raw.tests)} test(s) collected is not a ` +
|
|
155
|
+
`reading a gate can act on, whatever the count`,
|
|
156
|
+
);
|
|
157
|
+
}
|
|
139
158
|
if (verdict !== "PASS" && verdict !== "FAIL") {
|
|
140
159
|
return unmeasured(
|
|
141
160
|
`the suite measurement for ${measuredSha} carries no usable verdict (got ${describe(verdict)}) — ` +
|