rungate 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -0
- package/gates/SCHEMA-GUIDE.md +485 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +283 -0
- package/gates/brief-assembler.test.ts +487 -0
- package/gates/brief-assembler.ts +448 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +250 -0
- package/gates/error-classifier.ts +99 -0
- package/gates/gate-executor.ts +3071 -0
- package/gates/orchestrator.test.ts +968 -0
- package/gates/orchestrator.ts +827 -0
- package/gates/preload.ts +89 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prompts.sha256 +3 -0
- package/gates/prove.test.ts +164 -0
- package/gates/rook-state.test.ts +108 -0
- package/gates/run-attribution.ts +290 -0
- package/gates/run-gate.ts +163 -0
- package/gates/schema-parity.test.ts +378 -0
- package/gates/schema.ts +1209 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +125 -0
- package/gates/ship-orchestrator.test.ts +399 -0
- package/gates/ship-orchestrator.ts +384 -0
- package/gates/spec-compliance-probe.ts +159 -0
- package/gates/test-fixtures/broken-no-environments.json +110 -0
- package/gates/test-fixtures/consumer-directory-layout.json +90 -0
- package/gates/test-fixtures/valid-standard.json +91 -0
- package/gates/test-utils.ts +43 -0
- package/gates/typecheck-gate.ts +111 -0
- package/gates/witness.ts +349 -0
- package/gates/workflow.test.ts +2018 -0
- package/hooks/ActionVerificationGate.hook.ts +96 -0
- package/hooks/AgentBriefGuard.hook.ts +38 -0
- package/hooks/AgentVerdictCapture.hook.ts +87 -0
- package/hooks/AutoVerifyGate.hook.ts +79 -0
- package/hooks/BashToolGuard.hook.ts +65 -0
- package/hooks/BashWriteGuard.hook.ts +60 -0
- package/hooks/CommitEnforcement.hook.ts +130 -0
- package/hooks/DocHygiene.hook.ts +148 -0
- package/hooks/GateEnforcement.hook.ts +58 -0
- package/hooks/GateResultProtection.hook.ts +95 -0
- package/hooks/HarnessAFKGate.hook.ts +85 -0
- package/hooks/HookCurrency.hook.ts +99 -0
- package/hooks/IssueCloseGuard.hook.ts +149 -0
- package/hooks/MergeGuard.hook.ts +58 -0
- package/hooks/PostCompact.hook.ts +60 -0
- package/hooks/ProjectStateSync.hook.ts +103 -0
- package/hooks/SkillGuard.hook.ts +69 -0
- package/hooks/SpecConformityTrigger.hook.ts +43 -0
- package/hooks/SpecSCGuard.hook.ts +123 -0
- package/hooks/StaleTTLCleanup.hook.ts +141 -0
- package/hooks/TaskCompleted.hook.ts +51 -0
- package/hooks/TeamInit.hook.ts +85 -0
- package/hooks/TestSuiteGuard.hook.ts +70 -0
- package/hooks/TestSuiteRelease.hook.ts +85 -0
- package/hooks/TopLevelCdGuard.hook.ts +108 -0
- package/hooks/VerifyPhaseLock.hook.ts +84 -0
- package/hooks/WorkflowStateGuard.hook.ts +64 -0
- package/hooks/lib/agent-config.json +9 -0
- package/hooks/lib/agentDetection.ts +201 -0
- package/hooks/lib/comment-template.ts +86 -0
- package/hooks/lib/findWorkflow.ts +101 -0
- package/hooks/lib/parseStdin.ts +37 -0
- package/hooks/lib/paths.ts +111 -0
- package/hooks/lib/utils.ts +338 -0
- package/lib/affected-tests.ts +29 -0
- package/lib/bash-write-guard.ts +627 -0
- package/lib/branch-cleanup.ts +279 -7
- package/lib/config-trackable.ts +9 -0
- package/lib/distribution-currency.ts +55 -5
- package/lib/failure-intake.ts +7 -1
- package/lib/gate-enforcement.ts +360 -38
- package/lib/gate-names.ts +72 -0
- package/lib/gate-result-protection.ts +437 -0
- package/lib/generators/code-map.ts +55 -4
- package/lib/generators/types.ts +27 -1
- package/lib/intake-sources.ts +380 -0
- package/lib/js-top-level-code.ts +20 -2
- package/lib/junit-checks.ts +239 -1
- package/lib/late-env-spawn.ts +331 -0
- package/lib/launch-base.ts +128 -5
- package/lib/package-closure.ts +114 -0
- package/lib/paths.ts +96 -3
- package/lib/phase2-decomposition.ts +1 -1
- package/lib/published-artifact.ts +162 -18
- package/lib/sc-id-cross-tree.ts +84 -8
- package/lib/scaffold/ai-review-findings.ts +149 -4
- package/lib/scaffold/ai-review-step.ts +1006 -132
- package/lib/scaffold/defaults.ts +82 -1
- package/lib/scaffold/steps.ts +642 -54
- package/lib/scanner.ts +4 -2
- package/lib/security-verdict.ts +1 -1
- package/lib/shell-path-interpolation.ts +594 -0
- package/lib/text-dependents.ts +875 -0
- package/lib/transcript-checker.ts +6 -0
- package/lib/workflow-security.ts +29 -0
- package/lib/workflow-taint.ts +732 -0
- package/lib/worktree-cleanup.ts +65 -38
- package/lib/worktree-repo.ts +32 -2
- package/package.json +4 -1
- package/scripts/check-installed-hook-currency.ts +825 -0
- package/scripts/generate-code-map.ts +13 -8
|
@@ -0,0 +1,3071 @@
|
|
|
1
|
+
import { execSync, execFileSync } from "child_process";
|
|
2
|
+
import { readFileSync, writeFileSync, existsSync, unlinkSync, mkdirSync, readdirSync } from "fs";
|
|
3
|
+
import { join } from "path";
|
|
4
|
+
import { splitGateRuns } from "../lib/gate-run-split";
|
|
5
|
+
import { gateSaltPath, workDir as slugWorkDir } from "../lib/paths";
|
|
6
|
+
import {
|
|
7
|
+
writeGateResult,
|
|
8
|
+
generateHmac,
|
|
9
|
+
generateShipEvidence,
|
|
10
|
+
runConvergence,
|
|
11
|
+
writeGateSummary,
|
|
12
|
+
markCommandlessACsAsSkip,
|
|
13
|
+
type GateResult,
|
|
14
|
+
} from "./orchestrator";
|
|
15
|
+
import { signCensus, signStampedCensus, writeWitness } from "./witness";
|
|
16
|
+
import { normalizeGitRefs } from "../lib/git-ref-normalizer";
|
|
17
|
+
import { recordEvidenceRewrite } from "../lib/evidence-command-transport";
|
|
18
|
+
import { GATE_SUITE_TIMEOUT_MS } from "../lib/gate-timeouts";
|
|
19
|
+
import { checksRunFrom, readJunitCheckNames } from "../lib/junit-checks";
|
|
20
|
+
import { safeParseProjectHarness, type ProjectHarness } from "../lib/rungate-schema";
|
|
21
|
+
import { DEFAULT_TEST_TIMEOUT, resolveConfig, type ResolvedConfig } from "../lib/config-loader";
|
|
22
|
+
import { censusOf, censusRegressions, TEST_CENSUS_READER } from "../lib/test-census";
|
|
23
|
+
import { readSuiteRun } from "../lib/suite-run-reading";
|
|
24
|
+
import {
|
|
25
|
+
slugExists,
|
|
26
|
+
shouldRebuildContainer,
|
|
27
|
+
shouldRefreshScaffold,
|
|
28
|
+
worktreeBranchName,
|
|
29
|
+
checkPortCollision,
|
|
30
|
+
trackPreExistingFailures,
|
|
31
|
+
releaseLock,
|
|
32
|
+
detectFileSetOverlap,
|
|
33
|
+
sequentialMergeOrder,
|
|
34
|
+
shouldRunCIVerification,
|
|
35
|
+
requiresResearchEscalation,
|
|
36
|
+
} from "./ship-orchestrator";
|
|
37
|
+
import {
|
|
38
|
+
measureEvidencePrevalidation,
|
|
39
|
+
type EvidencePrevalidationReading,
|
|
40
|
+
} from "../lib/evidence-prevalidator";
|
|
41
|
+
import { EvidencePrevalidationSchema, type PrevalidationSource } from "./schema";
|
|
42
|
+
import { scanGaps, type GapScanResult } from "../lib/gap-scanner";
|
|
43
|
+
import { deepMerge } from "../lib/deep-merge";
|
|
44
|
+
import { livenessPlan, probeTarget } from "../lib/dev-liveness";
|
|
45
|
+
import { livenessEvidenceContradictions } from "../lib/liveness-contradiction";
|
|
46
|
+
import { createGitHubClient, getIssue, addLabels } from "../lib/github";
|
|
47
|
+
import {
|
|
48
|
+
classifySpecComplianceRun,
|
|
49
|
+
execErrorOutput,
|
|
50
|
+
missingSpecComplianceFiles,
|
|
51
|
+
SPEC_COMPLIANCE_FILES,
|
|
52
|
+
} from "./spec-compliance-probe";
|
|
53
|
+
// ── Test output parser (moved here from run-gate.ts to break circular dep) ──
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Bun's timeout explanation, which is printed AFTER its own `(fail)` marker
|
|
57
|
+
* where every other kind of detail is printed before it (#550).
|
|
58
|
+
*
|
|
59
|
+
* Captured from bun 1.3.14 rather than written from memory:
|
|
60
|
+
* ` ^ this test timed out after 300ms.`
|
|
61
|
+
*/
|
|
62
|
+
const TIMEOUT_EXPLANATION = /^\s*\^\s*this test timed out after\b/;
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* What a failure's detail says when the runner printed no reason for it.
|
|
66
|
+
*
|
|
67
|
+
* It used to be the bare string `"failed"`, which is a refusal with no
|
|
68
|
+
* reason in the one field a heal prompt reads — and worse, it reads like
|
|
69
|
+
* something the runner said rather than like the parser giving up. On the
|
|
70
|
+
* run that produced #550 the reader went looking for a check that had
|
|
71
|
+
* reported "failed" and found a check whose output said `Type check PASS`.
|
|
72
|
+
*
|
|
73
|
+
* So it names the component that had nothing to say. The behaviour is
|
|
74
|
+
* unchanged; what changed is where it sends the next reader.
|
|
75
|
+
*/
|
|
76
|
+
const UNEXPLAINED_FAILURE =
|
|
77
|
+
"no explanation was captured for this failure — the runner printed nothing between this " +
|
|
78
|
+
"check's marker and the previous one, and gate-executor.ts's parseTestResults has no reason " +
|
|
79
|
+
"to report. The check failed; why it failed was not recorded (#550).";
|
|
80
|
+
|
|
81
|
+
export function parseTestResults(output: string): GateResult[] {
|
|
82
|
+
const results: GateResult[] = [];
|
|
83
|
+
const lines = output.split("\n");
|
|
84
|
+
for (let i = 0; i < lines.length; i++) {
|
|
85
|
+
const pm = lines[i].match(/\(pass\)\s+(.+?)(?:\s+\[|$)/);
|
|
86
|
+
if (pm) {
|
|
87
|
+
results.push({ check: pm[1].trim(), result: "PASS", detail: "passed" });
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const fm = lines[i].match(/\(fail\)\s+(.+?)(?:\s+\[|$)/);
|
|
91
|
+
if (fm) {
|
|
92
|
+
// Bun prints a failure's source excerpt, error and stack BEFORE its
|
|
93
|
+
// `(fail)` line, so the detail is scanned BACKWARD to the previous
|
|
94
|
+
// marker (#314). Scanning forward gave every entry the NEXT failure's
|
|
95
|
+
// detail and left the last one holding the run summary, which is what
|
|
96
|
+
// the heal prompt was being fed on run wf_5cdce852-1e2.
|
|
97
|
+
//
|
|
98
|
+
// #550: a TIMEOUT is the one exception, and it goes the other way.
|
|
99
|
+
// Bun prints `^ this test timed out after Nms.` AFTER the marker, so
|
|
100
|
+
// the backward scan from the NEXT failure walked over it and took it,
|
|
101
|
+
// while the test it belonged to was left with nothing. Observed live:
|
|
102
|
+
// `tsc-pass` recorded `detail: "failed"` and `local-api-validated`'s
|
|
103
|
+
// detail opened with a timeout that was not its own — which reads as a
|
|
104
|
+
// diagnosis of the wrong check, and was read that way.
|
|
105
|
+
const own: string[] = [];
|
|
106
|
+
for (let j = i + 1; j < lines.length && TIMEOUT_EXPLANATION.test(lines[j]); j++) {
|
|
107
|
+
own.push(lines[j].trim());
|
|
108
|
+
}
|
|
109
|
+
const detailLines: string[] = [];
|
|
110
|
+
for (let j = i - 1; j >= 0; j--) {
|
|
111
|
+
if (/\(pass\)\s+/.test(lines[j]) || /\(fail\)\s+/.test(lines[j])) break;
|
|
112
|
+
if (/^bun test v/.test(lines[j])) break;
|
|
113
|
+
// Belongs to the marker above it, not to this one.
|
|
114
|
+
if (TIMEOUT_EXPLANATION.test(lines[j])) break;
|
|
115
|
+
const trimmed = lines[j].trim();
|
|
116
|
+
if (trimmed) detailLines.unshift(trimmed);
|
|
117
|
+
}
|
|
118
|
+
const detail = [...own, ...detailLines];
|
|
119
|
+
results.push({
|
|
120
|
+
check: fm[1].trim(),
|
|
121
|
+
result: "FAIL",
|
|
122
|
+
detail: detail.length > 0 ? detail.join("\n") : UNEXPLAINED_FAILURE,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return results;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// ── Test baseline functions (#599) ───────────────────────────────────────
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* The failure count a runner's output STATES, or `null` when it states none.
|
|
133
|
+
*
|
|
134
|
+
* Uses the last "N fail" line (bun may print intermediate summaries).
|
|
135
|
+
*
|
|
136
|
+
* The nullable answer is the whole of #307's parser half. Bun prints ` 0 fail`
|
|
137
|
+
* on a clean run, so output carrying no fail line at all is a runner that
|
|
138
|
+
* crashed, timed out, or is not bun — and a consumer whose `dev.testCmd` is
|
|
139
|
+
* jest, pytest or a Makefile target produces exactly that on EVERY run. "The
|
|
140
|
+
* suite reported zero failures" and "nothing here reports failures" are
|
|
141
|
+
* different facts and only the first is a clean tree.
|
|
142
|
+
*/
|
|
143
|
+
export function parseTestFailureCount(testOutput: string): number | null {
|
|
144
|
+
const failMatches = [...testOutput.matchAll(/^\s*(\d+)\s+fail\s*$/gm)];
|
|
145
|
+
if (failMatches.length === 0) return null;
|
|
146
|
+
return parseInt(failMatches[failMatches.length - 1][1]);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Extract the failure count from bun test output, treating "no count stated"
|
|
151
|
+
* as zero.
|
|
152
|
+
*
|
|
153
|
+
* Kept exactly as it was, deliberately: `gates/workflow.test.ts` pins this
|
|
154
|
+
* contract and #307 is forbidden from changing the gate checks. The honest
|
|
155
|
+
* reading is `parseTestFailureCount` above, and the two are compared against
|
|
156
|
+
* each other in `test/test-baseline-unmeasured.test.ts` so this wrapper cannot
|
|
157
|
+
* drift away from the parser it wraps.
|
|
158
|
+
*/
|
|
159
|
+
export function extractTestFailureCount(testOutput: string): number {
|
|
160
|
+
return parseTestFailureCount(testOutput) ?? 0;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Subtract test baseline from total failures to get new-only failure count.
|
|
165
|
+
* Clamps to 0 — if baseline exceeds total (tests were fixed), result is 0.
|
|
166
|
+
*/
|
|
167
|
+
export function subtractTestBaseline(totalFailures: number, baseline: number): number {
|
|
168
|
+
return Math.max(0, totalFailures - baseline);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* What a run can honestly say about the tree it started from (#307 SC-A6).
|
|
173
|
+
*
|
|
174
|
+
* `measured` is the only reading that licenses "the suite was clean before
|
|
175
|
+
* this change". `unmeasured` carries the reason, because a baseline nobody
|
|
176
|
+
* took is news an operator can act on — usually by setting one key.
|
|
177
|
+
*/
|
|
178
|
+
export type TestBaselineReading =
|
|
179
|
+
| { status: "measured"; failures: number; command: string }
|
|
180
|
+
| { status: "unmeasured"; reason: string };
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Capture the test baseline by running the project's suite at scope time.
|
|
184
|
+
*
|
|
185
|
+
* This used to return `0` for four different facts, and the scope gate
|
|
186
|
+
* printed "0 pre-existing failures — clean suite" for all of them:
|
|
187
|
+
*
|
|
188
|
+
* 1. no `.claude/rungate.json` — the path was built here by hand, so every
|
|
189
|
+
* consumer on the documented `.claude/rungate/` layout took this branch
|
|
190
|
+
* 2. no `test.command` — including every consumer that authored
|
|
191
|
+
* `dev.testCmd`, which is the canonical spelling ADR-002 D3 normalises TO
|
|
192
|
+
* 3. the suite ran and printed something unparseable
|
|
193
|
+
* 4. the suite ran and nothing was failing
|
|
194
|
+
*
|
|
195
|
+
* #490 added a fifth, which the four above could not distinguish because the
|
|
196
|
+
* reading was taken off the FAIL line alone: **the suite ran and executed
|
|
197
|
+
* nothing.** A renamed directory or a glob that stopped matching prints
|
|
198
|
+
* `0 pass / 0 fail / Ran 0 tests across 1 file.` and exits 0, and that was a
|
|
199
|
+
* `measured` baseline of zero — the strongest claim available about a tree
|
|
200
|
+
* nobody looked at, with the verify gate subtracting against it. The reading
|
|
201
|
+
* goes through `lib/suite-run-reading.ts` now, which refuses it.
|
|
202
|
+
*
|
|
203
|
+
* Only the fourth is a clean tree, and a fabricated baseline is the worst
|
|
204
|
+
* direction for this particular number to be wrong in: it asserts the
|
|
205
|
+
* strongest claim available about the tree, and the verify gate subtracts
|
|
206
|
+
* against it.
|
|
207
|
+
*
|
|
208
|
+
* It worked on rungate the whole time, which is why nothing caught it — this
|
|
209
|
+
* repo's own config carries `test.command`, so branch 2 was never taken here.
|
|
210
|
+
* See `test/test-baseline-unmeasured.test.ts` for the consumer-shaped
|
|
211
|
+
* fixtures that do take it.
|
|
212
|
+
*
|
|
213
|
+
* The timeout now comes from the resolved config rather than from a `|| 300000`
|
|
214
|
+
* that disagreed with `DEFAULT_TEST_TIMEOUT`. A consumer that sets no
|
|
215
|
+
* `test.timeout` therefore gets the shorter one — and a suite killed by it
|
|
216
|
+
* reports UNMEASURED naming the timeout, where before it reported a clean
|
|
217
|
+
* tree. The failure mode moved from silent and wrong to loud and fixable.
|
|
218
|
+
*/
|
|
219
|
+
export function captureTestBaseline(state: Record<string, any>): TestBaselineReading {
|
|
220
|
+
const projectRoot = state.projectRoot || process.cwd();
|
|
221
|
+
|
|
222
|
+
const resolution = resolveConfig(projectRoot);
|
|
223
|
+
if (resolution.status === "absent") {
|
|
224
|
+
return {
|
|
225
|
+
status: "unmeasured",
|
|
226
|
+
reason: `no rungate config under ${join(projectRoot, ".claude")} — nothing says how to run this project's suite`,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
const testCmd = resolution.config.test?.command;
|
|
231
|
+
if (!testCmd) {
|
|
232
|
+
// Both spellings named on purpose. "No test command" sends an operator
|
|
233
|
+
// looking for the key they already did not set.
|
|
234
|
+
return {
|
|
235
|
+
status: "unmeasured",
|
|
236
|
+
reason:
|
|
237
|
+
`the ${resolution.source} config at ${resolution.path} sets neither ` +
|
|
238
|
+
`dev.testCmd nor test.command`,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
const timeout = resolution.config.test?.timeout ?? DEFAULT_TEST_TIMEOUT;
|
|
243
|
+
console.log(`Test baseline: running "${testCmd}" to capture pre-existing failures...`);
|
|
244
|
+
|
|
245
|
+
let output: string;
|
|
246
|
+
try {
|
|
247
|
+
output = execSync(`${testCmd} 2>&1`, { encoding: "utf-8", timeout, cwd: projectRoot });
|
|
248
|
+
} catch (e: any) {
|
|
249
|
+
// A suite with failures exits non-zero, so the count normally arrives
|
|
250
|
+
// through here. `e.stdout` is also empty on a timeout kill, which the
|
|
251
|
+
// parser below turns into UNMEASURED rather than into a clean tree.
|
|
252
|
+
output = e.stdout || "";
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
const run = readSuiteRun(output);
|
|
256
|
+
if (run.status !== "MEASURED") {
|
|
257
|
+
return {
|
|
258
|
+
status: "unmeasured",
|
|
259
|
+
reason: `"${testCmd}" did not measure this project (timeout ${timeout}ms) — ${run.reason}`,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
return { status: "measured", failures: run.fail, command: testCmd };
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Write a reading into `gateContract.baselines`, in place (#307).
|
|
268
|
+
*
|
|
269
|
+
* Pure and exported so the propagation is EXECUTABLE. The reading being
|
|
270
|
+
* honest buys nothing if the two lines that record it quietly write a `0`,
|
|
271
|
+
* and a `captureTestBaseline` test cannot see that.
|
|
272
|
+
*
|
|
273
|
+
* The two keys are mutually exclusive, enforced by deleting the other one
|
|
274
|
+
* each way. A reason sitting beside a number would let `?? 0` keep reading
|
|
275
|
+
* the run as a measured clean tree — which is the defect the reason exists
|
|
276
|
+
* to report, reintroduced one field over.
|
|
277
|
+
*/
|
|
278
|
+
export function recordTestBaseline(
|
|
279
|
+
baselines: Record<string, any>,
|
|
280
|
+
reading: TestBaselineReading,
|
|
281
|
+
): void {
|
|
282
|
+
if (reading.status === "measured") {
|
|
283
|
+
baselines.testBaseline = reading.failures;
|
|
284
|
+
delete baselines.testBaselineUnmeasured;
|
|
285
|
+
} else {
|
|
286
|
+
delete baselines.testBaseline;
|
|
287
|
+
baselines.testBaselineUnmeasured = reading.reason;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** What a gate run does about the tests that existed when it last looked. */
|
|
292
|
+
export type CensusVerdict =
|
|
293
|
+
| { kind: "unmeasured"; reason: string }
|
|
294
|
+
/**
|
|
295
|
+
* There is a banked census, and it is not one this gate wrote (#494).
|
|
296
|
+
*
|
|
297
|
+
* Its own arm rather than a second `unmeasured`, because the two are
|
|
298
|
+
* different facts about the run and only one of them is about an agent:
|
|
299
|
+
* `unmeasured` is a tree with no suite the walk can see, and this is a
|
|
300
|
+
* reading that was taken and then went missing. Both are WARNs — see
|
|
301
|
+
* `censusReport`, which is the single place that decides.
|
|
302
|
+
*/
|
|
303
|
+
| { kind: "census-unverified"; reason: string }
|
|
304
|
+
/**
|
|
305
|
+
* The banked census was taken by a different reader, so the two numbers
|
|
306
|
+
* are not comparable (#883). UNMEASURED, and NOT a regression: every
|
|
307
|
+
* file whose count moved because the instrument changed would otherwise
|
|
308
|
+
* read as a deleted test on the first round after a reader bump.
|
|
309
|
+
*/
|
|
310
|
+
| { kind: "reader-changed"; reason: string }
|
|
311
|
+
| { kind: "first-reading"; files: number }
|
|
312
|
+
| { kind: "no-regression"; files: number }
|
|
313
|
+
| { kind: "regression"; regressions: string[] };
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* What the state file holds, and whether this gate is the one that put it
|
|
317
|
+
* there (#494).
|
|
318
|
+
*
|
|
319
|
+
* Three readings, not two. `absent` and `unverified` both mean "nothing to
|
|
320
|
+
* compare against", and collapsing them would be fine for the comparison
|
|
321
|
+
* and wrong for the report: an absent census on a run whose gate has never
|
|
322
|
+
* executed is ordinary, and the identical absence one round later is the
|
|
323
|
+
* erasure this issue exists to detect. The pair that tells them apart is
|
|
324
|
+
* this reading and `PriorGateRun`.
|
|
325
|
+
*/
|
|
326
|
+
export type PersistedCensus =
|
|
327
|
+
| { kind: "verified"; census: Record<string, number> }
|
|
328
|
+
| { kind: "unverified"; reason: string }
|
|
329
|
+
/**
|
|
330
|
+
* A census this gate did write, taken by a DIFFERENT reader (#883).
|
|
331
|
+
*
|
|
332
|
+
* Its own arm and not an `unverified`, because the two are opposite
|
|
333
|
+
* facts about the same file: `unverified` is a reading that went missing
|
|
334
|
+
* or was rewritten, and this is a reading that is intact and simply not
|
|
335
|
+
* comparable. Only one of them is about an agent, and only one of them
|
|
336
|
+
* should bank the new number — see `nextCensus`.
|
|
337
|
+
*/
|
|
338
|
+
| { kind: "stale-reader"; reason: string; census: Record<string, number> }
|
|
339
|
+
| { kind: "absent" };
|
|
340
|
+
|
|
341
|
+
/**
|
|
342
|
+
* Whether a gate has already run for this slug (#494).
|
|
343
|
+
*
|
|
344
|
+
* The signal has to live OUTSIDE `workflow-state.json`: an agent erasing
|
|
345
|
+
* the census erases the signature with it, so the signature cannot say
|
|
346
|
+
* whether there was ever anything to erase. `~/.rungate/<slug>/witnesses/`
|
|
347
|
+
* is the existing store, written by `writeWitness` on every gate run, and
|
|
348
|
+
* no prompt in this repository names it.
|
|
349
|
+
*
|
|
350
|
+
* `unreadable` is the arm that keeps this from having the same off switch
|
|
351
|
+
* one layer down. `writeWitness` is wrapped in a try/catch at the bottom of
|
|
352
|
+
* `runGateInternal` that only logs, and it throws when the gate salt is
|
|
353
|
+
* absent — so on a machine with no salt no witness is ever written, and
|
|
354
|
+
* "no witnesses" would mean "every round is a first round" forever. A
|
|
355
|
+
* witness store that cannot be trusted to have recorded an earlier round is
|
|
356
|
+
* UNMEASURED, not a first run.
|
|
357
|
+
*/
|
|
358
|
+
export type PriorGateRun =
|
|
359
|
+
| { kind: "ran"; witnesses: number }
|
|
360
|
+
| { kind: "first" }
|
|
361
|
+
| { kind: "unreadable"; reason: string };
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* Read the witness store for evidence that a gate already ran (#494).
|
|
365
|
+
*
|
|
366
|
+
* The salt check comes FIRST and deliberately: an empty witness directory
|
|
367
|
+
* on a machine that cannot write witnesses is indistinguishable from an
|
|
368
|
+
* empty one on a machine that simply has not run yet, and only one of those
|
|
369
|
+
* is a first round.
|
|
370
|
+
*/
|
|
371
|
+
export function priorGateRun(
|
|
372
|
+
slug: string,
|
|
373
|
+
opts: { saltPath?: string; witnessDir?: string } = {},
|
|
374
|
+
): PriorGateRun {
|
|
375
|
+
const salt = opts.saltPath ?? gateSaltPath();
|
|
376
|
+
if (!existsSync(salt)) {
|
|
377
|
+
return {
|
|
378
|
+
kind: "unreadable",
|
|
379
|
+
reason:
|
|
380
|
+
`the gate salt at ${salt} is absent, so no witness can be written and an earlier ` +
|
|
381
|
+
`gate run of this slug cannot be ruled out`,
|
|
382
|
+
};
|
|
383
|
+
}
|
|
384
|
+
let dir: string;
|
|
385
|
+
try {
|
|
386
|
+
dir = opts.witnessDir ?? join(slugWorkDir(slug), "witnesses");
|
|
387
|
+
} catch (e: any) {
|
|
388
|
+
return { kind: "unreadable", reason: `the witness directory for slug "${slug}" could not be resolved: ${e.message}` };
|
|
389
|
+
}
|
|
390
|
+
if (!existsSync(dir)) return { kind: "first" };
|
|
391
|
+
try {
|
|
392
|
+
const witnesses = readdirSync(dir).filter(f => f.endsWith(".json")).length;
|
|
393
|
+
return witnesses > 0 ? { kind: "ran", witnesses } : { kind: "first" };
|
|
394
|
+
} catch (e: any) {
|
|
395
|
+
return { kind: "unreadable", reason: `${dir} could not be listed: ${e.message}` };
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Whether tests disappeared since the last gate run (#307 SC-A19).
|
|
401
|
+
*
|
|
402
|
+
* Every heal round ends in a gate re-run, so "since the last gate run" and
|
|
403
|
+
* "across a heal" are the same window. A heal agent is told a gate failed and
|
|
404
|
+
* asked to make it pass; deleting the test is a correct solution to that
|
|
405
|
+
* instruction and an incorrect solution to the problem, and nothing
|
|
406
|
+
* downstream can tell, because a suite with the failing test removed is green
|
|
407
|
+
* and green is what it asked for. Run `wf_5cdce852-1e2` went from 34 tests to
|
|
408
|
+
* 23 across two rounds with no `skip` and no `todo` recorded.
|
|
409
|
+
*
|
|
410
|
+
* `first-reading` is NOT a pass. There is nothing to compare on the first
|
|
411
|
+
* gate run of a run, and a verdict that said "no regression" there would be
|
|
412
|
+
* the reading nobody took wearing the answer's clothes — the whole subject of
|
|
413
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`.
|
|
414
|
+
*
|
|
415
|
+
* Neither is `unmeasured`, and it is the arm that keeps this guard alive on a
|
|
416
|
+
* project that is not this one. A tree with no recognisable test file
|
|
417
|
+
* censuses to `{}`, `{}` compares clean against `{}` on every subsequent
|
|
418
|
+
* round, and the guard would be permanently and silently dead for exactly the
|
|
419
|
+
* repos it exists to protect — a polyglot consumer, or one whose suite the
|
|
420
|
+
* walk cannot see. Reported, every run, rather than inferred from silence.
|
|
421
|
+
*
|
|
422
|
+
* And neither is `census-unverified` (#494). A banked census that this gate
|
|
423
|
+
* did not sign is not a census; where a gate run already happened, reading
|
|
424
|
+
* it as `first-reading` is the erasure wearing the first round's clothes.
|
|
425
|
+
* The two facts are deliberately decided together here, because neither one
|
|
426
|
+
* answers the question alone: the signature cannot say whether there was
|
|
427
|
+
* ever anything to erase, and the witness count cannot say whether what is
|
|
428
|
+
* there now is what was banked.
|
|
429
|
+
*/
|
|
430
|
+
export function censusVerdict(
|
|
431
|
+
previous: PersistedCensus,
|
|
432
|
+
current: Record<string, number>,
|
|
433
|
+
prior: PriorGateRun,
|
|
434
|
+
): CensusVerdict {
|
|
435
|
+
const files = Object.keys(current).length;
|
|
436
|
+
if (files === 0) {
|
|
437
|
+
return {
|
|
438
|
+
kind: "unmeasured",
|
|
439
|
+
reason:
|
|
440
|
+
"no .test/.spec file in the JS/TS family was found in the tree under test, " +
|
|
441
|
+
"so a heal round that deleted one could not be seen",
|
|
442
|
+
};
|
|
443
|
+
}
|
|
444
|
+
if (previous.kind === "verified" && Object.keys(previous.census).length > 0) {
|
|
445
|
+
const regressions = censusRegressions(previous.census, current);
|
|
446
|
+
return regressions.length > 0 ? { kind: "regression", regressions } : { kind: "no-regression", files };
|
|
447
|
+
}
|
|
448
|
+
// An intact census from another reader is not "nothing to compare
|
|
449
|
+
// against" — it is something that must NOT be compared. Decided before
|
|
450
|
+
// the witness question below, which is about erasure and has no bearing
|
|
451
|
+
// on an instrument change.
|
|
452
|
+
if (previous.kind === "stale-reader") {
|
|
453
|
+
// …but say what the comparison WOULD have reported. A reader change
|
|
454
|
+
// moves almost every file's number, so refusing on the drops would make
|
|
455
|
+
// the migration never end — that is what `reader-changed` exists to
|
|
456
|
+
// avoid. What it must not also do is stay silent, because then a round
|
|
457
|
+
// that genuinely deleted a test is indistinguishable from the
|
|
458
|
+
// instrument change it is hiding behind, and `nextCensus` banks the
|
|
459
|
+
// lowered reading as the new baseline.
|
|
460
|
+
const wouldHaveBeen = censusRegressions(previous.census, current);
|
|
461
|
+
const named = wouldHaveBeen.slice(0, 5).join("; ");
|
|
462
|
+
const drops =
|
|
463
|
+
wouldHaveBeen.length === 0
|
|
464
|
+
? " No file counted fewer tests across the change."
|
|
465
|
+
: ` ${wouldHaveBeen.length} file(s) counted FEWER tests than the banked reading, which ` +
|
|
466
|
+
`this round cannot tell from deletions: ${named}` +
|
|
467
|
+
`${wouldHaveBeen.length > 5 ? `; and ${wouldHaveBeen.length - 5} more` : ""}.`;
|
|
468
|
+
return { kind: "reader-changed", reason: previous.reason + "." + drops };
|
|
469
|
+
}
|
|
470
|
+
// Nothing to compare against. Whether that is ordinary depends entirely on
|
|
471
|
+
// whether a gate has run for this slug before — the one fact the state
|
|
472
|
+
// file cannot be asked, because an agent erasing the census erases
|
|
473
|
+
// everything in it.
|
|
474
|
+
if (prior.kind === "first") return { kind: "first-reading", files };
|
|
475
|
+
const why = previous.kind === "unverified" ? previous.reason : "no census was banked in the state file";
|
|
476
|
+
const since =
|
|
477
|
+
prior.kind === "ran"
|
|
478
|
+
? `${prior.witnesses} witness(es) of an earlier gate run for this slug exist`
|
|
479
|
+
: `whether an earlier gate run happened could not be read — ${prior.reason}`;
|
|
480
|
+
return {
|
|
481
|
+
kind: "census-unverified",
|
|
482
|
+
reason:
|
|
483
|
+
`the banked test census could not be verified (${why}), and ${since}. A round that ` +
|
|
484
|
+
`deleted a test and the census with it cannot be told from a first reading, so this ` +
|
|
485
|
+
`round is UNMEASURED rather than clean (#494)`,
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
/**
|
|
490
|
+
* The census this gate is willing to compare against (#494).
|
|
491
|
+
*
|
|
492
|
+
* `censusVerdict` decides what the round IS; this decides what gets banked
|
|
493
|
+
* for the next one, and the two are separate questions asked of the same
|
|
494
|
+
* reading. An unverified census is never carried forward — `nextCensus`
|
|
495
|
+
* would otherwise keep an attacker's number as "the previous reading".
|
|
496
|
+
*/
|
|
497
|
+
export function verifiedCensus(previous: PersistedCensus): Record<string, number> | undefined {
|
|
498
|
+
return previous.kind === "verified" ? previous.census : undefined;
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/** What the gate counts and prints for a census verdict. */
|
|
502
|
+
export interface CensusReport {
|
|
503
|
+
fails: number;
|
|
504
|
+
warns: number;
|
|
505
|
+
results: Array<{ check: string; result: "FAIL" | "WARN"; detail: string }>;
|
|
506
|
+
console: Array<{ level: "error" | "warn" | "log"; line: string }>;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Turn a verdict into what the gate counts (#307 SC-A19, #494).
|
|
511
|
+
*
|
|
512
|
+
* One binding, so there is one thing to mutate and one place where an arm
|
|
513
|
+
* can be forgotten. The arm this issue adds is a WARN and not a FAIL on
|
|
514
|
+
* purpose: a legitimate run on a machine whose witness store cannot be read
|
|
515
|
+
* reaches it, and a check that refuses rounds nobody can fix is removed
|
|
516
|
+
* within a week. A WARN that names the state is the difference between
|
|
517
|
+
* "unmeasured" and "clean", which is the whole claim.
|
|
518
|
+
*/
|
|
519
|
+
export function censusReport(verdict: CensusVerdict, censusRoot: string): CensusReport {
|
|
520
|
+
if (verdict.kind === "regression") {
|
|
521
|
+
return {
|
|
522
|
+
fails: 1,
|
|
523
|
+
warns: 0,
|
|
524
|
+
results: [
|
|
525
|
+
{
|
|
526
|
+
check: "test-census: a round removed tests",
|
|
527
|
+
result: "FAIL",
|
|
528
|
+
detail:
|
|
529
|
+
`${verdict.regressions.join("; ")}. A gate that passes because its test is gone ` +
|
|
530
|
+
`is not a gate that passed. Restore the test, or say in the PR which check ` +
|
|
531
|
+
`replaced it (#307 SC-A19).`,
|
|
532
|
+
},
|
|
533
|
+
],
|
|
534
|
+
console: [{ level: "error", line: `Test census: REFUSED — ${verdict.regressions.join("; ")}` }],
|
|
535
|
+
};
|
|
536
|
+
}
|
|
537
|
+
// An empty census compares clean against an empty census on every
|
|
538
|
+
// subsequent round, and an erased one reads as a first round. Both are
|
|
539
|
+
// readings nobody took, so both are reported rather than inferred from
|
|
540
|
+
// silence.
|
|
541
|
+
if (
|
|
542
|
+
verdict.kind === "unmeasured" ||
|
|
543
|
+
verdict.kind === "census-unverified" ||
|
|
544
|
+
verdict.kind === "reader-changed"
|
|
545
|
+
) {
|
|
546
|
+
return {
|
|
547
|
+
fails: 0,
|
|
548
|
+
warns: 1,
|
|
549
|
+
results: [
|
|
550
|
+
{
|
|
551
|
+
check: "test-census: UNMEASURED",
|
|
552
|
+
result: "WARN",
|
|
553
|
+
detail: `${verdict.reason} (tree: ${censusRoot})`,
|
|
554
|
+
},
|
|
555
|
+
],
|
|
556
|
+
console: [{ level: "warn", line: `Test census: UNMEASURED — ${verdict.reason}` }],
|
|
557
|
+
};
|
|
558
|
+
}
|
|
559
|
+
if (verdict.kind === "first-reading") {
|
|
560
|
+
// Said out loud rather than passed quietly: a run with nothing to
|
|
561
|
+
// compare has not been checked, and silence is how that becomes
|
|
562
|
+
// "checked and clean".
|
|
563
|
+
return {
|
|
564
|
+
fails: 0,
|
|
565
|
+
warns: 0,
|
|
566
|
+
results: [],
|
|
567
|
+
console: [
|
|
568
|
+
{
|
|
569
|
+
level: "log",
|
|
570
|
+
line: `Test census: first reading this run — ${verdict.files} test file(s), nothing to compare yet`,
|
|
571
|
+
},
|
|
572
|
+
],
|
|
573
|
+
};
|
|
574
|
+
}
|
|
575
|
+
return {
|
|
576
|
+
fails: 0,
|
|
577
|
+
warns: 0,
|
|
578
|
+
results: [],
|
|
579
|
+
console: [{ level: "log", line: `Test census: no test was removed (${verdict.files} file(s) compared)` }],
|
|
580
|
+
};
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
/**
|
|
584
|
+
* The reading to carry into the next gate run (#307 SC-A19).
|
|
585
|
+
*
|
|
586
|
+
* A regression does NOT bank the reduced census. Banking it would make the
|
|
587
|
+
* next round compare against the tree the deletion produced, so a second
|
|
588
|
+
* round would find no regression and the refusal would clear itself — a
|
|
589
|
+
* guard that forgets what it refused is a guard you only have to get past
|
|
590
|
+
* once.
|
|
591
|
+
*/
|
|
592
|
+
/**
|
|
593
|
+
* Write the banked census into the state FILE (#307 SC-A19).
|
|
594
|
+
*
|
|
595
|
+
* Found by review, and without it the guard was permanently dead. Setting
|
|
596
|
+
* `state.gateContract.testCensus` in memory reaches disk only by accident:
|
|
597
|
+
* `writeGateResult` further down does its own `readState(sf)` and writes
|
|
598
|
+
* THAT back, so the in-memory mutation is discarded. Every subsequent gate
|
|
599
|
+
* run would have found no census, answered `first-reading`, logged a
|
|
600
|
+
* reassuring line, and compared nothing — forever.
|
|
601
|
+
*
|
|
602
|
+
* Read-modify-write against the file rather than a whole-state write,
|
|
603
|
+
* because the gate tests and any heal agent have been writing to it since
|
|
604
|
+
* this process loaded `state`, and clobbering their edits to record a census
|
|
605
|
+
* trades one silent loss for another.
|
|
606
|
+
*
|
|
607
|
+
* Signed, since #494. The signature goes beside the census in the same
|
|
608
|
+
* file — an agent can delete both, which is the point: it cannot delete
|
|
609
|
+
* them and still look like a first round, because the witness store says a
|
|
610
|
+
* gate already ran.
|
|
611
|
+
*
|
|
612
|
+
* A salt this machine does not have is NOT an excuse to write an unsigned
|
|
613
|
+
* census quietly. The census is still banked, because the comparison is
|
|
614
|
+
* worth having even unverified, and the stale signature of a previous
|
|
615
|
+
* reading is DELETED rather than left beside a census it does not cover —
|
|
616
|
+
* a signature that verifies the wrong census is worse than none. The next
|
|
617
|
+
* round reads the result as UNMEASURED.
|
|
618
|
+
*/
|
|
619
|
+
export function persistTestCensus(
|
|
620
|
+
stateFilePath: string,
|
|
621
|
+
census: Record<string, number>,
|
|
622
|
+
slug: string,
|
|
623
|
+
opts: { saltPath?: string } = {},
|
|
624
|
+
): void {
|
|
625
|
+
const fresh = JSON.parse(readFileSync(stateFilePath, "utf-8"));
|
|
626
|
+
fresh.gateContract = fresh.gateContract || {};
|
|
627
|
+
fresh.gateContract.testCensus = census;
|
|
628
|
+
// Stamped in the same write as the census it describes, and SIGNED with
|
|
629
|
+
// it. A stamp written separately could land without the reading or vice
|
|
630
|
+
// versa, and either half alone says the wrong thing about the other; a
|
|
631
|
+
// stamp outside the signature is a one-byte way past the #494 refusal,
|
|
632
|
+
// because `reader-changed` warns and banks instead of refusing.
|
|
633
|
+
fresh.gateContract.testCensusReader = TEST_CENSUS_READER;
|
|
634
|
+
try {
|
|
635
|
+
fresh.gateContract.testCensusSignature = signStampedCensus(
|
|
636
|
+
slug,
|
|
637
|
+
census,
|
|
638
|
+
TEST_CENSUS_READER,
|
|
639
|
+
opts.saltPath,
|
|
640
|
+
);
|
|
641
|
+
} catch (e: any) {
|
|
642
|
+
delete fresh.gateContract.testCensusSignature;
|
|
643
|
+
console.warn(
|
|
644
|
+
`Test census: banked UNSIGNED — ${e.message}. The next gate run will read it as ` +
|
|
645
|
+
`UNMEASURED rather than as a census this gate wrote (#494).`,
|
|
646
|
+
);
|
|
647
|
+
}
|
|
648
|
+
writeFileSync(stateFilePath, JSON.stringify(fresh, null, 2));
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
/**
|
|
652
|
+
* The census the last gate run banked, read off DISK (#307 SC-A19).
|
|
653
|
+
*
|
|
654
|
+
* Not off the in-memory `state`, which was loaded when the process started:
|
|
655
|
+
* the gate tests and any heal agent have been writing the file since, and
|
|
656
|
+
* comparing against a stale copy and then persisting over the fresh one is
|
|
657
|
+
* how a round's reading quietly disappears. The direction it disappears in
|
|
658
|
+
* is toward not refusing.
|
|
659
|
+
*
|
|
660
|
+
* Verified, since #494. Every way of not being a census this gate wrote is
|
|
661
|
+
* one reading — `unverified`, carrying the reason — and a state file with
|
|
662
|
+
* nothing banked at all is another. The distinction is not cosmetic: an
|
|
663
|
+
* absent census before the first gate run is ordinary, and the identical
|
|
664
|
+
* absence afterwards is the erasure. `censusVerdict` is where the two are
|
|
665
|
+
* told apart, using the witness store.
|
|
666
|
+
*
|
|
667
|
+
* Neither reading is a refusal on its own. A file that cannot be read or
|
|
668
|
+
* parsed on a run whose gate has never executed is still a first reading —
|
|
669
|
+
* said out loud rather than passed quietly.
|
|
670
|
+
*/
|
|
671
|
+
/**
|
|
672
|
+
* A reader stamp as it is allowed to appear in a report.
|
|
673
|
+
*
|
|
674
|
+
* The stamp is inside the signature, so nothing unsigned reaches this — but
|
|
675
|
+
* a value that renders a newline or a control character into a WARN detail
|
|
676
|
+
* is a report an operator cannot read, and the cost of refusing to print it
|
|
677
|
+
* is one line. One binding, so there is one thing to mutate.
|
|
678
|
+
*/
|
|
679
|
+
export function renderReaderStamp(reader: string | null): string {
|
|
680
|
+
if (reader === null) return "1 (unstamped, pre-#883)";
|
|
681
|
+
return /^[A-Za-z0-9._-]{1,16}$/.test(reader) ? `"${reader}"` : "(unprintable)";
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
export function readPersistedCensus(stateFilePath: string, slug: string): PersistedCensus {
|
|
685
|
+
let onDisk: any;
|
|
686
|
+
try {
|
|
687
|
+
onDisk = JSON.parse(readFileSync(stateFilePath, "utf-8"));
|
|
688
|
+
} catch (e: any) {
|
|
689
|
+
return { kind: "unverified", reason: `the state file could not be read or parsed: ${e.message}` };
|
|
690
|
+
}
|
|
691
|
+
const census = onDisk?.gateContract?.testCensus;
|
|
692
|
+
const signature = onDisk?.gateContract?.testCensusSignature;
|
|
693
|
+
if (census === undefined && signature === undefined) return { kind: "absent" };
|
|
694
|
+
if (!census || typeof census !== "object" || Array.isArray(census)) {
|
|
695
|
+
return { kind: "unverified", reason: "the banked census is not an object" };
|
|
696
|
+
}
|
|
697
|
+
if (typeof signature !== "string" || signature.length === 0) {
|
|
698
|
+
return { kind: "unverified", reason: "the banked census carries no signature" };
|
|
699
|
+
}
|
|
700
|
+
// The reader stamp is INSIDE the signature when it is present, so a
|
|
701
|
+
// forged or deleted stamp reads as a rewrite rather than as a migration
|
|
702
|
+
// (#883). An absent stamp is checked against the v1 payload, which is the
|
|
703
|
+
// only shape a genuinely pre-#883 state file has — and the only one an
|
|
704
|
+
// agent cannot mint, because minting it means removing a signature it
|
|
705
|
+
// cannot recompute.
|
|
706
|
+
const rawReader = onDisk?.gateContract?.testCensusReader;
|
|
707
|
+
const reader = typeof rawReader === "string" && rawReader.length > 0 ? rawReader : null;
|
|
708
|
+
let expected: string;
|
|
709
|
+
try {
|
|
710
|
+
expected =
|
|
711
|
+
reader === null ? signCensus(slug, census) : signStampedCensus(slug, census, reader);
|
|
712
|
+
} catch (e: any) {
|
|
713
|
+
return { kind: "unverified", reason: `the census signature could not be checked: ${e.message}` };
|
|
714
|
+
}
|
|
715
|
+
if (expected !== signature) {
|
|
716
|
+
return {
|
|
717
|
+
kind: "unverified",
|
|
718
|
+
reason:
|
|
719
|
+
"the banked census or its reader stamp does not match its signature — " +
|
|
720
|
+
"it was rewritten after the gate banked it",
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
// Asked LAST, and the order is the reading. A census that fails the
|
|
724
|
+
// signature is a tamper whatever instrument took it; a census that passes
|
|
725
|
+
// it and was taken by another reader is intact and merely incomparable.
|
|
726
|
+
// Asked first, every pre-#883 tamper would be reported as a migration.
|
|
727
|
+
if (reader !== TEST_CENSUS_READER) {
|
|
728
|
+
return {
|
|
729
|
+
kind: "stale-reader",
|
|
730
|
+
census,
|
|
731
|
+
reason:
|
|
732
|
+
`the banked census was taken by test-census reader ` +
|
|
733
|
+
`${renderReaderStamp(reader)} ` +
|
|
734
|
+
`and this gate reads with "${TEST_CENSUS_READER}". The two instruments count ` +
|
|
735
|
+
`different things, so a file whose number moved cannot be told from a file that ` +
|
|
736
|
+
`lost a test. This round banks a fresh reading and the next one compares (#883)`,
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
return { kind: "verified", census };
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
export function nextCensus(
|
|
743
|
+
previous: Record<string, number> | undefined,
|
|
744
|
+
current: Record<string, number>,
|
|
745
|
+
verdict: CensusVerdict,
|
|
746
|
+
): Record<string, number> {
|
|
747
|
+
// `unmeasured` holds the previous reading too. Banking an empty census
|
|
748
|
+
// over a real one would turn a tree the walk briefly could not read — a
|
|
749
|
+
// worktree mid-checkout, a `dist` rename — into a permanent amnesia, and
|
|
750
|
+
// the round after it would compare clean against nothing.
|
|
751
|
+
// `reader-changed` is deliberately NOT in this list. Holding the previous
|
|
752
|
+
// reading would leave an incomparable number banked forever and make every
|
|
753
|
+
// subsequent round UNMEASURED — the migration would never end. Banking the
|
|
754
|
+
// fresh one is what makes the NEXT round a real comparison, which is the
|
|
755
|
+
// whole purpose of reporting this round as unmeasured rather than clean.
|
|
756
|
+
if (verdict.kind === "regression" || verdict.kind === "unmeasured") return previous ?? current;
|
|
757
|
+
return current;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/** What the verify gate does with whatever scope recorded (#307). */
|
|
761
|
+
export type BaselineAdjustment =
|
|
762
|
+
| { kind: "subtract"; baseline: number; fails: number }
|
|
763
|
+
| { kind: "unmeasured"; reason: string }
|
|
764
|
+
| { kind: "none" };
|
|
765
|
+
|
|
766
|
+
/**
|
|
767
|
+
* The verify gate's reading of the recorded baseline (#307).
|
|
768
|
+
*
|
|
769
|
+
* `none` and `unmeasured` subtract the same amount — nothing — so the
|
|
770
|
+
* arithmetic is not what separates them. What `unmeasured` stops is the run
|
|
771
|
+
* claiming a clean starting tree it never looked at, and the only way to say
|
|
772
|
+
* that is to have kept the two apart all the way from `captureTestBaseline`.
|
|
773
|
+
*
|
|
774
|
+
* A number of `0` is `none`: the suite ran, nothing was failing, there is
|
|
775
|
+
* nothing to subtract and nothing to report.
|
|
776
|
+
*/
|
|
777
|
+
export function baselineAdjustment(
|
|
778
|
+
baselines: Record<string, any> | undefined,
|
|
779
|
+
fails: number,
|
|
780
|
+
): BaselineAdjustment {
|
|
781
|
+
const b = baselines ?? {};
|
|
782
|
+
if (typeof b.testBaseline === "number" && b.testBaseline > 0) {
|
|
783
|
+
return { kind: "subtract", baseline: b.testBaseline, fails: subtractTestBaseline(fails, b.testBaseline) };
|
|
784
|
+
}
|
|
785
|
+
if (typeof b.testBaselineUnmeasured === "string" && b.testBaselineUnmeasured.length > 0) {
|
|
786
|
+
return { kind: "unmeasured", reason: b.testBaselineUnmeasured };
|
|
787
|
+
}
|
|
788
|
+
return { kind: "none" };
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
// ── Gate executor contract interfaces (SC-373, SC-376) ───────────────────
|
|
792
|
+
|
|
793
|
+
export interface GateExecutorInput {
|
|
794
|
+
gate: string;
|
|
795
|
+
slug: string;
|
|
796
|
+
issue: number;
|
|
797
|
+
workDir: string;
|
|
798
|
+
stateFilePath: string;
|
|
799
|
+
/**
|
|
800
|
+
* The tree this gate runs its own subprocesses in, resolved once by the
|
|
801
|
+
* caller (#190). Required, not optional: a gate that resolves its own root
|
|
802
|
+
* can run its subprocesses — including `bun scripts/sync-spec-tests.ts`,
|
|
803
|
+
* which WRITES — in a different tree from the run it is grading, and say
|
|
804
|
+
* nothing, because both trees exist. `gates/run-gate.ts resolveRunRoot()`
|
|
805
|
+
* is where it comes from.
|
|
806
|
+
*/
|
|
807
|
+
harnessRoot: string;
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
export interface GateExecutorResult {
|
|
811
|
+
resultVal: "PASS" | "FAIL";
|
|
812
|
+
passes: number;
|
|
813
|
+
fails: number;
|
|
814
|
+
warns: number;
|
|
815
|
+
results: GateResult[];
|
|
816
|
+
attempt: number;
|
|
817
|
+
exitCode: number;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
// ── State + harness loading ──────────────────────────────────────────────
|
|
821
|
+
|
|
822
|
+
/**
|
|
823
|
+
* What a gate can say about the project's config (#311 SC-A8).
|
|
824
|
+
*
|
|
825
|
+
* THREE outcomes, not two. Four readers in this file built
|
|
826
|
+
* `.claude/rungate.json` by hand and branched on `existsSync`, which
|
|
827
|
+
* collapses "this project has no rungate" and "this project is on the
|
|
828
|
+
* documented directory layout" into one answer — and a third case,
|
|
829
|
+
* "configured and unreadable", into the same one again via a bare `catch`.
|
|
830
|
+
*
|
|
831
|
+
* The cost of that collapse is specific and was never a warning: a consumer
|
|
832
|
+
* on `.claude/rungate/` got its ceremony overrides switched off (#310), its
|
|
833
|
+
* type check skipped, its dev-server liveness unprobed and its prove-gate
|
|
834
|
+
* environment empty, and each of those looks exactly like a project that
|
|
835
|
+
* configured none of them.
|
|
836
|
+
*
|
|
837
|
+
* `unreadable` exists because a gate must not crash on a corrupt config and
|
|
838
|
+
* must not call it absent either. ADR-001 D2's corollary — a malformed
|
|
839
|
+
* config is never a missing config — survives the conversion from a throw to
|
|
840
|
+
* a value, which is the shape SC-1453 established for the one other site
|
|
841
|
+
* whose throw crosses into a gate.
|
|
842
|
+
*/
|
|
843
|
+
export type GateConfigReading =
|
|
844
|
+
| { status: "resolved"; config: ResolvedConfig; source: "directory" | "monolith"; path: string }
|
|
845
|
+
| { status: "absent" }
|
|
846
|
+
| { status: "unreadable"; reason: string };
|
|
847
|
+
|
|
848
|
+
/** The single config read for this file. One binding, so one thing to mutate. */
|
|
849
|
+
export function readGateConfig(projectRoot: string): GateConfigReading {
|
|
850
|
+
if (!projectRoot) return { status: "absent" };
|
|
851
|
+
try {
|
|
852
|
+
const resolution = resolveConfig(projectRoot);
|
|
853
|
+
return resolution.status === "resolved"
|
|
854
|
+
? {
|
|
855
|
+
status: "resolved",
|
|
856
|
+
config: resolution.config,
|
|
857
|
+
source: resolution.source,
|
|
858
|
+
path: resolution.path,
|
|
859
|
+
}
|
|
860
|
+
: { status: "absent" };
|
|
861
|
+
} catch (e: any) {
|
|
862
|
+
return { status: "unreadable", reason: e?.message || String(e) };
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
/**
|
|
867
|
+
* Say an unreadable config out loud, once, wherever it is hit.
|
|
868
|
+
*
|
|
869
|
+
* Returning `null` quietly is what every one of these sites used to do, and
|
|
870
|
+
* the whole point of the three-way reading is that one of the three is news.
|
|
871
|
+
*/
|
|
872
|
+
function reportUnreadable(where: string, reading: GateConfigReading): void {
|
|
873
|
+
if (reading.status === "unreadable") {
|
|
874
|
+
console.error(
|
|
875
|
+
`WARN: ${where} could not read the rungate config — ${reading.reason}. ` +
|
|
876
|
+
`This is UNMEASURED, not an unconfigured project.`,
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
export function loadAndValidateHarness(projectRoot: string): ProjectHarness | null {
|
|
882
|
+
const reading = readGateConfig(projectRoot);
|
|
883
|
+
reportUnreadable("ceremony overrides", reading);
|
|
884
|
+
if (reading.status !== "resolved") return null;
|
|
885
|
+
|
|
886
|
+
const result = safeParseProjectHarness(reading.config);
|
|
887
|
+
if (!result.success) {
|
|
888
|
+
console.warn(`WARN: rungate config schema validation failed (${reading.path}):`);
|
|
889
|
+
for (const err of result.error.issues) {
|
|
890
|
+
console.warn(` - ${err.path.join(".")}: ${err.message}`);
|
|
891
|
+
}
|
|
892
|
+
return null;
|
|
893
|
+
}
|
|
894
|
+
return result.data;
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
/**
|
|
898
|
+
* The URLs and commands the prove gate probes (#311 SC-A8).
|
|
899
|
+
*
|
|
900
|
+
* `apiBase` and `uiBase` come off the NORMALISED names. ADR-002 D3 resolves
|
|
901
|
+
* `dev.apiBase` to `apiUrl` and collapses an authored `null` — what a
|
|
902
|
+
* greenfield scaffold writes for a base URL it could not detect — into the
|
|
903
|
+
* same absence an omitted key gives. Reading `dev.apiBase` raw here would
|
|
904
|
+
* reintroduce the second spelling of "not set" that the normaliser exists to
|
|
905
|
+
* remove, and `if (apiBase)` would then be deciding on a `null` the rest of
|
|
906
|
+
* the harness has already agreed to treat as absent.
|
|
907
|
+
*
|
|
908
|
+
* Extracted from the prove block so it can be run against a fixture: every
|
|
909
|
+
* field below was empty for every consumer on the directory layout, and the
|
|
910
|
+
* symptom was a liveness probe that silently did not happen.
|
|
911
|
+
*/
|
|
912
|
+
export function proveEnvironment(projectRoot: string): {
|
|
913
|
+
apiBase: string;
|
|
914
|
+
uiBase: string;
|
|
915
|
+
devStart: string;
|
|
916
|
+
devHealthPath: string;
|
|
917
|
+
prodCfg: { apiBase?: string; healthPath?: string };
|
|
918
|
+
} {
|
|
919
|
+
const empty = { apiBase: "", uiBase: "", devStart: "", devHealthPath: "", prodCfg: {} };
|
|
920
|
+
const reading = readGateConfig(projectRoot);
|
|
921
|
+
reportUnreadable("prove gate environment", reading);
|
|
922
|
+
if (reading.status !== "resolved") return empty;
|
|
923
|
+
|
|
924
|
+
const c = reading.config;
|
|
925
|
+
return {
|
|
926
|
+
apiBase: c.apiUrl || "",
|
|
927
|
+
uiBase: c.uiUrl || "",
|
|
928
|
+
devStart: c.dev?.start || "",
|
|
929
|
+
devHealthPath: c.dev?.healthPath || "",
|
|
930
|
+
prodCfg: { apiBase: c.prod?.apiBase || "", healthPath: c.prod?.healthPath || "" },
|
|
931
|
+
};
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* Make a `<remote>/<branch>` ref resolvable, fetching it if it is not (#118).
|
|
936
|
+
*
|
|
937
|
+
* Qualifying evidence commands to `origin/main` only helps if that ref exists.
|
|
938
|
+
* It frequently does not: a CI checkout fetches the PR merge ref and nothing
|
|
939
|
+
* else, and a fresh agent worktree inherits whatever the shared clone last
|
|
940
|
+
* fetched. `git diff origin/main` against a missing ref exits non-zero with a
|
|
941
|
+
* message on stderr, which the verdict logic below reads as a code failure —
|
|
942
|
+
* reproducing the exact false FAIL this issue is about, one layer down.
|
|
943
|
+
*
|
|
944
|
+
* Returns false when the ref still cannot be resolved, so the caller can say
|
|
945
|
+
* "could not measure" instead of "failed".
|
|
946
|
+
*/
|
|
947
|
+
/** Exported for test: one entry per (cwd, ref) actually resolved. */
|
|
948
|
+
export const __refResolutionCache = new Map<string, boolean>();
|
|
949
|
+
const REF_RESOLUTION_CACHE = __refResolutionCache;
|
|
950
|
+
|
|
951
|
+
function ensureRemoteRef(ref: string, cwd: string): boolean {
|
|
952
|
+
// Memoized per (cwd, ref) for the life of the process. A gate run commonly
|
|
953
|
+
// has a dozen ACs all diffing against origin/main; without this each one
|
|
954
|
+
// pays its own fetch, and the 60s timeout below turns an offline remote into
|
|
955
|
+
// minutes of dead gate time before any verdict exists. Negative results are
|
|
956
|
+
// cached too — that is the expensive case, and re-attempting a fetch that
|
|
957
|
+
// just failed within the same run has nothing new to learn.
|
|
958
|
+
const key = `${cwd}\u0000${ref}`;
|
|
959
|
+
const cached = REF_RESOLUTION_CACHE.get(key);
|
|
960
|
+
if (cached !== undefined) return cached;
|
|
961
|
+
const ok = resolveRemoteRef(ref, cwd);
|
|
962
|
+
REF_RESOLUTION_CACHE.set(key, ok);
|
|
963
|
+
return ok;
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
function resolveRemoteRef(ref: string, cwd: string): boolean {
|
|
967
|
+
// execFileSync with an argv array throughout: no shell, so a ref name can
|
|
968
|
+
// never be interpreted as a command. The shape checks below stay as a
|
|
969
|
+
// second line, but they are no longer what makes this safe.
|
|
970
|
+
const resolves = () => {
|
|
971
|
+
try {
|
|
972
|
+
execFileSync("git", ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], {
|
|
973
|
+
cwd, stdio: "pipe", timeout: 10000,
|
|
974
|
+
});
|
|
975
|
+
return true;
|
|
976
|
+
} catch {
|
|
977
|
+
return false;
|
|
978
|
+
}
|
|
979
|
+
};
|
|
980
|
+
if (resolves()) return true;
|
|
981
|
+
const slash = ref.indexOf("/");
|
|
982
|
+
if (slash <= 0) return false;
|
|
983
|
+
const remote = ref.slice(0, slash);
|
|
984
|
+
const branch = ref.slice(slash + 1);
|
|
985
|
+
if (!/^[A-Za-z0-9._-]+$/.test(remote) || !/^[A-Za-z0-9._/-]+$/.test(branch)) return false;
|
|
986
|
+
try {
|
|
987
|
+
// Explicit refspec: `git fetch origin main` alone writes FETCH_HEAD and
|
|
988
|
+
// does not necessarily create refs/remotes/origin/main, which is the name
|
|
989
|
+
// the evidence command uses.
|
|
990
|
+
execFileSync("git", ["fetch", "--quiet", remote, `${branch}:refs/remotes/${remote}/${branch}`], {
|
|
991
|
+
cwd, stdio: "pipe", timeout: 60000,
|
|
992
|
+
});
|
|
993
|
+
} catch {
|
|
994
|
+
// Offline, no remote, or no permission. Not fatal here — the caller
|
|
995
|
+
// reports it as unmeasurable rather than guessing a verdict.
|
|
996
|
+
return false;
|
|
997
|
+
}
|
|
998
|
+
return resolves();
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
/**
|
|
1002
|
+
* Remote refs an evidence command depends on, e.g. `origin/main`.
|
|
1003
|
+
*
|
|
1004
|
+
* Scanned narrowly, because this function's mistakes go both ways. Reading a
|
|
1005
|
+
* ref that is not there makes a measurable AC unmeasurable and blocks it;
|
|
1006
|
+
* missing one that is there lets the stale-ref FAIL through. A bare scan for
|
|
1007
|
+
* `origin/...` anywhere in the string does both: `grep -r origin/main docs/`
|
|
1008
|
+
* names no ref at all, and `git log -- docs/origin/main.md` names a path.
|
|
1009
|
+
*
|
|
1010
|
+
* So: only inside a `git` invocation, and only before the `--` pathspec
|
|
1011
|
+
* separator, and not when preceded by a path character.
|
|
1012
|
+
*/
|
|
1013
|
+
export function remoteRefsIn(command: string): string[] {
|
|
1014
|
+
const refs = new Set<string>();
|
|
1015
|
+
for (const invocation of command.matchAll(/\bgit\s+(?:-C\s+\S+\s+)?([^|&;]*)/g)) {
|
|
1016
|
+
const beforePathspec = invocation[1].split(" -- ")[0];
|
|
1017
|
+
// Segment-wise, so a `.` can appear inside a ref name (`release-2.1`) but
|
|
1018
|
+
// `..` terminates it — `origin/main..HEAD` depends on `origin/main`, and
|
|
1019
|
+
// asking git to fetch `main..HEAD` fetches nothing.
|
|
1020
|
+
const inRefPosition = /(?<![\w/.-])(origin|upstream)\/([A-Za-z0-9_-]+(?:[./][A-Za-z0-9_-]+)*)/g;
|
|
1021
|
+
for (const m of beforePathspec.matchAll(inRefPosition)) refs.add(`${m[1]}/${m[2]}`);
|
|
1022
|
+
}
|
|
1023
|
+
return [...refs];
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/**
|
|
1027
|
+
* #705 — a command that printed nothing measured nothing, and `""` says the
|
|
1028
|
+
* wrong one of those.
|
|
1029
|
+
*
|
|
1030
|
+
* The recording below used to write `{ type: "command-output", content: "" }`
|
|
1031
|
+
* whenever an evidence command exited 0 without printing. `hasRecordedEvidence`
|
|
1032
|
+
* in `lib/evidence-required.ts` reads `Boolean("")` as false, so the verify
|
|
1033
|
+
* gate then refused with *"threshold == 0 with NO evidence recorded"* — a
|
|
1034
|
+
* sentence about an AC nobody measured, applied to an AC that was measured and
|
|
1035
|
+
* came back silent. Run `wf_b8f7b077-9cf` AC-3 died that way, and the message
|
|
1036
|
+
* names the AC and its `evidenceMethod.type`, so each occurrence reads as an
|
|
1037
|
+
* authoring mistake three phases after authoring.
|
|
1038
|
+
*
|
|
1039
|
+
* It is not about `GREP_CHECK`. This function has never read `ac.type` or
|
|
1040
|
+
* `evidenceMethod.type` — it keys on `evidenceMethod.command` alone — and two
|
|
1041
|
+
* `GREP_CHECK` ACs in the same batch of runs recorded `"0"` and passed. What
|
|
1042
|
+
* distinguishes them is whether the command PRINTS its count: `grep -c` does,
|
|
1043
|
+
* `grep -n` prints nothing when it matches nothing. A `COMMAND` AC reaches
|
|
1044
|
+
* this branch on identical terms.
|
|
1045
|
+
*
|
|
1046
|
+
* ONE binding, called from both recording sites, so there is a single thing to
|
|
1047
|
+
* mutate. A second site composing its own sentence would survive any mutation
|
|
1048
|
+
* of this one and the gap would go green for the wrong reason.
|
|
1049
|
+
*/
|
|
1050
|
+
export function emptyOutputUnmeasured(
|
|
1051
|
+
command: string,
|
|
1052
|
+
exitCode: number | null,
|
|
1053
|
+
detail?: string,
|
|
1054
|
+
): { type: "unmeasured"; content: string } {
|
|
1055
|
+
const exit = exitCode === null ? "with no status" : `with status ${exitCode}`;
|
|
1056
|
+
return {
|
|
1057
|
+
type: "unmeasured",
|
|
1058
|
+
content:
|
|
1059
|
+
`UNMEASURED: the evidence command exited ${exit} and wrote NOTHING to stdout, so no ` +
|
|
1060
|
+
`value was produced and there is nothing to compare against the threshold. An empty ` +
|
|
1061
|
+
`reading and a measured 0 are different states — recording the empty one as "" made ` +
|
|
1062
|
+
`every reader downstream report this AC as carrying no evidence at all (#705). Make ` +
|
|
1063
|
+
`the command PRINT the value it asserts about: \`grep -c <pattern> <file>\` prints 0 ` +
|
|
1064
|
+
`where \`grep -n\` prints nothing, and a count swallowed by \`$( )\` never reaches ` +
|
|
1065
|
+
`stdout. Command: ${command}` +
|
|
1066
|
+
(detail ? ` — ${detail}` : ""),
|
|
1067
|
+
};
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
/**
|
|
1071
|
+
* Exported for test. This is where evidence commands actually execute, and it
|
|
1072
|
+
* is the last place a stale git ref can be caught (#118).
|
|
1073
|
+
*/
|
|
1074
|
+
export function autoPopulateACs(state: Record<string, any>, sf: string): void {
|
|
1075
|
+
let acUpdated = false;
|
|
1076
|
+
for (let i = 0; i < (state.acs || []).length; i++) {
|
|
1077
|
+
const ac = state.acs[i];
|
|
1078
|
+
if (ac.verdict && ac.verdict !== "PENDING" && ac.verdict !== "FAIL") continue;
|
|
1079
|
+
const rawCmd = ac.evidenceMethod?.command;
|
|
1080
|
+
if (!rawCmd) continue;
|
|
1081
|
+
// Qualify bare `main`/`master` to `origin/...` at the point of execution,
|
|
1082
|
+
// not only at prevalidation (#118).
|
|
1083
|
+
//
|
|
1084
|
+
// Prevalidation is not a chokepoint: the verify-gate re-implementation path
|
|
1085
|
+
// authors NEW ACs after Scope has already run, so a command can reach here
|
|
1086
|
+
// having never been prevalidated. Normalizing only there would fix the
|
|
1087
|
+
// commands that were already least likely to be wrong.
|
|
1088
|
+
//
|
|
1089
|
+
// Writing it back onto the AC means the repaired form is what gets
|
|
1090
|
+
// persisted and reported, so a human reading the failure sees the command
|
|
1091
|
+
// that actually ran.
|
|
1092
|
+
//
|
|
1093
|
+
// #614: through `recordEvidenceRewrite`, so the authored form survives
|
|
1094
|
+
// the repair. This is the last site that changes an evidence command,
|
|
1095
|
+
// and it used to be the one that made the authored text unrecoverable.
|
|
1096
|
+
const cmd = normalizeGitRefs(rawCmd).command;
|
|
1097
|
+
recordEvidenceRewrite(ac.evidenceMethod, "normalize-git-refs", cmd);
|
|
1098
|
+
|
|
1099
|
+
// A ref the command needs but the clone does not have is an ENVIRONMENT
|
|
1100
|
+
// fault, not a code failure. Letting it fall through produces a non-zero
|
|
1101
|
+
// exit with a message on stderr, which the verdict logic below scores as
|
|
1102
|
+
// FAIL — recreating the false FAIL this whole issue is about, one layer
|
|
1103
|
+
// down. Found by CI: the runner checks out the PR merge ref and has no
|
|
1104
|
+
// refs/remotes/origin/main at all, so the corrected command failed there
|
|
1105
|
+
// while passing on every developer machine.
|
|
1106
|
+
const evidenceCwdForRefs = process.env.EVIDENCE_CWD || state.projectRoot || process.cwd();
|
|
1107
|
+
const missingRefs = remoteRefsIn(cmd).filter(r => !ensureRemoteRef(r, evidenceCwdForRefs));
|
|
1108
|
+
if (missingRefs.length > 0) {
|
|
1109
|
+
// Blocking, not SKIP.
|
|
1110
|
+
//
|
|
1111
|
+
// The first version of this marked the AC SKIP, reasoning that an
|
|
1112
|
+
// unmeasured AC should not be scored as a code failure. Security review
|
|
1113
|
+
// caught what that actually bought: gates/orchestrator.ts advances when
|
|
1114
|
+
// every AC is PASS or SKIP, so an unresolvable ref would have turned a
|
|
1115
|
+
// blocking gate into a passing one. Anything able to make a ref
|
|
1116
|
+
// unresolvable — a typo, a network blip, a ref name chosen for the
|
|
1117
|
+
// purpose — would have been a gate bypass.
|
|
1118
|
+
//
|
|
1119
|
+
// What #118 is actually about is narrower than "never FAIL here". The
|
|
1120
|
+
// #103 damage was a confident, specific, fabricated finding — named
|
|
1121
|
+
// files, named line numbers — that a second agent then acted on. Saying
|
|
1122
|
+
// "origin/main is unreachable, I measured nothing" is the opposite of
|
|
1123
|
+
// that. It blocks, and it is honest about why, which is what lets a
|
|
1124
|
+
// human or a heal step fix the environment instead of the code.
|
|
1125
|
+
console.warn(`WARN: AC ${ac.id} needs ${missingRefs.join(", ")} which could not be resolved or fetched — blocking as UNMEASURED (environment fault, not a code defect)`);
|
|
1126
|
+
state.acs[i].verdict = "FAIL";
|
|
1127
|
+
state.acs[i].evidence = {
|
|
1128
|
+
type: "unmeasured",
|
|
1129
|
+
content: `Evidence requires ${missingRefs.join(", ")}; the ref is absent and could not be fetched. This is an environment fault — the AC was NOT evaluated, so no claim is made about the code. Fix the checkout (fetch the ref), then re-run; do not change code in response to this.`,
|
|
1130
|
+
};
|
|
1131
|
+
acUpdated = true;
|
|
1132
|
+
continue;
|
|
1133
|
+
}
|
|
1134
|
+
try {
|
|
1135
|
+
const evidenceCwd = process.env.EVIDENCE_CWD || state.projectRoot || process.cwd();
|
|
1136
|
+
const isTestRunner = /bun test\b/.test(cmd);
|
|
1137
|
+
const cmdTimeout = isTestRunner ? 300000 : 10000;
|
|
1138
|
+
const output = execSync(cmd, { encoding: "utf-8", timeout: cmdTimeout, cwd: evidenceCwd }).trim();
|
|
1139
|
+
// #705. Blocking, not SKIP, and for the reason the missing-ref branch
|
|
1140
|
+
// above gives: gates/orchestrator.ts advances when every AC is PASS or
|
|
1141
|
+
// SKIP, so marking an AC nothing could measure as SKIP would turn a
|
|
1142
|
+
// blocking gate into a passing one.
|
|
1143
|
+
//
|
|
1144
|
+
// `isTestRunner` is carved out because bun writes its summary to stderr:
|
|
1145
|
+
// a `bun test` invocation can leave stdout holding only the version
|
|
1146
|
+
// banner, or nothing at all, and the verdict below does not read the
|
|
1147
|
+
// output anyway. Calling that UNMEASURED would refuse every BUN_TEST AC
|
|
1148
|
+
// in the project.
|
|
1149
|
+
if (!isTestRunner && output === "") {
|
|
1150
|
+
state.acs[i].verdict = "FAIL";
|
|
1151
|
+
state.acs[i].evidence = emptyOutputUnmeasured(cmd, 0);
|
|
1152
|
+
acUpdated = true;
|
|
1153
|
+
continue;
|
|
1154
|
+
}
|
|
1155
|
+
const lastLine = output.split("\n").pop() || "";
|
|
1156
|
+
let verdict: "PASS" | "FAIL" = "FAIL";
|
|
1157
|
+
if (isTestRunner) {
|
|
1158
|
+
verdict = "PASS";
|
|
1159
|
+
}
|
|
1160
|
+
const { op, value } = ac.threshold || {};
|
|
1161
|
+
if (!isTestRunner && op && value !== undefined) {
|
|
1162
|
+
const num = parseFloat(lastLine);
|
|
1163
|
+
const exp = parseFloat(String(value));
|
|
1164
|
+
switch (op) {
|
|
1165
|
+
case "==": verdict = lastLine === String(value) || (!isNaN(num) && num === exp) ? "PASS" : "FAIL"; break;
|
|
1166
|
+
case "!=": verdict = lastLine !== String(value) ? "PASS" : "FAIL"; break;
|
|
1167
|
+
case ">=": verdict = !isNaN(num) && num >= exp ? "PASS" : "FAIL"; break;
|
|
1168
|
+
case "<=": verdict = !isNaN(num) && num <= exp ? "PASS" : "FAIL"; break;
|
|
1169
|
+
case ">": verdict = !isNaN(num) && num > exp ? "PASS" : "FAIL"; break;
|
|
1170
|
+
case "<": verdict = !isNaN(num) && num < exp ? "PASS" : "FAIL"; break;
|
|
1171
|
+
case "contains": verdict = output.includes(String(value)) ? "PASS" : "FAIL"; break;
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
state.acs[i].verdict = verdict;
|
|
1175
|
+
state.acs[i].evidence = { type: "command-output", content: lastLine };
|
|
1176
|
+
acUpdated = true;
|
|
1177
|
+
} catch (e: any) {
|
|
1178
|
+
// grep returns exit 1 on zero matches — valid result, not an error
|
|
1179
|
+
const stderr = e.stderr?.toString?.() || "";
|
|
1180
|
+
const stdout = (e.stdout?.toString?.() || "").trim();
|
|
1181
|
+
if (stdout !== "" || e.status === 1) {
|
|
1182
|
+
const lastLine = stdout.split("\n").pop() || "0";
|
|
1183
|
+
let verdict: "PASS" | "FAIL" = "FAIL";
|
|
1184
|
+
const { op, value } = ac.threshold || {};
|
|
1185
|
+
if (op && value !== undefined) {
|
|
1186
|
+
const num = parseFloat(lastLine);
|
|
1187
|
+
const exp = parseFloat(String(value));
|
|
1188
|
+
switch (op) {
|
|
1189
|
+
case "==": verdict = lastLine === String(value) || (!isNaN(num) && num === exp) ? "PASS" : "FAIL"; break;
|
|
1190
|
+
case "!=": verdict = lastLine !== String(value) ? "PASS" : "FAIL"; break;
|
|
1191
|
+
case ">=": verdict = !isNaN(num) && num >= exp ? "PASS" : "FAIL"; break;
|
|
1192
|
+
case "<=": verdict = !isNaN(num) && num <= exp ? "PASS" : "FAIL"; break;
|
|
1193
|
+
case ">": verdict = !isNaN(num) && num > exp ? "PASS" : "FAIL"; break;
|
|
1194
|
+
case "<": verdict = !isNaN(num) && num < exp ? "PASS" : "FAIL"; break;
|
|
1195
|
+
case "contains": verdict = stdout.includes(String(value)) ? "PASS" : "FAIL"; break;
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
state.acs[i].verdict = verdict;
|
|
1199
|
+
state.acs[i].evidence = { type: "command-output", content: lastLine };
|
|
1200
|
+
acUpdated = true;
|
|
1201
|
+
} else {
|
|
1202
|
+
// #705. The branch that recorded nothing at all, which is the literal
|
|
1203
|
+
// "can never record evidence" the issue title describes: the AC kept a
|
|
1204
|
+
// PENDING verdict and an absent `evidence`, so the gate refused it with
|
|
1205
|
+
// the never-measured sentence and the warning below was the only place
|
|
1206
|
+
// the real cause existed. A warning is not a reading.
|
|
1207
|
+
console.warn(`WARN: AC ${ac.id} evidence command failed: ${e.message?.slice(0, 100) || 'unknown'}`);
|
|
1208
|
+
state.acs[i].verdict = "FAIL";
|
|
1209
|
+
state.acs[i].evidence = emptyOutputUnmeasured(
|
|
1210
|
+
cmd,
|
|
1211
|
+
typeof e.status === "number" ? e.status : null,
|
|
1212
|
+
(stderr.trim().split("\n").pop() || e.message || "").slice(0, 200),
|
|
1213
|
+
);
|
|
1214
|
+
acUpdated = true;
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
// #513 SC-A1: ACs without evidenceMethod.command get SKIP, not generic output
|
|
1219
|
+
const skipped = markCommandlessACsAsSkip(sf);
|
|
1220
|
+
if (skipped > 0) {
|
|
1221
|
+
const refreshed = JSON.parse(readFileSync(sf, "utf-8"));
|
|
1222
|
+
Object.assign(state, refreshed);
|
|
1223
|
+
} else if (acUpdated) {
|
|
1224
|
+
writeFileSync(sf, JSON.stringify(state, null, 2));
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
// ── Verify-gate early checks ────────────────────────────────────────────
|
|
1229
|
+
|
|
1230
|
+
function checkVerifyPendingACs(state: Record<string, any>): { results: GateResult[]; failCount: number } {
|
|
1231
|
+
const results: GateResult[] = [];
|
|
1232
|
+
let failCount = 0;
|
|
1233
|
+
const pendingCodeACs = (state.acs || []).filter(
|
|
1234
|
+
(ac: any) => ac.type !== "OUTCOME" && (!ac.verdict || ac.verdict === "PENDING")
|
|
1235
|
+
);
|
|
1236
|
+
if (pendingCodeACs.length > 0) {
|
|
1237
|
+
for (const ac of pendingCodeACs) {
|
|
1238
|
+
failCount++;
|
|
1239
|
+
results.push({
|
|
1240
|
+
check: `verify-ac-evidence-gap: ${ac.id} still PENDING after auto-populate`,
|
|
1241
|
+
result: "FAIL" as const,
|
|
1242
|
+
detail: `${ac.id} evidence command may be broken or matching 0 tests — check evidenceMethod.command: ${ac.evidenceMethod?.command || "(none)"}`,
|
|
1243
|
+
});
|
|
1244
|
+
}
|
|
1245
|
+
console.log(`Verify AC evidence gap: ${pendingCodeACs.length} CODE AC(s) still PENDING — ${pendingCodeACs.map((ac: any) => ac.id).join(", ")}`);
|
|
1246
|
+
}
|
|
1247
|
+
return { results, failCount };
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
// ── Prove reproducer (B3) ───────────────────────────────────────────────
|
|
1251
|
+
|
|
1252
|
+
async function runProveReproducer(state: Record<string, any>, issue: number, issueRepo: string, workDir: string, harnessRoot: string, stateFile: string): Promise<void> {
|
|
1253
|
+
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
1254
|
+
if (tier === "LIGHT") {
|
|
1255
|
+
console.log("prove gate: LIGHT tier — skipping reproducer (API check only)");
|
|
1256
|
+
return;
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
let issueBody = "";
|
|
1260
|
+
let issueTitle = "";
|
|
1261
|
+
if (issue && issueRepo) {
|
|
1262
|
+
try {
|
|
1263
|
+
const github = createGitHubClient();
|
|
1264
|
+
const issueData = await getIssue(github, issueRepo, issue);
|
|
1265
|
+
issueBody = issueData.body || "";
|
|
1266
|
+
issueTitle = issueData.title || "";
|
|
1267
|
+
console.log(`prove gate: read issue #${issue} from GitHub (${issueTitle.slice(0, 60)})`);
|
|
1268
|
+
} catch (e: any) {
|
|
1269
|
+
console.error(`prove gate: FAIL — could not read issue from GitHub: ${e.message?.slice(0, 200)}`);
|
|
1270
|
+
process.exit(1);
|
|
1271
|
+
}
|
|
1272
|
+
} else {
|
|
1273
|
+
console.error("prove gate: FAIL — issue number and repo required for prove");
|
|
1274
|
+
process.exit(1);
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
const projectRoot = state.projectRoot || "";
|
|
1278
|
+
const proveEnv = proveEnvironment(projectRoot);
|
|
1279
|
+
const { apiBase, uiBase, devStart, devHealthPath, prodCfg } = proveEnv;
|
|
1280
|
+
|
|
1281
|
+
// #304. This used to curl `${apiBase}/api/admin/health` and `process.exit(1)`
|
|
1282
|
+
// on failure — the worst of the three copies of that route, because the
|
|
1283
|
+
// consequence is a hard refusal rather than a FAIL line. A consumer whose
|
|
1284
|
+
// health route is anything else had its prove gate killed on a server that
|
|
1285
|
+
// was up, told to start it.
|
|
1286
|
+
//
|
|
1287
|
+
// The UNMEASURED branch is a REFUSAL RELAXATION and has to be recorded as
|
|
1288
|
+
// one. Before this change a consumer with `dev.apiBase` set got
|
|
1289
|
+
// `process.exit(1)` when its server was down; no consumer sets
|
|
1290
|
+
// `dev.healthPath` yet, so on merge every one of them stops being refused
|
|
1291
|
+
// here. Reporting DOWN instead would be wrong — that is the whole issue —
|
|
1292
|
+
// but letting the gap evaporate into a stdout line is the fail-open the
|
|
1293
|
+
// security review named. So the reading is written to the workflow state,
|
|
1294
|
+
// which is durable and which later gates read, as well as to
|
|
1295
|
+
// `prove-input.json`.
|
|
1296
|
+
//
|
|
1297
|
+
// It is both recorded and consumed since #346: the contradiction check
|
|
1298
|
+
// below refuses when an OUTCOME AC claims server-dependent evidence while
|
|
1299
|
+
// liveness was never measured. Recording it without reading it left the
|
|
1300
|
+
// control #304 relaxed genuinely removed — #213's shape, where a field is
|
|
1301
|
+
// written and nothing is decided on it.
|
|
1302
|
+
//
|
|
1303
|
+
// The reproducer prompt still does not read it, and that was once claimed
|
|
1304
|
+
// here without checking: `gates/prompts/prove-reproducer.md` never
|
|
1305
|
+
// mentions liveness. The refusal below is what reads it.
|
|
1306
|
+
const devTarget = probeTarget("dev", { apiBase, healthPath: devHealthPath });
|
|
1307
|
+
let devLiveness: { tier: string; status: "UP" | "UNMEASURED"; detail: string } | null = null;
|
|
1308
|
+
if (apiBase) {
|
|
1309
|
+
if (devTarget.kind === "unmeasured") {
|
|
1310
|
+
devLiveness = { tier: "dev", status: "UNMEASURED", detail: devTarget.detail };
|
|
1311
|
+
console.warn(`WARN: prove gate — dev liveness UNMEASURED: ${devTarget.detail}`);
|
|
1312
|
+
} else {
|
|
1313
|
+
try {
|
|
1314
|
+
execFileSync("curl", ["-sf", "--max-time", "10", "--", devTarget.url], {
|
|
1315
|
+
timeout: 10000, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"],
|
|
1316
|
+
});
|
|
1317
|
+
devLiveness = { tier: "dev", status: "UP", detail: devTarget.safeUrl };
|
|
1318
|
+
console.log(`prove gate: health check PASS (${devTarget.safeUrl})`);
|
|
1319
|
+
} catch {
|
|
1320
|
+
console.error(`prove gate: FAIL — dev server not responding at ${devTarget.safeUrl}`);
|
|
1321
|
+
if (devStart) console.error(`Start it with: ${devStart}`);
|
|
1322
|
+
process.exit(1);
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
// Prod health check — WARN only. Third copy of the same hardcoded route
|
|
1328
|
+
// (#304), and the one most likely to go unnoticed, because its only
|
|
1329
|
+
// consequence is a warning line nobody reads.
|
|
1330
|
+
let prodLiveness: { tier: string; status: "UP" | "DOWN" | "UNMEASURED"; detail: string } | null = null;
|
|
1331
|
+
if (prodCfg.apiBase) {
|
|
1332
|
+
const prodTarget = probeTarget("prod", prodCfg);
|
|
1333
|
+
if (prodTarget.kind === "unmeasured") {
|
|
1334
|
+
prodLiveness = { tier: "prod", status: "UNMEASURED", detail: prodTarget.detail };
|
|
1335
|
+
console.warn(`WARN: prove gate — prod liveness UNMEASURED: ${prodTarget.detail}`);
|
|
1336
|
+
} else {
|
|
1337
|
+
try {
|
|
1338
|
+
execFileSync("curl", ["-sf", "--max-time", "10", "--", prodTarget.url], {
|
|
1339
|
+
timeout: 10000, encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"],
|
|
1340
|
+
});
|
|
1341
|
+
prodLiveness = { tier: "prod", status: "UP", detail: prodTarget.safeUrl };
|
|
1342
|
+
console.log(`prove gate: prod health check PASS (${prodTarget.safeUrl})`);
|
|
1343
|
+
} catch {
|
|
1344
|
+
// DOWN here, unlike dev, is a measurement: the route was configured
|
|
1345
|
+
// and the probe ran. UNMEASURED above is not.
|
|
1346
|
+
prodLiveness = { tier: "prod", status: "DOWN", detail: prodTarget.safeUrl };
|
|
1347
|
+
console.warn(`WARN: prove gate — prod server not responding at ${prodTarget.safeUrl} (STANDARD+ requires prod evidence)`);
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
const proveInput = {
|
|
1353
|
+
issueBody, issueTitle, issueNumber: issue,
|
|
1354
|
+
acs: state.acs || [], apiBase, uiBase,
|
|
1355
|
+
beforeState: state.beforeState || null, projectRoot,
|
|
1356
|
+
// #304: the liveness readings, including the ones that measured nothing.
|
|
1357
|
+
// UNMEASURED has to be durable or relaxing the refusal above is a silent
|
|
1358
|
+
// fail-open.
|
|
1359
|
+
liveness: [devLiveness, prodLiveness].filter(Boolean),
|
|
1360
|
+
};
|
|
1361
|
+
// And on the workflow state, which is the durable artefact later gates
|
|
1362
|
+
// read. prove-input.json is consumed by one spawned agent and discarded;
|
|
1363
|
+
// writing only there would make the record as ephemeral as the stdout line
|
|
1364
|
+
// it replaced.
|
|
1365
|
+
state.liveness = proveInput.liveness;
|
|
1366
|
+
writeFileSync(stateFile, JSON.stringify(state, null, 2));
|
|
1367
|
+
const proveInputPath = join(workDir, "prove-input.json");
|
|
1368
|
+
writeFileSync(proveInputPath, JSON.stringify(proveInput, null, 2));
|
|
1369
|
+
|
|
1370
|
+
// #346: the reading is now CONSUMED. Written after the artefacts so the
|
|
1371
|
+
// record survives the refusal — a run that refuses here should still leave
|
|
1372
|
+
// behind the state that explains why.
|
|
1373
|
+
//
|
|
1374
|
+
// Not "refuse when liveness is UNMEASURED", which would reinstate the
|
|
1375
|
+
// false FAIL #304 removed. Only the contradiction: evidence claimed from a
|
|
1376
|
+
// server the run reports it never found.
|
|
1377
|
+
const contradictions = livenessEvidenceContradictions(state.acs || [], state.liveness || []);
|
|
1378
|
+
if (contradictions.length > 0) {
|
|
1379
|
+
console.error(`prove gate: FAIL — evidence claimed from a server whose liveness was never measured`);
|
|
1380
|
+
for (const c of contradictions) console.error(` - ${c}`);
|
|
1381
|
+
console.error(
|
|
1382
|
+
`Set the tier's \`healthPath\` in .claude/rungate/config.json so liveness can be measured, ` +
|
|
1383
|
+
`or correct the evidence.`,
|
|
1384
|
+
);
|
|
1385
|
+
process.exit(1);
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
const proveProjectPrompt = join(__dirname, "prompts", "prove-reproducer.md");
|
|
1389
|
+
const proveHomePrompt = join(process.env.HOME || "", ".claude", "gates", "prompts", "prove-reproducer.md");
|
|
1390
|
+
const provePromptPath = existsSync(proveProjectPrompt) ? proveProjectPrompt : proveHomePrompt;
|
|
1391
|
+
if (existsSync(provePromptPath)) {
|
|
1392
|
+
console.log("B3: Spawning Prove Reproducer (Sonnet)...");
|
|
1393
|
+
let reproducerOutput = "";
|
|
1394
|
+
try {
|
|
1395
|
+
reproducerOutput = execSync(
|
|
1396
|
+
`cat '${proveInputPath}' | claude -p - --model sonnet --system-prompt-file "${provePromptPath}" --output-format text --allowedTools "Bash,Read" --max-turns 10`,
|
|
1397
|
+
{ encoding: "utf-8", timeout: 180000, cwd: harnessRoot }
|
|
1398
|
+
);
|
|
1399
|
+
} catch (e: any) {
|
|
1400
|
+
reproducerOutput = e.stdout || "";
|
|
1401
|
+
console.warn(`WARN: B3 reproducer process exited with error: ${e.message?.slice(0, 200)}`);
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
const jsonMatch = reproducerOutput.match(/\{[\s\S]*"verdict"[\s\S]*\}/);
|
|
1405
|
+
if (jsonMatch) {
|
|
1406
|
+
try {
|
|
1407
|
+
const parsed = JSON.parse(jsonMatch[0]);
|
|
1408
|
+
const proveEvidence = {
|
|
1409
|
+
issueNumber: issue, verdict: parsed.verdict || "INCONCLUSIVE",
|
|
1410
|
+
commitSHA: execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: projectRoot || undefined }).trim(),
|
|
1411
|
+
capturedAt: new Date().toISOString(),
|
|
1412
|
+
criteriaResults: parsed.criteriaResults || [],
|
|
1413
|
+
reproduced: parsed.reproduced ?? false,
|
|
1414
|
+
evidence: parsed.evidence || [],
|
|
1415
|
+
source: "B3-prove-reproducer",
|
|
1416
|
+
};
|
|
1417
|
+
writeFileSync(join(workDir, "prove-evidence.json"), JSON.stringify(proveEvidence, null, 2));
|
|
1418
|
+
console.log(`B3: prove-evidence.json written — verdict: ${proveEvidence.verdict}`);
|
|
1419
|
+
} catch {
|
|
1420
|
+
console.warn("WARN: B3 reproducer JSON parse failed");
|
|
1421
|
+
}
|
|
1422
|
+
} else {
|
|
1423
|
+
console.warn("WARN: B3 reproducer produced no parseable JSON — prove.test.ts will generate evidence");
|
|
1424
|
+
}
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
|
|
1428
|
+
// ── Scope pre-flight checks ─────────────────────────────────────────────
|
|
1429
|
+
|
|
1430
|
+
/**
|
|
1431
|
+
* Reduce a pre-validation reading to the lines the gate prints.
|
|
1432
|
+
*
|
|
1433
|
+
* Printing is all this does. The verdict reaches later checks through
|
|
1434
|
+
* workflow-state.json, not through stdout — see runScopePreflightsAndPersist.
|
|
1435
|
+
*/
|
|
1436
|
+
function reportPrevalidation(reading: EvidencePrevalidationReading): void {
|
|
1437
|
+
if (reading.verdict === "UNMEASURED") {
|
|
1438
|
+
console.error(`EVIDENCE PRE-VALIDATION: UNMEASURED — ${reading.reason}`);
|
|
1439
|
+
return;
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
const broken = reading.acs.filter((r) => r.status === "broken");
|
|
1443
|
+
const empty = reading.acs.filter((r) => r.status === "empty");
|
|
1444
|
+
const autoFixed = reading.acs.filter((r) => r.autoFixed);
|
|
1445
|
+
|
|
1446
|
+
if (broken.length > 0) {
|
|
1447
|
+
console.error(`EVIDENCE PRE-VALIDATION: ${broken.length} AC(s) have broken evidence commands:`);
|
|
1448
|
+
for (const r of broken) {
|
|
1449
|
+
console.error(` - ${r.id}: ${r.diagnostic}`);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
if (empty.length > 0) {
|
|
1453
|
+
console.warn(`EVIDENCE PRE-VALIDATION: ${empty.length} AC(s) have evidence commands that return empty output:`);
|
|
1454
|
+
for (const r of empty) {
|
|
1455
|
+
console.warn(` - ${r.id}: command runs but produces no output`);
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
if (autoFixed.length > 0) {
|
|
1459
|
+
console.log(`EVIDENCE PRE-VALIDATION: ${autoFixed.length} AC(s) auto-fixed: ${autoFixed.map((r) => r.id).join(", ")}`);
|
|
1460
|
+
}
|
|
1461
|
+
if (broken.length === 0 && empty.length === 0) {
|
|
1462
|
+
console.log(`EVIDENCE PRE-VALIDATION: all ${reading.acs.filter((r) => r.status === "ok").length} evidence commands validated OK`);
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
export async function runScopePreflights(state: Record<string, any>, harnessRoot: string): Promise<void> {
|
|
1467
|
+
// Duplicate slug check (SC-44)
|
|
1468
|
+
const existingSlug = slugExists(state.issue);
|
|
1469
|
+
if (existingSlug && existingSlug !== state.slug) {
|
|
1470
|
+
console.warn(`WARN: Issue #${state.issue} already has slug "${existingSlug}" — current slug is "${state.slug}"`);
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1473
|
+
// Container rebuild check (SC-7)
|
|
1474
|
+
const canRebuild = shouldRebuildContainer(state.projectRoot || process.cwd());
|
|
1475
|
+
if (!canRebuild) {
|
|
1476
|
+
console.log("Container rebuild: SKIP (prod.rebuild is null)");
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
// Worktree branch naming (SC-136)
|
|
1480
|
+
if (state.worktree) {
|
|
1481
|
+
const branch = worktreeBranchName(state.issue, state.slug);
|
|
1482
|
+
console.log(`Worktree branch: ${branch}`);
|
|
1483
|
+
if (state.worktree?.activePorts) {
|
|
1484
|
+
const collision = checkPortCollision(state.worktree.basePort || 3000, state.worktree.offset || 0, state.worktree.activePorts);
|
|
1485
|
+
if (collision) console.error(`WARN: Port collision detected for offset ${state.worktree.offset}`);
|
|
1486
|
+
}
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
// File-set overlap detection (SC-69)
|
|
1490
|
+
if (state.parallelFiles && state.briefFiles) {
|
|
1491
|
+
const overlap = detectFileSetOverlap(state.parallelFiles, state.briefFiles);
|
|
1492
|
+
if (overlap.length > 0) {
|
|
1493
|
+
console.error(`WARN: File overlap with parallel issue: ${overlap.join(", ")}`);
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
// Spec-compliance check
|
|
1498
|
+
try {
|
|
1499
|
+
execSync(`bun scripts/sync-spec-tests.ts 2>&1`, { encoding: "utf-8", timeout: 10000, cwd: harnessRoot });
|
|
1500
|
+
} catch { /* sync script failure is non-blocking */ }
|
|
1501
|
+
|
|
1502
|
+
// Three outcomes, because there are three facts (#197). This used to print
|
|
1503
|
+
// `SPEC DRIFT DETECTED: spec-compliance tests failed` with an empty failure
|
|
1504
|
+
// list for every fixture project in the suite, because fixtures are not
|
|
1505
|
+
// this repo and have no spec-compliance suite — and `execSync` throws the
|
|
1506
|
+
// same way whether a test failed or the file was absent.
|
|
1507
|
+
{
|
|
1508
|
+
const missing = missingSpecComplianceFiles(harnessRoot);
|
|
1509
|
+
let ok = true;
|
|
1510
|
+
let output = "";
|
|
1511
|
+
if (missing.length === 0) {
|
|
1512
|
+
try {
|
|
1513
|
+
output = execSync(
|
|
1514
|
+
`bun test ${SPEC_COMPLIANCE_FILES.join(" ")} 2>&1`,
|
|
1515
|
+
{ encoding: "utf-8", timeout: 30000, cwd: harnessRoot }
|
|
1516
|
+
);
|
|
1517
|
+
} catch (e) {
|
|
1518
|
+
ok = false;
|
|
1519
|
+
output = execErrorOutput(e);
|
|
1520
|
+
}
|
|
1521
|
+
}
|
|
1522
|
+
const v = classifySpecComplianceRun({ ok, output, missing });
|
|
1523
|
+
if (v.verdict === "DRIFT") {
|
|
1524
|
+
console.error(`SPEC DRIFT DETECTED: ${v.detail}`);
|
|
1525
|
+
console.error(`Run: bun test test/spec-compliance.test.ts — to see which claims drifted`);
|
|
1526
|
+
if (v.failures.length) console.error(v.failures.join("\n"));
|
|
1527
|
+
} else if (v.verdict === "UNMEASURED") {
|
|
1528
|
+
// Not an alarm, and not silence either. An alarm naming zero drifted
|
|
1529
|
+
// claims teaches its reader to skip the line, and a line everyone skips
|
|
1530
|
+
// is the same as no line on the run where it matters.
|
|
1531
|
+
//
|
|
1532
|
+
// But "this project has no suite" and "this project has the suite and
|
|
1533
|
+
// it did not produce a reading" are not the same news. The first is an
|
|
1534
|
+
// ordinary fact about a fixture; the second is a timeout, a crash, or a
|
|
1535
|
+
// suite that ran nothing in a repo that is supposed to run it, and it
|
|
1536
|
+
// needs the same visibility a failure gets. Only the first stays quiet.
|
|
1537
|
+
const noSuiteHere = missing.length >= SPEC_COMPLIANCE_FILES.length;
|
|
1538
|
+
const report = noSuiteHere ? console.log : console.error;
|
|
1539
|
+
report(`Spec compliance: UNMEASURED — ${v.detail}`);
|
|
1540
|
+
} else {
|
|
1541
|
+
console.log(`Spec compliance: ${v.detail}`);
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
// Capture test baseline — pre-existing failures (#599, #307)
|
|
1546
|
+
//
|
|
1547
|
+
// An unmeasured baseline records NO number. Writing 0 beside a reason would
|
|
1548
|
+
// leave `?? 0` at the verify gate reading it as a measured clean tree, and
|
|
1549
|
+
// the two keys would be a second answer to the same question. Absent is the
|
|
1550
|
+
// only encoding a default cannot absorb.
|
|
1551
|
+
const testBaseline = captureTestBaseline(state);
|
|
1552
|
+
state.gateContract = state.gateContract || {};
|
|
1553
|
+
state.gateContract.baselines = state.gateContract.baselines || {};
|
|
1554
|
+
recordTestBaseline(state.gateContract.baselines, testBaseline);
|
|
1555
|
+
if (testBaseline.status === "measured") {
|
|
1556
|
+
console.log(
|
|
1557
|
+
testBaseline.failures > 0
|
|
1558
|
+
? `Test baseline: ${testBaseline.failures} pre-existing failure(s) captured — will be subtracted at verify`
|
|
1559
|
+
: `Test baseline: 0 pre-existing failures — clean suite`,
|
|
1560
|
+
);
|
|
1561
|
+
} else {
|
|
1562
|
+
console.error(`Test baseline: UNMEASURED — ${testBaseline.reason}`);
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
// Track pre-existing failures (SC-33)
|
|
1566
|
+
if (state.preExistingFailures?.length > 0) {
|
|
1567
|
+
trackPreExistingFailures(state.slug, state.preExistingFailures);
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
await stampEvidencePrevalidation(state, "scope");
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
/**
|
|
1574
|
+
* Take the evidence pre-validation reading and stamp it on the state (#598).
|
|
1575
|
+
*
|
|
1576
|
+
* AWAITED (#235). This used to be `prevalidateEvidence(...).then(...)` with no
|
|
1577
|
+
* await: the function returned immediately, the gate wrote workflow-state.json
|
|
1578
|
+
* and launched the gate tests, and the verdict landed in stdout some time
|
|
1579
|
+
* afterwards — too late for anything to read, and with rejections swallowed
|
|
1580
|
+
* into a warning that looked the same as a clean run.
|
|
1581
|
+
*
|
|
1582
|
+
* One function rather than an inline block because there are now two gates
|
|
1583
|
+
* that need it (#261) and a second copy of this is a second source of truth
|
|
1584
|
+
* for a field whose whole purpose is to be the single reading.
|
|
1585
|
+
*
|
|
1586
|
+
* The cwd is `EVIDENCE_CWD` first (#264), which is what the evidence COLLECTOR
|
|
1587
|
+
* at `evidenceCwdForRefs` and `evidenceCwd` already uses. `workflows/ship.js`
|
|
1588
|
+
* sets it exactly when the gate's working directory differs from the project
|
|
1589
|
+
* root — that is, under the worktree isolation `marcus` always runs with. This
|
|
1590
|
+
* line read `state.projectRoot` and so dry-ran the branch's commands in the
|
|
1591
|
+
* MAIN repo, where the branch's new files do not exist, while the collector
|
|
1592
|
+
* ran the same commands in the worktree, where they do. One set of commands,
|
|
1593
|
+
* two directories, opposite answers; on run wf_c71f950f-c88 the three commands
|
|
1594
|
+
* this called broken reported 18 pass in the worktree. A pre-validator that
|
|
1595
|
+
* does not look where the evidence will be collected is not pre-validating
|
|
1596
|
+
* anything.
|
|
1597
|
+
*/
|
|
1598
|
+
export async function stampEvidencePrevalidation(
|
|
1599
|
+
state: Record<string, any>,
|
|
1600
|
+
takenAt: PrevalidationSource,
|
|
1601
|
+
): Promise<void> {
|
|
1602
|
+
if (!(state.acs?.length > 0)) return;
|
|
1603
|
+
|
|
1604
|
+
const evidenceCwd = process.env.EVIDENCE_CWD || state.projectRoot || process.cwd();
|
|
1605
|
+
const reading = { ...(await measureEvidencePrevalidation(state.acs, evidenceCwd)), takenAt };
|
|
1606
|
+
|
|
1607
|
+
// Validated before it is stamped, not after: an unparseable reading must
|
|
1608
|
+
// not reach the file at all, and the refusal has to leave the state saying
|
|
1609
|
+
// "unmeasured" rather than silently saying nothing.
|
|
1610
|
+
const parsed = EvidencePrevalidationSchema.safeParse(reading);
|
|
1611
|
+
state.evidencePrevalidation = parsed.success
|
|
1612
|
+
? reading
|
|
1613
|
+
: {
|
|
1614
|
+
verdict: "UNMEASURED",
|
|
1615
|
+
checkedAt: reading.checkedAt,
|
|
1616
|
+
takenAt,
|
|
1617
|
+
acs: [],
|
|
1618
|
+
reason:
|
|
1619
|
+
"the evidence pre-validation reading did not validate against EvidencePrevalidationSchema: " +
|
|
1620
|
+
parsed.error.issues.map((i) => `${i.path.join(".")}: ${i.message}`).join("; ").slice(0, 400),
|
|
1621
|
+
};
|
|
1622
|
+
|
|
1623
|
+
reportPrevalidation(state.evidencePrevalidation as EvidencePrevalidationReading);
|
|
1624
|
+
}
|
|
1625
|
+
|
|
1626
|
+
/**
|
|
1627
|
+
* Make sure the reading exists before the verify gate's suite reads for it
|
|
1628
|
+
* (#261).
|
|
1629
|
+
*
|
|
1630
|
+
* The scope gate was the only writer, and `workflows/ship.js` skips the scope
|
|
1631
|
+
* gate outright when Discovery sizes a run LIGHT:
|
|
1632
|
+
*
|
|
1633
|
+
* const skipScope = discovery.ceremonyTier === 'LIGHT'
|
|
1634
|
+
*
|
|
1635
|
+
* The consumer — `evidence-prevalidation` in gates/workflow.test.ts — is
|
|
1636
|
+
* guarded by `isVerifyPlus()`, so it ran anyway, and an absent reading is a
|
|
1637
|
+
* refusal (#235, correctly: "the writer never ran" and "the writer ran and
|
|
1638
|
+
* measured nothing" are the same failure). Composed, EVERY LIGHT-tier run
|
|
1639
|
+
* failed its verify gate on every attempt, regardless of the code it was
|
|
1640
|
+
* shipping, and no change to that code could have healed it. Run
|
|
1641
|
+
* wf_ed24496e-e49 reached VERIFY with all six ACs PASS and spent two heal
|
|
1642
|
+
* rounds against the gate's own plumbing.
|
|
1643
|
+
*
|
|
1644
|
+
* Same root as #126 with the sign flipped: there a check that could not fail,
|
|
1645
|
+
* here a check that could not pass, both because a check's phase did not imply
|
|
1646
|
+
* its producer's phase.
|
|
1647
|
+
*
|
|
1648
|
+
* A SCOPE reading is preserved. It measured this run's starting conditions,
|
|
1649
|
+
* and re-measuring them after Marcus has run answers a different question than
|
|
1650
|
+
* the one the scope gate's refusal was about.
|
|
1651
|
+
*
|
|
1652
|
+
* A VERIFY reading is RE-TAKEN (#264). The first version of this said "only
|
|
1653
|
+
* when absent", which froze both kinds alike, and that is the half of #261 the
|
|
1654
|
+
* fix for #261 reintroduced. Run wf_c71f950f-c88 made four verify attempts
|
|
1655
|
+
* over thirteen minutes with all six of its ACs PASS, every one of them judged
|
|
1656
|
+
* against one reading taken at 00:29 — so the heal rounds had nothing they
|
|
1657
|
+
* could change. A verify attempt is a new measurement opportunity by
|
|
1658
|
+
* definition: the whole point of the attempt is that the branch moved.
|
|
1659
|
+
*
|
|
1660
|
+
* "A reading exists" cannot tell those two apart, which is why the reading now
|
|
1661
|
+
* records which gate took it rather than being inferred from the tier. Absent
|
|
1662
|
+
* means a run older than this field, and absent is preserved — the behaviour
|
|
1663
|
+
* it already had.
|
|
1664
|
+
*
|
|
1665
|
+
* The reading does not mean quite the same thing on the two tiers, and that is
|
|
1666
|
+
* worth saying out loud rather than leaving for someone to infer wrongly. At
|
|
1667
|
+
* scope it means "these evidence commands were already broken before any work
|
|
1668
|
+
* started" — #598's point, which is to stop Marcus running against commands
|
|
1669
|
+
* that cannot produce evidence. Here it means "these evidence commands are
|
|
1670
|
+
* broken now". For the gate's actual question — can this run's claims be
|
|
1671
|
+
* evidenced? — the verify-time reading is the more relevant of the two, and a
|
|
1672
|
+
* LIGHT run has no earlier one to prefer. Nothing downstream consumes the
|
|
1673
|
+
* reading's auto-fix suggestions, so taking it late costs nothing but the
|
|
1674
|
+
* earliness #598 wanted, which a LIGHT run was never going to get.
|
|
1675
|
+
*
|
|
1676
|
+
* Persisting is the other half, and `runScopePreflightsAndPersist`'s comment
|
|
1677
|
+
* already says why: `runGateTests` spawns bun against workflow-state.json as
|
|
1678
|
+
* it stands on disk, so a reading written after the tests start is a reading
|
|
1679
|
+
* they never see. The write is one statement after the await for the same
|
|
1680
|
+
* reason it is there.
|
|
1681
|
+
*/
|
|
1682
|
+
export async function ensurePrevalidationPersisted(
|
|
1683
|
+
state: Record<string, any>,
|
|
1684
|
+
sf: string,
|
|
1685
|
+
takenAt: "verify" | "ship" = "verify",
|
|
1686
|
+
): Promise<boolean> {
|
|
1687
|
+
if (!shouldRetakePrevalidation(state.evidencePrevalidation, takenAt)) return false;
|
|
1688
|
+
await stampEvidencePrevalidation(state, takenAt);
|
|
1689
|
+
writeFileSync(sf, JSON.stringify(state, null, 2));
|
|
1690
|
+
return true;
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
/**
|
|
1694
|
+
* Which gates take their own pre-validation reading, as one executable
|
|
1695
|
+
* function rather than a branch per gate in `runGate` (#280).
|
|
1696
|
+
*
|
|
1697
|
+
* `runGate` had `if (gate === "verify")` and nothing for ship, so the ship
|
|
1698
|
+
* gate consumed whatever block verify had left behind. The shape of that bug
|
|
1699
|
+
* is why this is a function: a per-gate branch inside a 300-line dispatch is
|
|
1700
|
+
* invisible to every test that does not spawn the whole gate, and the gate
|
|
1701
|
+
* spawns bun. Adding the ship branch there would have left the mapping
|
|
1702
|
+
* unprovable — measured, not assumed: with the branch inline, deleting it
|
|
1703
|
+
* scored ZERO red across 52 tests.
|
|
1704
|
+
*
|
|
1705
|
+
* The scope gate is absent on purpose. It takes its reading through
|
|
1706
|
+
* `runScopePreflightsAndPersist`, which has an ordering constraint of its own
|
|
1707
|
+
* (measure, then persist, before `runGateTests` spawns bun against the file),
|
|
1708
|
+
* and folding it in here would put one of the two writes out of that order.
|
|
1709
|
+
*/
|
|
1710
|
+
export async function prevalidationForGate(
|
|
1711
|
+
gate: string,
|
|
1712
|
+
state: Record<string, any>,
|
|
1713
|
+
sf: string,
|
|
1714
|
+
): Promise<boolean> {
|
|
1715
|
+
if (gate !== "verify" && gate !== "ship") return false;
|
|
1716
|
+
return ensurePrevalidationPersisted(state, sf, gate);
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
/**
|
|
1720
|
+
* Does the gate now running need its OWN reading, or does the one on the state
|
|
1721
|
+
* still answer its question?
|
|
1722
|
+
*
|
|
1723
|
+
* One exported predicate, so there is one binding to mutate and one place the
|
|
1724
|
+
* policy is written down. Every previous fix in this family moved the boundary
|
|
1725
|
+
* one phase and stopped — #261 (producer in a phase the consumer did not
|
|
1726
|
+
* share), #264 (reading taken in the wrong directory and never re-taken on
|
|
1727
|
+
* heal), #280 (reading not re-taken at the next phase). The condition used to
|
|
1728
|
+
* be inline and phase-blind, which is how each of them got through.
|
|
1729
|
+
*
|
|
1730
|
+
* **At verify, a SCOPE reading survives.** It measured the run's starting
|
|
1731
|
+
* conditions, and re-measuring those after Marcus has run answers a different
|
|
1732
|
+
* question than the one the scope gate refuses on. A VERIFY reading does not
|
|
1733
|
+
* survive: the whole premise of another verify attempt is that the branch
|
|
1734
|
+
* moved, and freezing the reading means no heal round can clear a refusal.
|
|
1735
|
+
*
|
|
1736
|
+
* **At ship, nothing survives — this is #280's decision, taken explicitly
|
|
1737
|
+
* rather than inherited (SC-5).** The old condition, `takenAt !== "verify"`,
|
|
1738
|
+
* would have preserved a scope reading all the way to ship. That is defensible
|
|
1739
|
+
* one phase later and indefensible three: between verify and ship the run
|
|
1740
|
+
* commits, pushes, and may move the branch, and on run `wf_ea3e428f-64f` the
|
|
1741
|
+
* implementation never reached `projectRoot` at all, so the two readings
|
|
1742
|
+
* differed by four ACs. A reading whose own `takenAt` says it describes an
|
|
1743
|
+
* earlier tree is not evidence about the tree being shipped. Re-measuring is a
|
|
1744
|
+
* dry-run of commands that already ran once; the cost of taking it again is
|
|
1745
|
+
* far below the cost of a ship gate recording a stale artefact as current.
|
|
1746
|
+
*
|
|
1747
|
+
* An ABSENT reading is always re-taken, on both — that is the #261 case, and
|
|
1748
|
+
* it is the behaviour this already had.
|
|
1749
|
+
*/
|
|
1750
|
+
export function shouldRetakePrevalidation(
|
|
1751
|
+
existing: unknown,
|
|
1752
|
+
takenAt: "verify" | "ship",
|
|
1753
|
+
): boolean {
|
|
1754
|
+
if (!existing || typeof existing !== "object") return true;
|
|
1755
|
+
if (takenAt === "ship") return true;
|
|
1756
|
+
return (existing as { takenAt?: unknown }).takenAt === "verify";
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
/**
|
|
1760
|
+
* The scope gate's pre-flight step: measure, then persist — in that order.
|
|
1761
|
+
*
|
|
1762
|
+
* The ordering is the whole point. `runGateTests` spawns bun against
|
|
1763
|
+
* workflow-state.json as it stands on disk, so a reading written after the
|
|
1764
|
+
* tests start is a reading they never see, and a gate check on a field that is
|
|
1765
|
+
* always absent is a check that cannot fail. The write is here, one statement
|
|
1766
|
+
* after the await, so the two cannot drift apart.
|
|
1767
|
+
*
|
|
1768
|
+
* writeFileSync rather than writeWorkflowState on purpose: the scope gate's
|
|
1769
|
+
* job is to REPORT an invalid state as a FAIL, and writeWorkflowState throws
|
|
1770
|
+
* on one, which would turn that report into a crash. The reading itself is
|
|
1771
|
+
* schema-validated in runScopePreflights before it is stamped on.
|
|
1772
|
+
*/
|
|
1773
|
+
export async function runScopePreflightsAndPersist(
|
|
1774
|
+
state: Record<string, any>,
|
|
1775
|
+
sf: string,
|
|
1776
|
+
harnessRoot: string,
|
|
1777
|
+
): Promise<void> {
|
|
1778
|
+
await runScopePreflights(state, harnessRoot);
|
|
1779
|
+
writeFileSync(sf, JSON.stringify(state, null, 2));
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
// ── Verify pre-flight checks ────────────────────────────────────────────
|
|
1783
|
+
|
|
1784
|
+
function runVerifyPreflights(state: Record<string, any>): void {
|
|
1785
|
+
// Blast radius check (SC-59, SC-60)
|
|
1786
|
+
if (state.blastRadius) {
|
|
1787
|
+
const { filesRead = 0, filesChanged = 0 } = state.blastRadius;
|
|
1788
|
+
if (filesRead < filesChanged) {
|
|
1789
|
+
console.error(`WARN: Blast radius — read ${filesRead} files but changed ${filesChanged} (read should be >= changed)`);
|
|
1790
|
+
}
|
|
1791
|
+
const readWriteRatio = filesRead / Math.max(filesChanged, 1);
|
|
1792
|
+
if (readWriteRatio < 3) {
|
|
1793
|
+
console.error(`WARN: Read/write ratio ${readWriteRatio.toFixed(1)} (target >= 3:1)`);
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
// Fix-on-find check (SC-109)
|
|
1798
|
+
if (state.foundIssues) {
|
|
1799
|
+
const dropped = (state.foundIssues || []).filter((i: any) => !i.disposition);
|
|
1800
|
+
if (dropped.length > 0) {
|
|
1801
|
+
console.error(`FAIL: ${dropped.length} found issues silently dropped — must be fixed, filed, or scoped out`);
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
// Files changed outside brief scope (SC-134)
|
|
1806
|
+
if (state.filesChangedOutsideBrief?.length > 0) {
|
|
1807
|
+
console.error(`WARN: Files changed outside brief: ${state.filesChangedOutsideBrief.join(", ")}`);
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
// Scaffold refresh check (SC-38)
|
|
1811
|
+
if (shouldRefreshScaffold(state.slug)) {
|
|
1812
|
+
console.log("Post-verify: scaffold refresh recommended (verify gate PASS)");
|
|
1813
|
+
}
|
|
1814
|
+
|
|
1815
|
+
// Research escalation check (SC-112)
|
|
1816
|
+
if (requiresResearchEscalation(state.slug)) {
|
|
1817
|
+
console.log("Research escalation: iteration 2+ — research tool invocation required before next attempt");
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
// ── Type check ──────────────────────────────────────────────────────────
|
|
1822
|
+
|
|
1823
|
+
export function typeCheckCommand(state: Record<string, any>): string | null {
|
|
1824
|
+
// State first, config second — unchanged. What changed is that the config
|
|
1825
|
+
// half can now see a directory-layout project, which it could not, so a
|
|
1826
|
+
// consumer that configured `dev.typeCheck` and got no type check and no
|
|
1827
|
+
// message is no longer a reachable state (#311 SC-A8).
|
|
1828
|
+
//
|
|
1829
|
+
// `null` is a real authored value here: #87 made `typeCheck` nullable so
|
|
1830
|
+
// the scaffold can record "we looked and there is nothing to type-check",
|
|
1831
|
+
// which is a different fact from nobody configuring it. Both end in "do
|
|
1832
|
+
// not run a command", so this returns `null` for both and the caller does
|
|
1833
|
+
// not have to know which.
|
|
1834
|
+
const reading = readGateConfig(state.projectRoot || process.cwd());
|
|
1835
|
+
reportUnreadable("type check", reading);
|
|
1836
|
+
const fromConfig = reading.status === "resolved" ? reading.config.dev?.typeCheck : undefined;
|
|
1837
|
+
return state.dev?.typeCheck || fromConfig || null;
|
|
1838
|
+
}
|
|
1839
|
+
|
|
1840
|
+
function runTypeCheck(state: Record<string, any>): void {
|
|
1841
|
+
const typeCheckCmd = typeCheckCommand(state);
|
|
1842
|
+
if (typeCheckCmd) {
|
|
1843
|
+
try {
|
|
1844
|
+
execSync(typeCheckCmd, { encoding: "utf-8", timeout: 30000, cwd: state.projectRoot || process.cwd() });
|
|
1845
|
+
console.log(`Type check PASS: ${typeCheckCmd}`);
|
|
1846
|
+
} catch (e: any) {
|
|
1847
|
+
console.error(`Type check FAIL: ${typeCheckCmd}`);
|
|
1848
|
+
console.error((e.stdout || "").split("\n").slice(0, 10).join("\n"));
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
// ── Test execution ──────────────────────────────────────────────────────
|
|
1854
|
+
|
|
1855
|
+
/**
|
|
1856
|
+
* Where the run's JUnit report goes (#532).
|
|
1857
|
+
*
|
|
1858
|
+
* The console output names only the FAILURES — bun 1.3.14 prints no `(pass)`
|
|
1859
|
+
* markers at all — so it cannot say which checks a gate ran and did not
|
|
1860
|
+
* fail, which is the one thing refusal retirement needs. The JUnit reporter
|
|
1861
|
+
* writes alongside the console output rather than replacing it, so a single
|
|
1862
|
+
* spawn still feeds the detail parser its `(fail)` lines.
|
|
1863
|
+
*/
|
|
1864
|
+
export function junitReportPath(workDir: string, gate: string): string {
|
|
1865
|
+
return join(workDir, `gate-${gate}-junit.xml`);
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
function runGateTests(gate: string, workDir: string, state: Record<string, any>, harnessRoot: string): { testOutput: string; testExitCode: number; junitPath: string } {
|
|
1869
|
+
let testOutput: string;
|
|
1870
|
+
let testExitCode = 0;
|
|
1871
|
+
const junitPath = junitReportPath(workDir, gate);
|
|
1872
|
+
try {
|
|
1873
|
+
testOutput = execSync(
|
|
1874
|
+
`bun test gates/ --test-name-pattern "${gate === 'scope' ? 'schema|scope' : gate === 'verify' ? 'schema|scope|verify|B1|B2' : gate === 'ship' ? 'schema|scope|verify|ship|B1|B2' : gate === 'prove' ? 'prove' : '.*'}" --reporter=junit --reporter-outfile=${JSON.stringify(junitPath)} 2>&1`,
|
|
1875
|
+
{
|
|
1876
|
+
encoding: "utf-8",
|
|
1877
|
+
timeout: GATE_SUITE_TIMEOUT_MS,
|
|
1878
|
+
cwd: harnessRoot,
|
|
1879
|
+
env: { ...process.env, TEST_WORK_DIR: workDir, GATE: gate, PROJECT_ROOT: state.projectRoot || process.cwd() },
|
|
1880
|
+
},
|
|
1881
|
+
);
|
|
1882
|
+
} catch (e: any) {
|
|
1883
|
+
testOutput = e.stdout || "";
|
|
1884
|
+
testExitCode = e.status || 1;
|
|
1885
|
+
}
|
|
1886
|
+
return { testOutput, testExitCode, junitPath };
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
// ── Result parsing ──────────────────────────────────────────────────────
|
|
1890
|
+
|
|
1891
|
+
/**
|
|
1892
|
+
* Exported so a case can drive it (#737 SC-3).
|
|
1893
|
+
*
|
|
1894
|
+
* The two readings it composes were scoped differently for the whole of
|
|
1895
|
+
* this function's life, and nothing could see it because nothing called it
|
|
1896
|
+
* from a test — the scoping was only observable through the two numbers it
|
|
1897
|
+
* printed, eleven lines apart, in a log.
|
|
1898
|
+
*/
|
|
1899
|
+
export function parseGateResults(testOutput: string, earlyResults: GateResult[], earlyFails: number): { passes: number; fails: number; warns: number; results: GateResult[] } {
|
|
1900
|
+
let passes = 0;
|
|
1901
|
+
let fails = 0;
|
|
1902
|
+
const warns = 0;
|
|
1903
|
+
const results: GateResult[] = [];
|
|
1904
|
+
|
|
1905
|
+
// One split, two readings off it (#737).
|
|
1906
|
+
//
|
|
1907
|
+
// These used to be scoped differently: the summary took the last `N fail`
|
|
1908
|
+
// in the WHOLE buffer while `parseTestResults` was handed the whole buffer
|
|
1909
|
+
// and pushed a result for every `(fail)` marker in it. On a retried gate
|
|
1910
|
+
// that is `1 fail` beside `2 failures`, eleven lines apart, with one check
|
|
1911
|
+
// named twice — two deciders for one question, agreeing on every
|
|
1912
|
+
// single-run output and diverging only on the retry nobody fixtures.
|
|
1913
|
+
//
|
|
1914
|
+
// Last attempt wins; `lib/gate-run-split.ts` argues why and
|
|
1915
|
+
// `specs/GATE-CONTRACTS-SPEC.md` records it.
|
|
1916
|
+
const buffer = splitGateRuns(testOutput);
|
|
1917
|
+
|
|
1918
|
+
if (buffer.verdict === null) {
|
|
1919
|
+
// A run that printed no summary did not report zero failures — it
|
|
1920
|
+
// reported nothing. The pre-#737 code took whichever earlier run had
|
|
1921
|
+
// the last matching line, which is a superseded attempt reported as
|
|
1922
|
+
// this one's result.
|
|
1923
|
+
results.push({
|
|
1924
|
+
check: "test-suite",
|
|
1925
|
+
result: "FAIL",
|
|
1926
|
+
detail: `UNMEASURED: ${buffer.unmeasuredReasons.join("; ")}`,
|
|
1927
|
+
});
|
|
1928
|
+
if (earlyResults.length > 0) {
|
|
1929
|
+
results.push(...earlyResults);
|
|
1930
|
+
fails += earlyFails;
|
|
1931
|
+
}
|
|
1932
|
+
return { passes: 0, fails: fails + 1, warns, results };
|
|
1933
|
+
}
|
|
1934
|
+
|
|
1935
|
+
passes = buffer.verdict.summary!.pass;
|
|
1936
|
+
fails = buffer.verdict.summary!.fail;
|
|
1937
|
+
|
|
1938
|
+
results.push(...parseTestResults(buffer.verdict.text));
|
|
1939
|
+
|
|
1940
|
+
if (earlyResults.length > 0) {
|
|
1941
|
+
results.push(...earlyResults);
|
|
1942
|
+
fails += earlyFails;
|
|
1943
|
+
}
|
|
1944
|
+
|
|
1945
|
+
if (results.length === earlyResults.length) {
|
|
1946
|
+
if (fails > 0) {
|
|
1947
|
+
results.push({ check: "test-suite", result: "FAIL", detail: `${fails} tests failed` });
|
|
1948
|
+
} else {
|
|
1949
|
+
results.push({ check: "test-suite", result: "PASS", detail: `${passes} tests passed` });
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1953
|
+
return { passes, fails, warns, results };
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
// ── Conformity findings ─────────────────────────────────────────────────
|
|
1957
|
+
|
|
1958
|
+
/**
|
|
1959
|
+
* #213, second verdict-bearing catch.
|
|
1960
|
+
*
|
|
1961
|
+
* The `catch {}` at the end of this function covered both the read and the
|
|
1962
|
+
* parse, so a findings file that exists and cannot be read left
|
|
1963
|
+
* `state.conformityFindings` absent — exactly what a project with no findings
|
|
1964
|
+
* file at all produces. Every consumer tests `if (state.conformityFindings)`,
|
|
1965
|
+
* so "the conformity scan found nothing" and "the conformity scan's output is
|
|
1966
|
+
* corrupt" were the same value, and the second one reads as the first.
|
|
1967
|
+
*
|
|
1968
|
+
* The marker is a distinct shape rather than a thrown error on purpose. This
|
|
1969
|
+
* runs inside gate post-processing after the verdict is computed; making it
|
|
1970
|
+
* fatal would turn a bookkeeping fault into a crash. UNMEASURED has to be
|
|
1971
|
+
* recorded as UNMEASURED, and the one thing it must never do is pass for
|
|
1972
|
+
* clean.
|
|
1973
|
+
*
|
|
1974
|
+
* The error message is withheld from the recorded marker on purpose: a JSON
|
|
1975
|
+
* parse error quotes the bytes it choked on, and this file is machine-written
|
|
1976
|
+
* from a scan of the project. `err.name` says enough to tell a parse failure
|
|
1977
|
+
* from a read failure without copying file contents into workflow-state.json.
|
|
1978
|
+
*/
|
|
1979
|
+
export function unreadableConformityMarker(path: string, err: unknown): Record<string, unknown> {
|
|
1980
|
+
const name = err instanceof Error ? err.name : typeof err;
|
|
1981
|
+
return {
|
|
1982
|
+
unreadable: true,
|
|
1983
|
+
path,
|
|
1984
|
+
reason: name,
|
|
1985
|
+
detail:
|
|
1986
|
+
`${path} exists and could not be read as JSON (${name}). This is UNMEASURED, not an ` +
|
|
1987
|
+
`empty findings list — treat it as "the conformity scan's output is unusable".`,
|
|
1988
|
+
};
|
|
1989
|
+
}
|
|
1990
|
+
|
|
1991
|
+
/** True when a recorded findings value is the unreadable marker above. */
|
|
1992
|
+
export function conformityFindingsUnreadable(findings: unknown): boolean {
|
|
1993
|
+
return !!findings && typeof findings === "object" &&
|
|
1994
|
+
(findings as Record<string, unknown>).unreadable === true;
|
|
1995
|
+
}
|
|
1996
|
+
|
|
1997
|
+
export function readConformityFindings(state: Record<string, any>): void {
|
|
1998
|
+
const projectRoot = state.projectRoot || process.cwd();
|
|
1999
|
+
const findingsPath = join(projectRoot, ".rungate", "conformity-findings.json");
|
|
2000
|
+
if (existsSync(findingsPath)) {
|
|
2001
|
+
try {
|
|
2002
|
+
const findings = JSON.parse(readFileSync(findingsPath, "utf-8"));
|
|
2003
|
+
if (findings.total > 0) {
|
|
2004
|
+
console.log(`\n── CONFORMITY FINDINGS (${findingsPath}) ──`);
|
|
2005
|
+
if (findings.findings?.length > 0) {
|
|
2006
|
+
console.log(` Issues: ${findings.failures} FAIL, ${findings.warnings} WARN`);
|
|
2007
|
+
for (const f of findings.findings) {
|
|
2008
|
+
console.log(` ${f.severity}: ${f.file} — ${f.message}`);
|
|
2009
|
+
if (f.fixCommand) console.log(` Fix: ${f.fixCommand}`);
|
|
2010
|
+
}
|
|
2011
|
+
}
|
|
2012
|
+
if (findings.constraintCandidates?.length > 0) {
|
|
2013
|
+
console.log(` Constraint candidates: ${findings.candidateCount} pending review`);
|
|
2014
|
+
for (const c of findings.constraintCandidates) {
|
|
2015
|
+
console.log(` "${c.rule}" (${c.source})`);
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
if (findings.staleness?.length > 0) {
|
|
2019
|
+
console.log(` Stale docs: ${findings.staleCount}`);
|
|
2020
|
+
for (const s of findings.staleness) {
|
|
2021
|
+
console.log(` ${s.file} — ${s.daysSince}d old (${s.type} threshold: ${s.threshold}d)`);
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
console.log(`──────────────────────────────────────────`);
|
|
2025
|
+
}
|
|
2026
|
+
if (!state.conformityFindings) {
|
|
2027
|
+
state.conformityFindings = findings;
|
|
2028
|
+
}
|
|
2029
|
+
} catch (e) {
|
|
2030
|
+
const marker = unreadableConformityMarker(findingsPath, e);
|
|
2031
|
+
console.error(`WARN: ${marker.detail}`);
|
|
2032
|
+
if (!state.conformityFindings) {
|
|
2033
|
+
state.conformityFindings = marker;
|
|
2034
|
+
}
|
|
2035
|
+
}
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
// ── Batch diagnosis ─────────────────────────────────────────────────────
|
|
2040
|
+
|
|
2041
|
+
/**
|
|
2042
|
+
* How many lines of one failure's own output the diagnosis relays (#700).
|
|
2043
|
+
*
|
|
2044
|
+
* There is a cap because the diagnosis is read at the END of a very long log,
|
|
2045
|
+
* and an unbounded relay reintroduces the problem it exists to solve from the
|
|
2046
|
+
* other side: #726 removed a 3000-character window because it discarded the
|
|
2047
|
+
* answer, and a reporter that prints a megabyte discards it too — one runaway
|
|
2048
|
+
* failure buries every other check's reason.
|
|
2049
|
+
*
|
|
2050
|
+
* It is a LINE cap and not a character cap on purpose. Bun's excerpts are
|
|
2051
|
+
* source quotes, so a character cut lands mid-token and the operator cannot
|
|
2052
|
+
* tell a truncated line from a short one. And when it fires it SAYS so, with
|
|
2053
|
+
* the number it dropped — a relay that just stops is indistinguishable from a
|
|
2054
|
+
* failure that had nothing more to say.
|
|
2055
|
+
*/
|
|
2056
|
+
export const DETAIL_LINE_CAP = 12;
|
|
2057
|
+
|
|
2058
|
+
/** What a failure carrying no usable detail reports. */
|
|
2059
|
+
export const NO_DETAIL_RECORDED = "no detail was recorded for this failure";
|
|
2060
|
+
|
|
2061
|
+
/**
|
|
2062
|
+
* The batch diagnosis, as lines, with each failure's own reason under it.
|
|
2063
|
+
*
|
|
2064
|
+
* ONE binding, exported, and `printBatchDiagnosis` is its only caller — so
|
|
2065
|
+
* there is one thing a mutation battery can short-circuit, and a second
|
|
2066
|
+
* composition inlined at the call site cannot hold a case up while this one
|
|
2067
|
+
* is broken (`.claude/rules/checks-must-be-able-to-fail.md`, fourth shape).
|
|
2068
|
+
*
|
|
2069
|
+
* ## Why this changed (#700)
|
|
2070
|
+
*
|
|
2071
|
+
* `parseTestResults` has recovered bun's error excerpt for every failing
|
|
2072
|
+
* check since #314, and stores it on `GateResult.detail`. This function
|
|
2073
|
+
* printed `r.check` and dropped `detail` on the floor one line later.
|
|
2074
|
+
*
|
|
2075
|
+
* What that cost, with the run beside it. #700's STANDARD-tier trio
|
|
2076
|
+
* reproduces only in CI, which makes a red run the scarce resource. #726
|
|
2077
|
+
* widened `expectGateGreen` so the gate's output survives, and the next red —
|
|
2078
|
+
* `main` at `71e3f413`, run `39325676895` — printed:
|
|
2079
|
+
*
|
|
2080
|
+
* BATCH DIAGNOSIS: 2 failures — fix all before re-running
|
|
2081
|
+
* AC/Evidence (2): brief-ac-alignment: brief covers all ACs,
|
|
2082
|
+
* brief-ac-alignment: brief covers all ACs
|
|
2083
|
+
*
|
|
2084
|
+
* A name, and only a name. `gates/workflow.test.ts:1212` fails two different
|
|
2085
|
+
* ways — the brief file absent at `:1221`, or present and missing an id at
|
|
2086
|
+
* `:1230` — and those are different bugs with different fixes. `grep -c` over
|
|
2087
|
+
* that entire run gives 0 for `brief missing AC ids` and 0 for `brief file
|
|
2088
|
+
* required when Marcus spawned`. The answer was on the result object and
|
|
2089
|
+
* never reached the log.
|
|
2090
|
+
*
|
|
2091
|
+
* ## The shape of the output, and why it is not a comma list
|
|
2092
|
+
*
|
|
2093
|
+
* The cheapest wrong fix is to append `detail` to the joined name list. One
|
|
2094
|
+
* expression, the detail stops being dropped, and N failures become one
|
|
2095
|
+
* paragraph with no boundary between them — the reason present and still not
|
|
2096
|
+
* attributable, which is this defect wearing a fix. So each failure gets its
|
|
2097
|
+
* own line at one indent and its reason beneath it at another, and
|
|
2098
|
+
* `test/batch-diagnosis-detail.test.ts` asserts attribution by SPAN rather
|
|
2099
|
+
* than by "both strings appear somewhere", because both strings appearing
|
|
2100
|
+
* somewhere is exactly what the wrong fix achieves.
|
|
2101
|
+
*/
|
|
2102
|
+
export function batchDiagnosisLines(fails: number, results: GateResult[]): string[] {
|
|
2103
|
+
if (fails <= 0) return [];
|
|
2104
|
+
const failResults = results.filter(r => r.result === "FAIL");
|
|
2105
|
+
if (failResults.length === 0) return [];
|
|
2106
|
+
|
|
2107
|
+
const infra = failResults.filter(r => /B1|B2|B3|adversary|reproducer|health|port/.test(r.check));
|
|
2108
|
+
const acRelated = failResults.filter(r => /evidence|verdict|AC|ac[- ]|threshold/.test(r.check));
|
|
2109
|
+
const chain = failResults.filter(r => /hash|chain|pushed|merge|spec-trace/.test(r.check));
|
|
2110
|
+
const other = failResults.filter(r => !infra.includes(r) && !acRelated.includes(r) && !chain.includes(r));
|
|
2111
|
+
|
|
2112
|
+
const lines = [`BATCH DIAGNOSIS: ${failResults.length} failures — fix all before re-running`];
|
|
2113
|
+
|
|
2114
|
+
for (const [label, group] of [
|
|
2115
|
+
["Infrastructure", infra],
|
|
2116
|
+
["AC/Evidence", acRelated],
|
|
2117
|
+
["Chain integrity", chain],
|
|
2118
|
+
["Other", other],
|
|
2119
|
+
] as const) {
|
|
2120
|
+
if (group.length === 0) continue;
|
|
2121
|
+
lines.push(` ${label} (${group.length}):`);
|
|
2122
|
+
for (const r of group) {
|
|
2123
|
+
lines.push(` - ${r.check.split(" > ").pop()}`);
|
|
2124
|
+
// An empty or whitespace-only detail is reported as absent, never as a
|
|
2125
|
+
// blank line. A blank line under a check name reads as a reason that
|
|
2126
|
+
// was asked for and came back empty — `0`, `{}` and silence are all
|
|
2127
|
+
// indistinguishable from success (`.claude/rules/config-access.md`).
|
|
2128
|
+
const detail = (r.detail ?? "").split("\n").map(l => l.trimEnd()).filter(l => l.trim().length > 0);
|
|
2129
|
+
if (detail.length === 0) {
|
|
2130
|
+
lines.push(` ${NO_DETAIL_RECORDED}`);
|
|
2131
|
+
continue;
|
|
2132
|
+
}
|
|
2133
|
+
for (const l of detail.slice(0, DETAIL_LINE_CAP)) lines.push(` ${l}`);
|
|
2134
|
+
if (detail.length > DETAIL_LINE_CAP) {
|
|
2135
|
+
lines.push(` … ${detail.length - DETAIL_LINE_CAP} more lines truncated`);
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
2139
|
+
return lines;
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
function printBatchDiagnosis(fails: number, results: GateResult[]): void {
|
|
2143
|
+
const lines = batchDiagnosisLines(fails, results);
|
|
2144
|
+
if (lines.length === 0) return;
|
|
2145
|
+
console.log("");
|
|
2146
|
+
for (const l of lines) console.log(l);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
// ── B1/B2 agent logic ───────────────────────────────────────────────────
|
|
2150
|
+
|
|
2151
|
+
function spawnB1AdversaryAtScope(state: Record<string, any>, workDir: string, harnessRoot: string): void {
|
|
2152
|
+
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2153
|
+
if (tier === "LIGHT") return;
|
|
2154
|
+
|
|
2155
|
+
const b1ProjectPrompt = join(__dirname, "prompts", "ac-adversary.md");
|
|
2156
|
+
const b1HomePrompt = join(process.env.HOME || "", ".claude", "gates", "prompts", "ac-adversary.md");
|
|
2157
|
+
const promptPath = existsSync(b1ProjectPrompt) ? b1ProjectPrompt : b1HomePrompt;
|
|
2158
|
+
if (!existsSync(promptPath)) {
|
|
2159
|
+
console.warn("WARN: ac-adversary.md prompt not found — adversary will not run");
|
|
2160
|
+
return;
|
|
2161
|
+
}
|
|
2162
|
+
|
|
2163
|
+
const reportPath = join(workDir, "adversary-report.json");
|
|
2164
|
+
try { unlinkSync(reportPath); } catch {}
|
|
2165
|
+
const acInputFile = join(workDir, "adversary-input.json");
|
|
2166
|
+
writeFileSync(acInputFile, JSON.stringify(state.acs || [], null, 2));
|
|
2167
|
+
const { spawn } = require("child_process");
|
|
2168
|
+
const child = spawn("sh", ["-c",
|
|
2169
|
+
`claude -p "$(cat '${acInputFile}')" --model haiku --system-prompt-file "${promptPath}" --output-format json --allowedTools "" --max-turns 3 > "${reportPath}.tmp" 2>/dev/null && mv "${reportPath}.tmp" "${reportPath}"`
|
|
2170
|
+
], { detached: true, stdio: "ignore", cwd: harnessRoot });
|
|
2171
|
+
child.unref();
|
|
2172
|
+
console.log("B1: AC Adversary spawned in background — verify gate will check result");
|
|
2173
|
+
}
|
|
2174
|
+
|
|
2175
|
+
/**
|
|
2176
|
+
* #304 — the probe used to curl a literal `${apiBase}/api/admin/health`.
|
|
2177
|
+
*
|
|
2178
|
+
* `apiBase` came from config; the path did not. It is one consumer's
|
|
2179
|
+
* route, and because this only runs for a project that sets `dev.apiBase` —
|
|
2180
|
+
* which rungate does not — DDB is the only project that has ever reached the
|
|
2181
|
+
* line. It worked for one consumer by coincidence.
|
|
2182
|
+
*
|
|
2183
|
+
* The decision now lives in `livenessPlan` in lib/dev-liveness.ts, which is
|
|
2184
|
+
* pure, so every branch is exercisable without a server. This function does
|
|
2185
|
+
* the one thing that needs a network. Three outcomes, all recorded:
|
|
2186
|
+
*
|
|
2187
|
+
* not-applicable — nothing asked for a server. Silent, as before.
|
|
2188
|
+
* unmeasured — asked for, not configured. WARNed and recorded as a
|
|
2189
|
+
* WARN result, never as a FAIL. A probe with no route
|
|
2190
|
+
* reporting DOWN is a false FAIL on a healthy server, and
|
|
2191
|
+
* its stated remedy is to start something already running.
|
|
2192
|
+
* probe — exactly one http(s) URL, argv-form, no shell.
|
|
2193
|
+
*/
|
|
2194
|
+
/**
|
|
2195
|
+
* The `dev` shape `livenessPlan` asks about, off the normalised names (#311).
|
|
2196
|
+
*
|
|
2197
|
+
* `apiBase` is `apiUrl`, for `proveEnvironment`'s reason: the normaliser
|
|
2198
|
+
* collapses an authored `null` into absence, and a raw read would put the
|
|
2199
|
+
* second spelling of "not set" back in front of a probe that treats the two
|
|
2200
|
+
* differently.
|
|
2201
|
+
*/
|
|
2202
|
+
export function livenessDev(
|
|
2203
|
+
reading: GateConfigReading,
|
|
2204
|
+
): { apiBase?: string; healthPath?: string; start?: string } | undefined {
|
|
2205
|
+
if (reading.status !== "resolved") return undefined;
|
|
2206
|
+
const c = reading.config;
|
|
2207
|
+
return {
|
|
2208
|
+
apiBase: c.apiUrl || undefined,
|
|
2209
|
+
healthPath: c.dev?.healthPath ?? undefined,
|
|
2210
|
+
start: c.dev?.start,
|
|
2211
|
+
};
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
function checkDevServerLiveness(state: Record<string, any>, results: GateResult[]): number {
|
|
2215
|
+
let addedFails = 0;
|
|
2216
|
+
if (!state.projectRoot) return 0;
|
|
2217
|
+
|
|
2218
|
+
// The `catch { harness = null }` that was here is the instance
|
|
2219
|
+
// `.claude/rules/config-access.md` names by rule: it converted corruption
|
|
2220
|
+
// into absence, and absence into a probe that does not happen. Now the
|
|
2221
|
+
// three readings are distinct, and the unreadable one is said out loud
|
|
2222
|
+
// before being treated the way absence is — because the ALTERNATIVE for a
|
|
2223
|
+
// liveness probe with no target is reporting DOWN on a healthy server,
|
|
2224
|
+
// which is #304 and is worse (#311 SC-A8).
|
|
2225
|
+
const reading = readGateConfig(state.projectRoot);
|
|
2226
|
+
reportUnreadable("dev-server liveness", reading);
|
|
2227
|
+
const plan = livenessPlan(state.acs, livenessDev(reading));
|
|
2228
|
+
if (plan.kind === "not-applicable") return 0;
|
|
2229
|
+
|
|
2230
|
+
if (plan.kind === "unmeasured") {
|
|
2231
|
+
results.push({
|
|
2232
|
+
check: "dev-server-liveness: UNMEASURED",
|
|
2233
|
+
result: "WARN" as const,
|
|
2234
|
+
detail: plan.detail,
|
|
2235
|
+
});
|
|
2236
|
+
console.log(`Dev server liveness: UNMEASURED — ${plan.detail}`);
|
|
2237
|
+
return 0;
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
try {
|
|
2241
|
+
// Argv form: both halves of the URL come out of the consumer's own
|
|
2242
|
+
// rungate.json and the old line interpolated them into a shell string.
|
|
2243
|
+
execFileSync("curl", ["-sf", "--max-time", "5", "--", plan.url], {
|
|
2244
|
+
timeout: 5000,
|
|
2245
|
+
encoding: "utf-8",
|
|
2246
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
2247
|
+
});
|
|
2248
|
+
// safeUrl everywhere it is printed or recorded: `detail` lands in
|
|
2249
|
+
// ship-evidence.json, which is durable (#213).
|
|
2250
|
+
console.log(`Dev server liveness: ${plan.safeUrl} — UP`);
|
|
2251
|
+
} catch {
|
|
2252
|
+
addedFails++;
|
|
2253
|
+
results.push({
|
|
2254
|
+
check: "dev-server-liveness: OUTCOME ACs require running dev server",
|
|
2255
|
+
result: "FAIL" as const,
|
|
2256
|
+
detail: `${plan.safeUrl} not responding — run '${plan.startHint}' first`,
|
|
2257
|
+
});
|
|
2258
|
+
console.log(`Dev server liveness: ${plan.safeUrl} — DOWN. Start with: ${plan.startHint}`);
|
|
2259
|
+
}
|
|
2260
|
+
return addedFails;
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
function checkB1ReportAtVerify(state: Record<string, any>, workDir: string, results: GateResult[]): { fails: number; warns: number } {
|
|
2264
|
+
let addedFails = 0;
|
|
2265
|
+
let addedWarns = 0;
|
|
2266
|
+
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2267
|
+
if (tier === "LIGHT") return { fails: 0, warns: 0 };
|
|
2268
|
+
|
|
2269
|
+
const reportPath = join(workDir, "adversary-report.json");
|
|
2270
|
+
if (!existsSync(reportPath)) {
|
|
2271
|
+
addedFails++;
|
|
2272
|
+
results.push({ check: "B1-adversary: report missing", result: "FAIL" as const, detail: "adversary-report.json not found — adversary may still be running or failed. Re-run verify." });
|
|
2273
|
+
console.log("B1: adversary report not yet available — verify FAIL");
|
|
2274
|
+
} else {
|
|
2275
|
+
try {
|
|
2276
|
+
const raw = readFileSync(reportPath, "utf-8");
|
|
2277
|
+
const jsonMatch = raw.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
|
|
2278
|
+
if (jsonMatch) {
|
|
2279
|
+
const report = JSON.parse(jsonMatch[0]);
|
|
2280
|
+
console.log(`B1: Adversary report — gameable: ${report.gameable}, approved: ${report.approved}`);
|
|
2281
|
+
if (report.gameable > 0) {
|
|
2282
|
+
let outcomeCount = 0;
|
|
2283
|
+
for (const exploit of report.exploits || []) {
|
|
2284
|
+
const ac = (state.acs || []).find((a: any) => a.id === exploit.acId);
|
|
2285
|
+
const isOutcome = ac?.type === "OUTCOME";
|
|
2286
|
+
if (isOutcome) {
|
|
2287
|
+
addedFails++;
|
|
2288
|
+
outcomeCount++;
|
|
2289
|
+
results.push({ check: `B1-adversary: ${exploit.acId} gameable (OUTCOME)`, result: "FAIL" as const, detail: exploit.exploit });
|
|
2290
|
+
} else {
|
|
2291
|
+
addedWarns++;
|
|
2292
|
+
results.push({ check: `B1-adversary: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
if (outcomeCount > 0) console.log(`B1: ${outcomeCount} OUTCOME AC(s) gameable — FAIL (need behavioral evidence)`);
|
|
2296
|
+
const codeCount = (report.gameable || 0) - outcomeCount;
|
|
2297
|
+
if (codeCount > 0) console.log(`B1: ${codeCount} CODE AC(s) gameable — WARN`);
|
|
2298
|
+
}
|
|
2299
|
+
} else {
|
|
2300
|
+
console.warn("WARN: B1 adversary report has no parseable JSON — treating as incomplete");
|
|
2301
|
+
}
|
|
2302
|
+
} catch (e: any) {
|
|
2303
|
+
console.warn(`WARN: B1 report read failed: ${e.message?.slice(0, 200)}`);
|
|
2304
|
+
}
|
|
2305
|
+
}
|
|
2306
|
+
return { fails: addedFails, warns: addedWarns };
|
|
2307
|
+
}
|
|
2308
|
+
|
|
2309
|
+
function runB2EvidenceValidation(state: Record<string, any>): void {
|
|
2310
|
+
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2311
|
+
if (tier === "LIGHT") return;
|
|
2312
|
+
|
|
2313
|
+
const scopeSha = state.gates?.scope?.commitSha;
|
|
2314
|
+
const fixSha = execSync("git rev-parse HEAD", { encoding: "utf-8", cwd: state.projectRoot || undefined }).trim();
|
|
2315
|
+
|
|
2316
|
+
if (!scopeSha || !state.projectRoot) return;
|
|
2317
|
+
|
|
2318
|
+
console.log("B2: Running evidence commands in clean worktree...");
|
|
2319
|
+
const tmpWorktree = join("/tmp", `pai-evidence-${Date.now()}`);
|
|
2320
|
+
|
|
2321
|
+
try {
|
|
2322
|
+
execSync(`git worktree add "${tmpWorktree}" ${fixSha} --detach`, { cwd: state.projectRoot, encoding: "utf-8" });
|
|
2323
|
+
|
|
2324
|
+
const verdicts: any[] = [];
|
|
2325
|
+
for (const ac of state.acs || []) {
|
|
2326
|
+
if (!ac.evidenceMethod?.command) continue;
|
|
2327
|
+
const cmd = ac.evidenceMethod.command;
|
|
2328
|
+
|
|
2329
|
+
let fixOutput = "";
|
|
2330
|
+
try {
|
|
2331
|
+
fixOutput = execSync(cmd, { encoding: "utf-8", timeout: 15000, cwd: tmpWorktree }).trim();
|
|
2332
|
+
} catch (e: any) {
|
|
2333
|
+
fixOutput = (e.stdout || "").trim() || `ERROR: ${e.message?.slice(0, 100)}`;
|
|
2334
|
+
}
|
|
2335
|
+
|
|
2336
|
+
let preFixOutput = "";
|
|
2337
|
+
const scopeWorktree = join("/tmp", `pai-metamorphic-${Date.now()}`);
|
|
2338
|
+
try {
|
|
2339
|
+
execSync(`git worktree add "${scopeWorktree}" ${scopeSha} --detach`, { cwd: state.projectRoot, encoding: "utf-8" });
|
|
2340
|
+
preFixOutput = execSync(cmd, { encoding: "utf-8", timeout: 15000, cwd: scopeWorktree }).trim();
|
|
2341
|
+
} catch {
|
|
2342
|
+
preFixOutput = "METAMORPHIC_SKIP";
|
|
2343
|
+
} finally {
|
|
2344
|
+
try { execSync(`git worktree remove "${scopeWorktree}" --force`, { cwd: state.projectRoot }); } catch {}
|
|
2345
|
+
}
|
|
2346
|
+
|
|
2347
|
+
const actual = parseFloat(fixOutput) || 0;
|
|
2348
|
+
const expected = parseFloat(ac.threshold?.value || "0");
|
|
2349
|
+
const op = ac.threshold?.op || ">=";
|
|
2350
|
+
let passes = false;
|
|
2351
|
+
if (op === ">=") passes = !isNaN(actual) && actual >= expected;
|
|
2352
|
+
else if (op === "==") passes = fixOutput === String(ac.threshold?.value) || (!isNaN(actual) && actual === expected);
|
|
2353
|
+
else if (op === "!=") passes = fixOutput !== String(ac.threshold?.value);
|
|
2354
|
+
else if (op === ">") passes = !isNaN(actual) && actual > expected;
|
|
2355
|
+
else if (op === "<=") passes = !isNaN(actual) && actual <= expected;
|
|
2356
|
+
else if (op === "<") passes = !isNaN(actual) && actual < expected;
|
|
2357
|
+
else if (op === "contains") passes = fixOutput.includes(String(ac.threshold?.value || ""));
|
|
2358
|
+
else if (op === "exists") passes = fixOutput.length > 0;
|
|
2359
|
+
|
|
2360
|
+
if (fixOutput === preFixOutput && preFixOutput !== "METAMORPHIC_SKIP") {
|
|
2361
|
+
console.warn(`WARN: metamorphic — ${ac.id} evidence identical pre/post fix (trivially satisfiable)`);
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2364
|
+
verdicts.push({
|
|
2365
|
+
acId: ac.id, command: cmd, rawOutput: fixOutput,
|
|
2366
|
+
threshold: ac.threshold, actual: fixOutput,
|
|
2367
|
+
verdict: passes ? "PASS" : "FAIL",
|
|
2368
|
+
metamorphic: fixOutput === preFixOutput ? "IDENTICAL" : "DIFFERENT",
|
|
2369
|
+
});
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
for (const v of verdicts) {
|
|
2373
|
+
const ac = (state.acs || []).find((a: any) => a.id === v.acId);
|
|
2374
|
+
if (ac) {
|
|
2375
|
+
ac.verdict = v.verdict;
|
|
2376
|
+
ac.evidence = { type: "command-output", content: v.rawOutput };
|
|
2377
|
+
}
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
console.log(`B2: ${verdicts.filter(v => v.verdict === "PASS").length}/${verdicts.length} evidence commands PASS`);
|
|
2381
|
+
} catch (e: any) {
|
|
2382
|
+
console.warn(`WARN: B2 worktree setup failed: ${e.message?.slice(0, 200)}`);
|
|
2383
|
+
} finally {
|
|
2384
|
+
try { execSync(`git worktree remove "${tmpWorktree}" --force`, { cwd: state.projectRoot }); } catch {}
|
|
2385
|
+
}
|
|
2386
|
+
}
|
|
2387
|
+
|
|
2388
|
+
/**
|
|
2389
|
+
* Why the Verify fan-out is not going to run this time, or null if it is (#126).
|
|
2390
|
+
*
|
|
2391
|
+
* One function, one body, so the branches can be driven directly and the call
|
|
2392
|
+
* site can be mutated — `test/verify-fanout-recording.test.ts` does both. The
|
|
2393
|
+
* three conditions were previously inline and repeated on two `if`s with no
|
|
2394
|
+
* `else`, which is how the fan-out came to be skippable without a trace.
|
|
2395
|
+
*
|
|
2396
|
+
* Order is deliberate and is the reporting order, not a precedence accident:
|
|
2397
|
+
* a gate that already has failures is the reason a reader needs first, because
|
|
2398
|
+
* it is the one that explains the run. `RUNGATE_SKIP_AGENTS` is last because a
|
|
2399
|
+
* run with that set has usually also failed something, and reporting the env
|
|
2400
|
+
* var would bury the real cause.
|
|
2401
|
+
*
|
|
2402
|
+
* Returning null means RUN — the only path on which the agents are spawned, so
|
|
2403
|
+
* a new skip condition that forgets to come through here is a silent skip
|
|
2404
|
+
* again, and that is what the call-site mutation checks.
|
|
2405
|
+
*/
|
|
2406
|
+
export function fanoutSkipReason(
|
|
2407
|
+
fails: number,
|
|
2408
|
+
testExitCode: number,
|
|
2409
|
+
skipAgents: unknown,
|
|
2410
|
+
): string | null {
|
|
2411
|
+
if (fails !== 0) return `verify gate had ${fails} failing check(s) before the fan-out`;
|
|
2412
|
+
if (testExitCode !== 0) return `the project test command exited ${testExitCode}`;
|
|
2413
|
+
if (skipAgents !== undefined && skipAgents !== null && skipAgents !== "") {
|
|
2414
|
+
return "RUNGATE_SKIP_AGENTS is set";
|
|
2415
|
+
}
|
|
2416
|
+
return null;
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
function runB2EvidenceValidatorAgent(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
|
|
2420
|
+
let addedWarns = 0;
|
|
2421
|
+
let agentResult: any = null;
|
|
2422
|
+
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2423
|
+
// Recorded rather than returned empty (#126): the consumer allows a LIGHT
|
|
2424
|
+
// absence, but an artefact that SAYS why is one a reader does not have to
|
|
2425
|
+
// reconstruct the tier rules to interpret.
|
|
2426
|
+
if (tier === "LIGHT") return { agentResult: { skipped: "LIGHT ceremony tier" }, warns: 0 };
|
|
2427
|
+
|
|
2428
|
+
const b2ProjectPrompt = join(__dirname, "prompts", "evidence-validator.md");
|
|
2429
|
+
const b2HomePrompt = join(process.env.HOME || "", ".claude", "gates", "prompts", "evidence-validator.md");
|
|
2430
|
+
const b2PromptPath = existsSync(b2ProjectPrompt) ? b2ProjectPrompt : b2HomePrompt;
|
|
2431
|
+
if (!existsSync(b2PromptPath)) {
|
|
2432
|
+
console.warn("WARN: evidence-validator.md prompt not found — B2 agent will not run");
|
|
2433
|
+
// Recorded, not just warned (#126). A missing prompt disabled evidence
|
|
2434
|
+
// validation entirely, silently, and the gate check read the resulting
|
|
2435
|
+
// absence as clean. The run now carries the reason it did not happen.
|
|
2436
|
+
return { agentResult: { skipped: `evidence-validator.md prompt not found at ${b2PromptPath}` }, warns: 0 };
|
|
2437
|
+
}
|
|
2438
|
+
|
|
2439
|
+
console.log("B2: Spawning Evidence Validator agent...");
|
|
2440
|
+
const b2Input = JSON.stringify({
|
|
2441
|
+
acs: (state.acs || []).map((ac: any) => ({
|
|
2442
|
+
id: ac.id, type: ac.type, statement: ac.statement,
|
|
2443
|
+
threshold: ac.threshold, evidence: ac.evidence, verdict: ac.verdict,
|
|
2444
|
+
evidenceMethod: ac.evidenceMethod,
|
|
2445
|
+
})),
|
|
2446
|
+
}, null, 2);
|
|
2447
|
+
const b2InputPath = join(workDir, "b2-input.json");
|
|
2448
|
+
writeFileSync(b2InputPath, b2Input);
|
|
2449
|
+
|
|
2450
|
+
let b2Output = "";
|
|
2451
|
+
try {
|
|
2452
|
+
b2Output = execSync(
|
|
2453
|
+
`cat '${b2InputPath}' | claude -p - --model haiku --system-prompt-file "${b2PromptPath}" --output-format text --allowedTools "" --max-turns 3`,
|
|
2454
|
+
{ encoding: "utf-8", timeout: 60000, cwd: harnessRoot },
|
|
2455
|
+
);
|
|
2456
|
+
} catch (e: any) {
|
|
2457
|
+
b2Output = e.stdout || "";
|
|
2458
|
+
console.warn(`WARN: B2 Evidence Validator exited with error: ${e.message?.slice(0, 200)}`);
|
|
2459
|
+
}
|
|
2460
|
+
|
|
2461
|
+
const b2Json = b2Output.match(/\{[\s\S]*"verdicts"[\s\S]*\}/);
|
|
2462
|
+
if (b2Json) {
|
|
2463
|
+
try {
|
|
2464
|
+
const b2Report = JSON.parse(b2Json[0]);
|
|
2465
|
+
agentResult = {
|
|
2466
|
+
ts: new Date().toISOString(),
|
|
2467
|
+
verdicts: b2Report.verdicts || [],
|
|
2468
|
+
source: "B2-evidence-validator-agent",
|
|
2469
|
+
};
|
|
2470
|
+
const b2Fails = (b2Report.verdicts || []).filter((v: any) => v.verdict === "FAIL");
|
|
2471
|
+
if (b2Fails.length > 0) {
|
|
2472
|
+
for (const v of b2Fails) {
|
|
2473
|
+
addedWarns++;
|
|
2474
|
+
results.push({ check: `B2-validator: ${v.acId} evidence gap`, result: "WARN" as const, detail: v.rawOutput || "threshold not met" });
|
|
2475
|
+
}
|
|
2476
|
+
console.log(`B2: ${b2Fails.length} evidence gap(s) found — WARN`);
|
|
2477
|
+
} else {
|
|
2478
|
+
console.log(`B2: All evidence validated — ${(b2Report.verdicts || []).length} ACs checked`);
|
|
2479
|
+
}
|
|
2480
|
+
} catch {
|
|
2481
|
+
console.warn("WARN: B2 Evidence Validator JSON parse failed");
|
|
2482
|
+
}
|
|
2483
|
+
} else {
|
|
2484
|
+
console.warn("WARN: B2 Evidence Validator produced no parseable JSON");
|
|
2485
|
+
}
|
|
2486
|
+
|
|
2487
|
+
return { agentResult, warns: addedWarns };
|
|
2488
|
+
}
|
|
2489
|
+
|
|
2490
|
+
function runB1AdversaryAtVerify(state: Record<string, any>, workDir: string, results: GateResult[], harnessRoot: string): { agentResult: any; warns: number } {
|
|
2491
|
+
let addedWarns = 0;
|
|
2492
|
+
let agentResult: any = null;
|
|
2493
|
+
const tier = state.sizing?.ceremonyTier || "STANDARD";
|
|
2494
|
+
if (tier === "LIGHT") return { agentResult: { skipped: "LIGHT ceremony tier" }, warns: 0 };
|
|
2495
|
+
|
|
2496
|
+
const b1ProjectPrompt = join(__dirname, "prompts", "ac-adversary.md");
|
|
2497
|
+
const b1HomePrompt = join(process.env.HOME || "", ".claude", "gates", "prompts", "ac-adversary.md");
|
|
2498
|
+
const b1PromptPath = existsSync(b1ProjectPrompt) ? b1ProjectPrompt : b1HomePrompt;
|
|
2499
|
+
if (!existsSync(b1PromptPath)) {
|
|
2500
|
+
console.warn("WARN: ac-adversary.md prompt not found — B1 verify agent will not run");
|
|
2501
|
+
// Same as B2 above (#126): a missing prompt turned adversarial
|
|
2502
|
+
// verification off without a trace anyone downstream could read.
|
|
2503
|
+
return { agentResult: { skipped: `ac-adversary.md prompt not found at ${b1PromptPath}` }, warns: 0 };
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2506
|
+
console.log("B1: Spawning AC Adversary agent at verify...");
|
|
2507
|
+
const b1Input = JSON.stringify({
|
|
2508
|
+
acs: (state.acs || []).map((ac: any) => ({
|
|
2509
|
+
id: ac.id, type: ac.type, statement: ac.statement,
|
|
2510
|
+
threshold: ac.threshold, evidence: ac.evidence, verdict: ac.verdict,
|
|
2511
|
+
evidenceMethod: ac.evidenceMethod,
|
|
2512
|
+
})),
|
|
2513
|
+
}, null, 2);
|
|
2514
|
+
const b1InputPath = join(workDir, "b1-verify-input.json");
|
|
2515
|
+
writeFileSync(b1InputPath, b1Input);
|
|
2516
|
+
|
|
2517
|
+
let b1Output = "";
|
|
2518
|
+
try {
|
|
2519
|
+
b1Output = execSync(
|
|
2520
|
+
`cat '${b1InputPath}' | claude -p - --model haiku --system-prompt-file "${b1PromptPath}" --output-format text --allowedTools "" --max-turns 3`,
|
|
2521
|
+
{ encoding: "utf-8", timeout: 60000, cwd: harnessRoot },
|
|
2522
|
+
);
|
|
2523
|
+
} catch (e: any) {
|
|
2524
|
+
b1Output = e.stdout || "";
|
|
2525
|
+
console.warn(`WARN: B1 AC Adversary exited with error: ${e.message?.slice(0, 200)}`);
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2528
|
+
const b1Json = b1Output.match(/\{[\s\S]*"gameable"[\s\S]*\}/);
|
|
2529
|
+
if (b1Json) {
|
|
2530
|
+
try {
|
|
2531
|
+
const b1Report = JSON.parse(b1Json[0]);
|
|
2532
|
+
agentResult = {
|
|
2533
|
+
ts: new Date().toISOString(),
|
|
2534
|
+
gameable: b1Report.gameable || 0,
|
|
2535
|
+
approved: b1Report.approved ?? true,
|
|
2536
|
+
exploits: b1Report.exploits || [],
|
|
2537
|
+
source: "B1-adversary-verify-agent",
|
|
2538
|
+
};
|
|
2539
|
+
if (b1Report.gameable > 0) {
|
|
2540
|
+
for (const exploit of b1Report.exploits || []) {
|
|
2541
|
+
addedWarns++;
|
|
2542
|
+
results.push({ check: `B1-adversary-verify: ${exploit.acId} gameable`, result: "WARN" as const, detail: exploit.exploit });
|
|
2543
|
+
}
|
|
2544
|
+
console.log(`B1: ${b1Report.gameable} gameable AC(s) found at verify — WARN`);
|
|
2545
|
+
} else {
|
|
2546
|
+
console.log(`B1: All ACs approved at verify — no exploits found`);
|
|
2547
|
+
}
|
|
2548
|
+
} catch {
|
|
2549
|
+
console.warn("WARN: B1 AC Adversary JSON parse failed");
|
|
2550
|
+
}
|
|
2551
|
+
} else {
|
|
2552
|
+
console.warn("WARN: B1 AC Adversary produced no parseable JSON");
|
|
2553
|
+
}
|
|
2554
|
+
|
|
2555
|
+
return { agentResult, warns: addedWarns };
|
|
2556
|
+
}
|
|
2557
|
+
|
|
2558
|
+
// ── Ship post-processing ────────────────────────────────────────────────
|
|
2559
|
+
|
|
2560
|
+
function runShipPreflights(state: Record<string, any>): void {
|
|
2561
|
+
releaseLock(state.slug);
|
|
2562
|
+
if (state.mergeQueue) {
|
|
2563
|
+
const ordered = sequentialMergeOrder(state.mergeQueue);
|
|
2564
|
+
const next = ordered[0];
|
|
2565
|
+
if (next && shouldRunCIVerification(next)) {
|
|
2566
|
+
console.log(`Next in merge queue: ${next.slug} (CI verification required)`);
|
|
2567
|
+
}
|
|
2568
|
+
}
|
|
2569
|
+
}
|
|
2570
|
+
|
|
2571
|
+
// ── Merge gate ──────────────────────────────────────────────────────────
|
|
2572
|
+
|
|
2573
|
+
function runMergeGate(state: Record<string, any>, sf: string): never {
|
|
2574
|
+
const marcusBranch = state.agents?.marcus?.branch;
|
|
2575
|
+
const scopeSha = state.gates?.scope?.commitSha;
|
|
2576
|
+
|
|
2577
|
+
let diffBase: string;
|
|
2578
|
+
if (marcusBranch && marcusBranch !== "main") {
|
|
2579
|
+
diffBase = `main...${marcusBranch}`;
|
|
2580
|
+
} else if (scopeSha) {
|
|
2581
|
+
diffBase = `${scopeSha}...HEAD`;
|
|
2582
|
+
} else {
|
|
2583
|
+
console.log("merge gate: no branch or scope SHA — skipping diff checks");
|
|
2584
|
+
const { resultVal, attempt } = writeGateResult(sf, "merge", 1, 0, 0, [], state.projectRoot);
|
|
2585
|
+
console.log(`Gate merge: ${resultVal} (attempt ${attempt})`);
|
|
2586
|
+
process.exit(0);
|
|
2587
|
+
}
|
|
2588
|
+
|
|
2589
|
+
let mergePass = 0;
|
|
2590
|
+
let mergeFail = 0;
|
|
2591
|
+
const mergeResults: any[] = [];
|
|
2592
|
+
|
|
2593
|
+
const deletedResult = execSync(`git diff ${diffBase} --diff-filter=D --name-only`, { encoding: "utf-8", cwd: state.projectRoot || undefined }).trim();
|
|
2594
|
+
const deletedTests = deletedResult.split("\n").filter(f => /\.test\.(ts|tsx)$/.test(f));
|
|
2595
|
+
if (deletedTests.length > 0) {
|
|
2596
|
+
mergeFail++;
|
|
2597
|
+
mergeResults.push({ check: "merge: no test file deletions", result: "FAIL", detail: deletedTests.join(", ") });
|
|
2598
|
+
} else {
|
|
2599
|
+
mergePass++;
|
|
2600
|
+
mergeResults.push({ check: "merge: no test file deletions", result: "PASS" });
|
|
2601
|
+
}
|
|
2602
|
+
|
|
2603
|
+
const changedResult = execSync(`git diff ${diffBase} --name-only`, { encoding: "utf-8", cwd: state.projectRoot || undefined }).trim();
|
|
2604
|
+
const gateHookChanges = changedResult.split("\n").filter(f => /^(gates|hooks)\//.test(f));
|
|
2605
|
+
const issueTargetsGates = (state.issueGoal || "").toLowerCase().includes("gate");
|
|
2606
|
+
if (gateHookChanges.length > 0 && !issueTargetsGates) {
|
|
2607
|
+
mergeResults.push({ check: "merge: gate/hook modifications", result: "WARN", detail: gateHookChanges.join(", ") });
|
|
2608
|
+
} else {
|
|
2609
|
+
mergePass++;
|
|
2610
|
+
mergeResults.push({ check: "merge: gate/hook modifications", result: "PASS" });
|
|
2611
|
+
}
|
|
2612
|
+
|
|
2613
|
+
const statResult = execSync(`git diff ${diffBase} --shortstat`, { encoding: "utf-8", cwd: state.projectRoot || undefined }).trim();
|
|
2614
|
+
const insertions = parseInt((statResult.match(/(\d+) insertion/) || ["0","0"])[1]);
|
|
2615
|
+
const size = state.sizing?.predicted || "M";
|
|
2616
|
+
if ((size === "XS" && insertions > 200) || (size === "S" && insertions > 500)) {
|
|
2617
|
+
mergeResults.push({ check: "merge: scope proportionality", result: "WARN", detail: `${insertions} insertions for ${size} issue` });
|
|
2618
|
+
} else {
|
|
2619
|
+
mergePass++;
|
|
2620
|
+
mergeResults.push({ check: "merge: scope proportionality", result: "PASS" });
|
|
2621
|
+
}
|
|
2622
|
+
|
|
2623
|
+
const { resultVal: mergeResultVal, attempt: mergeAttempt } = writeGateResult(sf, "merge", mergePass, mergeFail, 0, mergeResults, state.projectRoot);
|
|
2624
|
+
console.log(`Gate merge: ${mergeResultVal} (attempt ${mergeAttempt})`);
|
|
2625
|
+
process.exit(mergeResultVal === "PASS" ? 0 : 1);
|
|
2626
|
+
}
|
|
2627
|
+
|
|
2628
|
+
// ── Prove gate post-processing ──────────────────────────────────────────
|
|
2629
|
+
|
|
2630
|
+
async function runProvePostProcessing(state: Record<string, any>, workDir: string, issue: number, issueRepo: string, resultVal: string): Promise<never> {
|
|
2631
|
+
const proveEvidencePath = join(workDir, "prove-evidence.json");
|
|
2632
|
+
if (existsSync(proveEvidencePath)) {
|
|
2633
|
+
try {
|
|
2634
|
+
const proveEvidence = JSON.parse(readFileSync(proveEvidencePath, "utf-8"));
|
|
2635
|
+
if (proveEvidence.verdict === "PROVEN" && issue && issueRepo) {
|
|
2636
|
+
try {
|
|
2637
|
+
// Use Octokit addLabels (POST additive — D-5) instead of gh CLI
|
|
2638
|
+
const github = createGitHubClient();
|
|
2639
|
+
await addLabels(github, issueRepo, issue, ["proven"]);
|
|
2640
|
+
console.log(`prove gate: added "proven" label to #${issue}`);
|
|
2641
|
+
} catch (e: any) {
|
|
2642
|
+
console.warn(`WARN: prove gate — failed to add proven label: ${e.message?.slice(0, 200)}`);
|
|
2643
|
+
}
|
|
2644
|
+
}
|
|
2645
|
+
} catch {
|
|
2646
|
+
// prove-evidence.json parse failure — already warned earlier
|
|
2647
|
+
}
|
|
2648
|
+
}
|
|
2649
|
+
process.exit(resultVal === "PASS" ? 0 : 1);
|
|
2650
|
+
throw new Error("unreachable");
|
|
2651
|
+
}
|
|
2652
|
+
|
|
2653
|
+
// ── Ceremony override support ───────────────────────────────────────────
|
|
2654
|
+
|
|
2655
|
+
const PROTECTED_CHECKS = [
|
|
2656
|
+
"tests-pass",
|
|
2657
|
+
"tsc-pass",
|
|
2658
|
+
"code-committed",
|
|
2659
|
+
"all-acs-have-evidence",
|
|
2660
|
+
"all-acs-pass",
|
|
2661
|
+
"branch-merged",
|
|
2662
|
+
"code-pushed",
|
|
2663
|
+
];
|
|
2664
|
+
|
|
2665
|
+
/**
|
|
2666
|
+
* Apply ceremony overrides from rungate.json and ensure protected checks are present.
|
|
2667
|
+
* Returns path to the modified ceremony profile (written to temp location).
|
|
2668
|
+
*/
|
|
2669
|
+
export function applyCeremonyOverrides(
|
|
2670
|
+
baseCeremonyPath: string,
|
|
2671
|
+
harnessConfig: ProjectHarness | null,
|
|
2672
|
+
projectRoot: string
|
|
2673
|
+
): string {
|
|
2674
|
+
if (!existsSync(baseCeremonyPath)) {
|
|
2675
|
+
return baseCeremonyPath;
|
|
2676
|
+
}
|
|
2677
|
+
|
|
2678
|
+
const baseProfile = JSON.parse(readFileSync(baseCeremonyPath, "utf-8"));
|
|
2679
|
+
|
|
2680
|
+
// No overrides? Use base profile as-is
|
|
2681
|
+
if (!harnessConfig?.ceremonyOverrides) {
|
|
2682
|
+
return baseCeremonyPath;
|
|
2683
|
+
}
|
|
2684
|
+
|
|
2685
|
+
// Deep merge overrides into base tiers
|
|
2686
|
+
const mergedProfile = { ...baseProfile };
|
|
2687
|
+
mergedProfile.tiers = deepMerge(baseProfile.tiers || {}, harnessConfig.ceremonyOverrides);
|
|
2688
|
+
|
|
2689
|
+
// Validate protected checks are present in all tiers
|
|
2690
|
+
for (const [tierName, tierConfig] of Object.entries(mergedProfile.tiers)) {
|
|
2691
|
+
const checks = (tierConfig as any)?.checks;
|
|
2692
|
+
if (!checks || typeof checks !== "object") continue;
|
|
2693
|
+
|
|
2694
|
+
for (const [gateName, checkList] of Object.entries(checks)) {
|
|
2695
|
+
if (!Array.isArray(checkList)) continue;
|
|
2696
|
+
|
|
2697
|
+
// Find which protected checks apply to this gate
|
|
2698
|
+
const gateProtectedChecks = PROTECTED_CHECKS.filter(pc => {
|
|
2699
|
+
// tests-pass, tsc-pass, code-committed, all-acs-* belong to verify gate
|
|
2700
|
+
if (gateName === "verify" && ["tests-pass", "tsc-pass", "code-committed", "all-acs-have-evidence", "all-acs-pass"].includes(pc)) {
|
|
2701
|
+
return true;
|
|
2702
|
+
}
|
|
2703
|
+
// branch-merged, code-pushed belong to ship gate
|
|
2704
|
+
if (gateName === "ship" && ["branch-merged", "code-pushed"].includes(pc)) {
|
|
2705
|
+
return true;
|
|
2706
|
+
}
|
|
2707
|
+
return false;
|
|
2708
|
+
});
|
|
2709
|
+
|
|
2710
|
+
// Re-add any missing protected checks
|
|
2711
|
+
const missing = gateProtectedChecks.filter(pc => !checkList.includes(pc));
|
|
2712
|
+
if (missing.length > 0) {
|
|
2713
|
+
console.warn(`WARN: Protected checks re-added to ${tierName}.${gateName}: ${missing.join(", ")}`);
|
|
2714
|
+
(checks as any)[gateName] = [...checkList, ...missing];
|
|
2715
|
+
}
|
|
2716
|
+
}
|
|
2717
|
+
}
|
|
2718
|
+
|
|
2719
|
+
// Write merged profile to temp location
|
|
2720
|
+
const tempDir = join(projectRoot, ".rungate");
|
|
2721
|
+
if (!existsSync(tempDir)) {
|
|
2722
|
+
mkdirSync(tempDir, { recursive: true });
|
|
2723
|
+
}
|
|
2724
|
+
const tempProfilePath = join(tempDir, "ceremony-profile-merged.json");
|
|
2725
|
+
writeFileSync(tempProfilePath, JSON.stringify(mergedProfile, null, 2));
|
|
2726
|
+
|
|
2727
|
+
return tempProfilePath;
|
|
2728
|
+
}
|
|
2729
|
+
|
|
2730
|
+
// ── Main executor ───────────────────────────────────────────────────────
|
|
2731
|
+
|
|
2732
|
+
export async function executeGate(input: GateExecutorInput): Promise<GateExecutorResult> {
|
|
2733
|
+
const { gate, slug, issue, workDir, stateFilePath: sf, harnessRoot } = input;
|
|
2734
|
+
|
|
2735
|
+
const baseCeremonyProfile = existsSync(join(__dirname, "ceremony-profiles.json"))
|
|
2736
|
+
? join(__dirname, "ceremony-profiles.json")
|
|
2737
|
+
: join(process.env.HOME || "", ".claude", "skills", "ship", "ceremony-profiles.json");
|
|
2738
|
+
|
|
2739
|
+
const state = JSON.parse(readFileSync(sf, "utf-8"));
|
|
2740
|
+
const issueRepo = state.issueRepo || state.repo || "";
|
|
2741
|
+
|
|
2742
|
+
// Validate rungate.json (SC-6)
|
|
2743
|
+
const harnessConfig = loadAndValidateHarness(state.projectRoot || "");
|
|
2744
|
+
|
|
2745
|
+
// Apply ceremony overrides from rungate.json
|
|
2746
|
+
const CEREMONY_PROFILE = applyCeremonyOverrides(
|
|
2747
|
+
baseCeremonyProfile,
|
|
2748
|
+
harnessConfig,
|
|
2749
|
+
state.projectRoot || process.cwd()
|
|
2750
|
+
);
|
|
2751
|
+
|
|
2752
|
+
// Auto-populate AC verdicts
|
|
2753
|
+
autoPopulateACs(state, sf);
|
|
2754
|
+
|
|
2755
|
+
// Gaps gate: WARN-only mechanical drift detection
|
|
2756
|
+
if (gate === "gaps") {
|
|
2757
|
+
const projectRoot = state.projectRoot || process.cwd();
|
|
2758
|
+
const gapResults = scanGaps(projectRoot, workDir);
|
|
2759
|
+
|
|
2760
|
+
console.log("\n── GAP SCANNER ──");
|
|
2761
|
+
console.log(`Drift checks: ${gapResults.warnings} WARN, ${gapResults.passes} PASS\n`);
|
|
2762
|
+
|
|
2763
|
+
const results: GateResult[] = [];
|
|
2764
|
+
for (const gapResult of gapResults.results) {
|
|
2765
|
+
// Map SKIP to PASS for GateResult (which only supports PASS|FAIL|WARN)
|
|
2766
|
+
const mappedStatus = gapResult.status === "SKIP" ? "PASS" : gapResult.status;
|
|
2767
|
+
results.push({
|
|
2768
|
+
check: gapResult.check,
|
|
2769
|
+
result: mappedStatus,
|
|
2770
|
+
detail: gapResult.detail,
|
|
2771
|
+
});
|
|
2772
|
+
console.log(`${gapResult.status}: ${gapResult.check} — ${gapResult.detail}`);
|
|
2773
|
+
}
|
|
2774
|
+
|
|
2775
|
+
// Gaps gate is WARN-only: never fails, never blocks ship
|
|
2776
|
+
const { resultVal, attempt } = writeGateResult(sf, "gaps", gapResults.passes, 0, gapResults.warnings, results, projectRoot);
|
|
2777
|
+
console.log(`\nGate gaps: ${resultVal} (attempt ${attempt}) — informational only, does not block ship`);
|
|
2778
|
+
|
|
2779
|
+
return {
|
|
2780
|
+
resultVal: "PASS" as const,
|
|
2781
|
+
passes: gapResults.passes,
|
|
2782
|
+
fails: 0,
|
|
2783
|
+
warns: gapResults.warnings,
|
|
2784
|
+
results,
|
|
2785
|
+
attempt,
|
|
2786
|
+
exitCode: 0,
|
|
2787
|
+
};
|
|
2788
|
+
}
|
|
2789
|
+
|
|
2790
|
+
// Early results accumulator
|
|
2791
|
+
let earlyResults: GateResult[] = [];
|
|
2792
|
+
let earlyFails = 0;
|
|
2793
|
+
|
|
2794
|
+
// Verify gate: check for PENDING ACs
|
|
2795
|
+
if (gate === "verify") {
|
|
2796
|
+
const pending = checkVerifyPendingACs(state);
|
|
2797
|
+
earlyResults = pending.results;
|
|
2798
|
+
earlyFails = pending.failCount;
|
|
2799
|
+
}
|
|
2800
|
+
|
|
2801
|
+
// B3: Prove Reproducer
|
|
2802
|
+
if (gate === "prove") {
|
|
2803
|
+
await runProveReproducer(state, issue, issueRepo, workDir, harnessRoot, sf);
|
|
2804
|
+
}
|
|
2805
|
+
|
|
2806
|
+
// Scope pre-flight checks — awaited, and persisted before the gate tests
|
|
2807
|
+
// read the file (#235)
|
|
2808
|
+
if (gate === "scope") {
|
|
2809
|
+
await runScopePreflightsAndPersist(state, sf, harnessRoot);
|
|
2810
|
+
}
|
|
2811
|
+
|
|
2812
|
+
// The reading the verify gate is about to check for, if nothing took it
|
|
2813
|
+
// (#261). Before the gate suite is spawned, which is the whole ordering
|
|
2814
|
+
// constraint — see ensurePrevalidationPersisted.
|
|
2815
|
+
await prevalidationForGate(gate, state, sf);
|
|
2816
|
+
|
|
2817
|
+
// Type check (scope or verify)
|
|
2818
|
+
if (gate === "scope" || gate === "verify") {
|
|
2819
|
+
runTypeCheck(state);
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2822
|
+
// Verify pre-flight checks
|
|
2823
|
+
if (gate === "verify") {
|
|
2824
|
+
runVerifyPreflights(state);
|
|
2825
|
+
}
|
|
2826
|
+
|
|
2827
|
+
// Ship pre-flight checks
|
|
2828
|
+
if (gate === "ship") {
|
|
2829
|
+
runShipPreflights(state);
|
|
2830
|
+
}
|
|
2831
|
+
|
|
2832
|
+
// Set phase for gate tests
|
|
2833
|
+
const GATE_PHASE_MAP: Record<string, string> = { scope: "SCOPE", verify: "VERIFY", ship: "SHIP", prove: "DONE" };
|
|
2834
|
+
const gatePhase = GATE_PHASE_MAP[gate];
|
|
2835
|
+
if (gatePhase && state.phase !== gatePhase) {
|
|
2836
|
+
const priorPhase = state.phase;
|
|
2837
|
+
state.phase = gatePhase;
|
|
2838
|
+
writeFileSync(sf, JSON.stringify(state, null, 2));
|
|
2839
|
+
console.log(`Phase set to ${gatePhase} for gate tests (was ${priorPhase})`);
|
|
2840
|
+
}
|
|
2841
|
+
|
|
2842
|
+
// Run tests
|
|
2843
|
+
const { testOutput, testExitCode, junitPath } = runGateTests(gate, workDir, state, harnessRoot);
|
|
2844
|
+
|
|
2845
|
+
// Parse results
|
|
2846
|
+
let { passes, fails, warns, results } = parseGateResults(testOutput, earlyResults, earlyFails);
|
|
2847
|
+
|
|
2848
|
+
// Subtract test baseline at verify gate (#599) — pre-existing failures don't block
|
|
2849
|
+
if (gate === "verify") {
|
|
2850
|
+
const adjustment = baselineAdjustment(state.gateContract?.baselines, fails);
|
|
2851
|
+
if (adjustment.kind === "subtract") {
|
|
2852
|
+
console.log(`Test baseline subtraction: ${fails} total - ${adjustment.baseline} baseline = ${adjustment.fails} new failure(s)`);
|
|
2853
|
+
fails = adjustment.fails;
|
|
2854
|
+
} else if (adjustment.kind === "unmeasured") {
|
|
2855
|
+
// #307. WARN, not FAIL: the house precedent is
|
|
2856
|
+
// `dev-server-liveness: UNMEASURED` above, and a probe that could not
|
|
2857
|
+
// run must never become a defect in the code it could not measure.
|
|
2858
|
+
warns++;
|
|
2859
|
+
results.push({
|
|
2860
|
+
check: "test-baseline: UNMEASURED",
|
|
2861
|
+
result: "WARN" as const,
|
|
2862
|
+
detail:
|
|
2863
|
+
`no pre-existing-failure count was taken at scope, so every failure below ` +
|
|
2864
|
+
`is attributed to this change — ${adjustment.reason}`,
|
|
2865
|
+
});
|
|
2866
|
+
console.warn(`Test baseline: UNMEASURED — ${adjustment.reason}`);
|
|
2867
|
+
}
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
// Did a heal round delete a test? (#307 SC-A19)
|
|
2871
|
+
//
|
|
2872
|
+
// Runs on every gate, not just verify: a heal round ends in a gate re-run
|
|
2873
|
+
// whichever gate it was healing, and the scope gate heals too.
|
|
2874
|
+
//
|
|
2875
|
+
// AFTER the baseline subtraction above, deliberately. This was written
|
|
2876
|
+
// before it and a security review caught the fail-open: a census FAIL
|
|
2877
|
+
// incremented into `fails` is indistinguishable from a suite failure to
|
|
2878
|
+
// `subtractTestBaseline`, so a run carrying three pre-existing failures
|
|
2879
|
+
// would have had the deleted-test refusal subtracted to zero. A baseline
|
|
2880
|
+
// is a statement about the SUITE and can say nothing about whether a heal
|
|
2881
|
+
// round removed a test.
|
|
2882
|
+
//
|
|
2883
|
+
// The census reads `treeUnderTest`, not `projectRoot`. On a decomposed run
|
|
2884
|
+
// the agent's edits are in a worktree, and a census of the main checkout
|
|
2885
|
+
// would compare a tree nobody touched against itself — green every round,
|
|
2886
|
+
// for the one kind of run this guard most needs to watch.
|
|
2887
|
+
{
|
|
2888
|
+
const censusRoot = state.treeUnderTest || state.projectRoot || process.cwd();
|
|
2889
|
+
const censusSkipped: string[] = [];
|
|
2890
|
+
const current = censusOf(censusRoot, censusSkipped);
|
|
2891
|
+
if (censusSkipped.length > 0) {
|
|
2892
|
+
// Reported every run. "No test was removed" is a sentence about the
|
|
2893
|
+
// files that were read, and a reader is owed the list it is not about.
|
|
2894
|
+
console.warn(
|
|
2895
|
+
`Test census: ${censusSkipped.length} file(s) too large to count and excluded from the ` +
|
|
2896
|
+
`comparison: ${censusSkipped.join(", ")}`,
|
|
2897
|
+
);
|
|
2898
|
+
}
|
|
2899
|
+
// The PREVIOUS census comes off disk, not off the in-memory `state`.
|
|
2900
|
+
// `state` was loaded when this process started; the gate tests and any
|
|
2901
|
+
// heal agent have been writing the file since. Comparing against the
|
|
2902
|
+
// stale copy and then persisting over the fresh one is how a round's
|
|
2903
|
+
// reading quietly disappears — and the direction it disappears in is
|
|
2904
|
+
// toward not refusing.
|
|
2905
|
+
const previous = readPersistedCensus(sf, slug);
|
|
2906
|
+
// Whether a gate already ran for this slug comes from the witness store
|
|
2907
|
+
// and not from the state file (#494). An agent erasing the census
|
|
2908
|
+
// erases the signature with it, so the state file cannot be asked
|
|
2909
|
+
// whether there was ever anything to erase.
|
|
2910
|
+
const prior = priorGateRun(slug);
|
|
2911
|
+
const verdict = censusVerdict(previous, current, prior);
|
|
2912
|
+
const banked = nextCensus(verifiedCensus(previous), current, verdict);
|
|
2913
|
+
state.gateContract = state.gateContract || {};
|
|
2914
|
+
state.gateContract.testCensus = banked;
|
|
2915
|
+
// To DISK, not just to `state` — `writeGateResult` below re-reads the
|
|
2916
|
+
// file, so an in-memory mutation here never lands and the next run
|
|
2917
|
+
// would find no census to compare against. See `persistTestCensus`.
|
|
2918
|
+
persistTestCensus(sf, banked, slug);
|
|
2919
|
+
|
|
2920
|
+
const report = censusReport(verdict, censusRoot);
|
|
2921
|
+
fails += report.fails;
|
|
2922
|
+
warns += report.warns;
|
|
2923
|
+
results.push(...report.results);
|
|
2924
|
+
for (const line of report.console) console[line.level](line.line);
|
|
2925
|
+
}
|
|
2926
|
+
|
|
2927
|
+
console.log(`\nTest results: ${passes} pass, ${fails} fail, ${warns} warn`);
|
|
2928
|
+
|
|
2929
|
+
// Conformity findings
|
|
2930
|
+
readConformityFindings(state);
|
|
2931
|
+
|
|
2932
|
+
// Batch diagnosis
|
|
2933
|
+
printBatchDiagnosis(fails, results);
|
|
2934
|
+
|
|
2935
|
+
// B1/B2 agent logic
|
|
2936
|
+
let b2AgentResult: any = null;
|
|
2937
|
+
let b1VerifyAgentResult: any = null;
|
|
2938
|
+
|
|
2939
|
+
if (gate === "scope" && fails === 0 && testExitCode === 0 && !process.env.RUNGATE_SKIP_AGENTS) {
|
|
2940
|
+
spawnB1AdversaryAtScope(state, workDir, harnessRoot);
|
|
2941
|
+
}
|
|
2942
|
+
|
|
2943
|
+
if (gate === "verify" && fails === 0 && testExitCode === 0) {
|
|
2944
|
+
fails += checkDevServerLiveness(state, results);
|
|
2945
|
+
}
|
|
2946
|
+
|
|
2947
|
+
if (gate === "verify" && fails === 0 && testExitCode === 0 && !process.env.RUNGATE_SKIP_AGENTS) {
|
|
2948
|
+
const b1Report = checkB1ReportAtVerify(state, workDir, results);
|
|
2949
|
+
fails += b1Report.fails;
|
|
2950
|
+
warns += b1Report.warns;
|
|
2951
|
+
}
|
|
2952
|
+
|
|
2953
|
+
if (gate === "verify" && fails === 0 && testExitCode === 0 && !process.env.RUNGATE_SKIP_AGENTS) {
|
|
2954
|
+
runB2EvidenceValidation(state);
|
|
2955
|
+
}
|
|
2956
|
+
|
|
2957
|
+
// The fan-out runs, or records why it did not (#126).
|
|
2958
|
+
//
|
|
2959
|
+
// The guard below was three conditions repeated on two `if`s with no `else`,
|
|
2960
|
+
// so a verify gate that already had a failure skipped both agents and left
|
|
2961
|
+
// their slots empty — and the gate then printed "B2 agent may not have run",
|
|
2962
|
+
// which is true and reads like the cause when it is the consequence. On run
|
|
2963
|
+
// wf_e105dd33-220 that sent me looking for a crashed agent that had never
|
|
2964
|
+
// started.
|
|
2965
|
+
//
|
|
2966
|
+
// Skipping is still right: the gate is failing anyway and these agents are
|
|
2967
|
+
// expensive. Skipping SILENTLY is what made the absence unreadable.
|
|
2968
|
+
if (gate === "verify") {
|
|
2969
|
+
const fanoutSkip = fanoutSkipReason(fails, testExitCode, process.env.RUNGATE_SKIP_AGENTS);
|
|
2970
|
+
|
|
2971
|
+
if (fanoutSkip) {
|
|
2972
|
+
console.log(`B1/B2: skipped — ${fanoutSkip}`);
|
|
2973
|
+
b2AgentResult = { skipped: fanoutSkip };
|
|
2974
|
+
b1VerifyAgentResult = { skipped: fanoutSkip };
|
|
2975
|
+
} else {
|
|
2976
|
+
const b2 = runB2EvidenceValidatorAgent(state, workDir, results, harnessRoot);
|
|
2977
|
+
b2AgentResult = b2.agentResult;
|
|
2978
|
+
warns += b2.warns;
|
|
2979
|
+
|
|
2980
|
+
const b1 = runB1AdversaryAtVerify(state, workDir, results, harnessRoot);
|
|
2981
|
+
b1VerifyAgentResult = b1.agentResult;
|
|
2982
|
+
warns += b1.warns;
|
|
2983
|
+
}
|
|
2984
|
+
}
|
|
2985
|
+
|
|
2986
|
+
// What this gate RAN, not only what it failed (#532).
|
|
2987
|
+
//
|
|
2988
|
+
// Read here rather than inside `writeGateResult` so the reading is taken
|
|
2989
|
+
// after every late push to `results` above — the AC-verdict failures, the
|
|
2990
|
+
// census WARN, the liveness probe. Those have no testcase behind them and
|
|
2991
|
+
// the JUnit report cannot know about them, so they are unioned in; a
|
|
2992
|
+
// `checksRun` that omitted them would not cover the failure ledger, and
|
|
2993
|
+
// the subtraction in lib/gate-refusal-currency.ts would read a check that
|
|
2994
|
+
// is still failing as resolved.
|
|
2995
|
+
const junit = readJunitCheckNames(junitPath);
|
|
2996
|
+
const checksRun = checksRunFrom(junit, results.map(r => r.check));
|
|
2997
|
+
if (checksRun === null) {
|
|
2998
|
+
// UNMEASURED, said out loud. An absent `checksRun` retires nothing, which
|
|
2999
|
+
// is the safe direction — but silence is how that becomes "this gate ran
|
|
3000
|
+
// nothing", and those are opposite facts.
|
|
3001
|
+
console.warn(`checksRun: UNMEASURED — ${junit.reason || "no reading"} (${junitPath})`);
|
|
3002
|
+
} else {
|
|
3003
|
+
console.log(`checksRun: ${checksRun.length} check(s) ran, ${junit.skipped} filtered out or skipped`);
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
// Write gate result
|
|
3007
|
+
const { resultVal, attempt } = writeGateResult(sf, gate, passes, fails, warns, results, state.projectRoot, checksRun);
|
|
3008
|
+
console.log(`Gate ${gate}: ${resultVal} (attempt ${attempt})`);
|
|
3009
|
+
|
|
3010
|
+
// Persist B1/B2 agent results
|
|
3011
|
+
if (gate === "verify" && (b2AgentResult || b1VerifyAgentResult)) {
|
|
3012
|
+
const freshState = JSON.parse(readFileSync(sf, "utf-8"));
|
|
3013
|
+
freshState.gates = freshState.gates || {};
|
|
3014
|
+
freshState.gates.verify = freshState.gates.verify || {};
|
|
3015
|
+
if (b2AgentResult) {
|
|
3016
|
+
freshState.gates.verify.evidenceValidator = b2AgentResult;
|
|
3017
|
+
}
|
|
3018
|
+
if (b1VerifyAgentResult) {
|
|
3019
|
+
freshState.gates.verify.adversary = b1VerifyAgentResult;
|
|
3020
|
+
}
|
|
3021
|
+
if (state.conformityFindings) {
|
|
3022
|
+
freshState.conformityFindings = state.conformityFindings;
|
|
3023
|
+
}
|
|
3024
|
+
writeFileSync(sf, JSON.stringify(freshState, null, 2));
|
|
3025
|
+
}
|
|
3026
|
+
|
|
3027
|
+
// Write tamper-evident witness
|
|
3028
|
+
try {
|
|
3029
|
+
const witnessPath = writeWitness(slug, gate, resultVal, testOutput, issue, state.projectRoot);
|
|
3030
|
+
console.log(`Witness: ${witnessPath}`);
|
|
3031
|
+
} catch (e: any) {
|
|
3032
|
+
console.error(`Witness write failed: ${e.message}`);
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
// Gate-specific post-processing (merge and prove exit directly)
|
|
3036
|
+
if (gate === "merge") {
|
|
3037
|
+
runMergeGate(state, sf);
|
|
3038
|
+
}
|
|
3039
|
+
|
|
3040
|
+
if (gate === "prove") {
|
|
3041
|
+
await runProvePostProcessing(state, workDir, issue, issueRepo, resultVal);
|
|
3042
|
+
}
|
|
3043
|
+
|
|
3044
|
+
// Ship-specific: HMAC + evidence
|
|
3045
|
+
if (gate === "ship" && resultVal === "PASS") {
|
|
3046
|
+
const hash = generateHmac(sf, slug, issue, state.projectRoot);
|
|
3047
|
+
console.log(`HMAC: ${hash}`);
|
|
3048
|
+
const evidencePath = generateShipEvidence(sf, workDir, issue, issueRepo, passes, fails, warns);
|
|
3049
|
+
console.log(`Ship evidence: ${evidencePath}`);
|
|
3050
|
+
}
|
|
3051
|
+
|
|
3052
|
+
// Verify-specific: convergence
|
|
3053
|
+
if (gate === "verify" && resultVal === "FAIL" && existsSync(CEREMONY_PROFILE)) {
|
|
3054
|
+
const convergenceResult = runConvergence(sf, gate, resultVal, CEREMONY_PROFILE);
|
|
3055
|
+
if (convergenceResult.action === "circuit-break") {
|
|
3056
|
+
console.log(`CIRCUIT BREAK — iteration ${convergenceResult.iteration}`);
|
|
3057
|
+
return { resultVal, passes, fails, warns, results, attempt, exitCode: 2 };
|
|
3058
|
+
} else if (convergenceResult.action === "stall") {
|
|
3059
|
+
console.log(`STALL DETECTED — same failures, iteration ${convergenceResult.iteration}`);
|
|
3060
|
+
} else if (convergenceResult.action === "loop-back") {
|
|
3061
|
+
console.log(`LOOP-BACK — iteration ${convergenceResult.iteration}`);
|
|
3062
|
+
}
|
|
3063
|
+
}
|
|
3064
|
+
|
|
3065
|
+
// Gate summary
|
|
3066
|
+
const sessionId = process.env.SESSION_ID || "unknown";
|
|
3067
|
+
const summaryResult = writeGateSummary(gate, passes, fails, warns, results, issue, slug, sessionId);
|
|
3068
|
+
const exitCode = summaryResult === "BLOCKED" ? 1 : 0;
|
|
3069
|
+
|
|
3070
|
+
return { resultVal, passes, fails, warns, results, attempt, exitCode };
|
|
3071
|
+
}
|