rungate 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -0
- package/gates/SCHEMA-GUIDE.md +485 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +283 -0
- package/gates/brief-assembler.test.ts +487 -0
- package/gates/brief-assembler.ts +448 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +250 -0
- package/gates/error-classifier.ts +99 -0
- package/gates/gate-executor.ts +3071 -0
- package/gates/orchestrator.test.ts +968 -0
- package/gates/orchestrator.ts +827 -0
- package/gates/preload.ts +89 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prompts.sha256 +3 -0
- package/gates/prove.test.ts +164 -0
- package/gates/rook-state.test.ts +108 -0
- package/gates/run-attribution.ts +290 -0
- package/gates/run-gate.ts +163 -0
- package/gates/schema-parity.test.ts +378 -0
- package/gates/schema.ts +1209 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +125 -0
- package/gates/ship-orchestrator.test.ts +399 -0
- package/gates/ship-orchestrator.ts +384 -0
- package/gates/spec-compliance-probe.ts +159 -0
- package/gates/test-fixtures/broken-no-environments.json +110 -0
- package/gates/test-fixtures/consumer-directory-layout.json +90 -0
- package/gates/test-fixtures/valid-standard.json +91 -0
- package/gates/test-utils.ts +43 -0
- package/gates/typecheck-gate.ts +111 -0
- package/gates/witness.ts +349 -0
- package/gates/workflow.test.ts +2018 -0
- package/hooks/BashWriteGuard.hook.ts +19 -0
- package/hooks/GateResultProtection.hook.ts +27 -1
- package/hooks/VerifyPhaseLock.hook.ts +51 -11
- package/hooks/WorkflowStateGuard.hook.ts +24 -0
- package/lib/ai-review-rounds.ts +462 -0
- package/lib/bash-file-read.ts +17 -3
- package/lib/bash-write-guard.ts +1285 -50
- package/lib/distribution-standing.ts +75 -4
- package/lib/failure-intake.ts +12 -4
- package/lib/file-extensions.ts +81 -0
- package/lib/finding-arbiter.ts +1015 -0
- package/lib/finding-jury.ts +537 -0
- package/lib/gate-enforcement.ts +171 -43
- package/lib/gate-result-protection.ts +12 -2
- package/lib/generators/code-map.ts +55 -4
- package/lib/generators/types.ts +27 -1
- package/lib/github.ts +55 -0
- package/lib/interpreters.ts +64 -0
- package/lib/issue-create-override.ts +69 -8
- package/lib/issue-filing-gate.ts +605 -0
- package/lib/js-top-level-code.ts +20 -2
- package/lib/junit-checks.ts +239 -1
- package/lib/package-closure.ts +114 -0
- package/lib/publish-window.ts +287 -0
- package/lib/scaffold/ai-review-findings.ts +142 -3
- package/lib/scaffold/ai-review-step.ts +1012 -18
- package/lib/scaffold/managed-workflow.ts +163 -4
- package/lib/scaffold/steps.ts +498 -22
- package/lib/scanner.ts +13 -4
- package/lib/ticked-sc-symbols.ts +361 -0
- package/lib/transcript-checker.ts +6 -0
- package/lib/workflow-actions.ts +18 -0
- package/package.json +3 -1
- package/scripts/arbitrate-finding.ts +224 -0
- package/scripts/generate-code-map.ts +18 -10
- package/scripts/sync-sc-status.ts +82 -8
|
@@ -0,0 +1,2018 @@
|
|
|
1
|
+
import { test, expect, describe, setDefaultTimeout } from "bun:test";
|
|
2
|
+
import { readFileSync, existsSync, readdirSync } from "fs";
|
|
3
|
+
import { execSync, execFileSync, spawnSync } from "child_process";
|
|
4
|
+
import { createHash } from "crypto";
|
|
5
|
+
import { join } from "path";
|
|
6
|
+
import { WorkflowStateSchema } from "./schema";
|
|
7
|
+
import { extractTestFailureCount, subtractTestBaseline } from "./gate-executor";
|
|
8
|
+
import { typecheckFailures } from "./typecheck-gate";
|
|
9
|
+
import {
|
|
10
|
+
missingEvidenceViolations,
|
|
11
|
+
thresholdCrosscheckViolations,
|
|
12
|
+
} from "../lib/evidence-required";
|
|
13
|
+
import {
|
|
14
|
+
classifyContainerProvenance,
|
|
15
|
+
containerProvenanceViolations,
|
|
16
|
+
resolveContainerRuntime,
|
|
17
|
+
} from "../lib/container-provenance";
|
|
18
|
+
import { evidencePortabilityViolations } from "../lib/evidence-portability";
|
|
19
|
+
import { alreadySatisfiedIds } from "../lib/evidence-prevalidator";
|
|
20
|
+
import { readBranchDiff } from "../lib/branch-diff";
|
|
21
|
+
import { proveLabelGateVerdict, type LabelRead } from "../lib/prove-label-gate";
|
|
22
|
+
import { worktreeDiskIdentity, type RemoteReading } from "../lib/worktree-repo";
|
|
23
|
+
import { readSuiteMeasurement, type SuiteReading } from "../lib/suite-measurement";
|
|
24
|
+
import { briefAcAlignment } from "../lib/brief-ac-alignment";
|
|
25
|
+
import {
|
|
26
|
+
EXEC_TIMEOUT_MS,
|
|
27
|
+
GATE_TEST_TIMEOUT_MS,
|
|
28
|
+
TYPECHECK_TIMEOUT_MS,
|
|
29
|
+
} from "../lib/gate-timeouts";
|
|
30
|
+
import {
|
|
31
|
+
outcomeEvidenceViolations,
|
|
32
|
+
goalKeywordCoverage,
|
|
33
|
+
GOAL_COVERAGE_WARN_BELOW,
|
|
34
|
+
} from "../lib/ac-quality";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Every check in this file gets longer than the processes it starts (#550).
|
|
38
|
+
*
|
|
39
|
+
* File level and not per-test on purpose: `}, 120000);` on `tsc-pass` alone
|
|
40
|
+
* fixes one check, and the exec helpers below are called from dozens. The
|
|
41
|
+
* ordering against the spawns is owned by lib/gate-timeouts.ts and asserted
|
|
42
|
+
* in test/gate-suite-timeouts.test.ts.
|
|
43
|
+
*/
|
|
44
|
+
setDefaultTimeout(GATE_TEST_TIMEOUT_MS);
|
|
45
|
+
|
|
46
|
+
const STANDALONE = !process.env.TEST_WORK_DIR;
|
|
47
|
+
const TEST_DIR = process.env.TEST_WORK_DIR || "/tmp/gate-viability-test";
|
|
48
|
+
const SF = `${TEST_DIR}/workflow-state.json`;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* A path inside THIS run's directory — the one the gate handed us (#700).
|
|
52
|
+
*
|
|
53
|
+
* Six checks used to rebuild this directory a second way, from
|
|
54
|
+
* `process.env.RUNGATE_WORK_DIR` (or a `$HOME/.rungate` fallback) joined to the
|
|
55
|
+
* state's slug. That is the same answer derived twice, and the two can
|
|
56
|
+
* disagree: `bun test gates/` runs every file in this directory in ONE process,
|
|
57
|
+
* so a module-scope `process.env.RUNGATE_WORK_DIR = …` in any sibling file is a
|
|
58
|
+
* write to this one's environment, and which of them lands first is a
|
|
59
|
+
* filesystem walk order rather than a decision anybody made.
|
|
60
|
+
*
|
|
61
|
+
* `gate-executor.ts:1523` already passes the resolved directory as
|
|
62
|
+
* `TEST_WORK_DIR`, and `SF` above reads the state out of it. So the directory
|
|
63
|
+
* is not in doubt here; it only became doubtful when a check went and asked a
|
|
64
|
+
* global for it again.
|
|
65
|
+
*
|
|
66
|
+
* One binding, so there is one thing to mutate
|
|
67
|
+
* (`.claude/rules/checks-must-be-able-to-fail.md`). Shared, cross-run paths —
|
|
68
|
+
* `signals` — are deliberately NOT routed through this; they want the base.
|
|
69
|
+
*/
|
|
70
|
+
function runArtifact(...parts: string[]): string {
|
|
71
|
+
return join(TEST_DIR, ...parts);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function loadRaw(): any | null {
|
|
75
|
+
if (!existsSync(SF)) return null;
|
|
76
|
+
return JSON.parse(readFileSync(SF, "utf-8"));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function sf(path: string): any {
|
|
80
|
+
const wf = loadRaw();
|
|
81
|
+
if (!wf) return undefined;
|
|
82
|
+
const parts = path.replace(/^\./, "").split(".").filter(Boolean);
|
|
83
|
+
let val: any = wf;
|
|
84
|
+
for (const p of parts) {
|
|
85
|
+
if (val == null) return undefined;
|
|
86
|
+
val = val[p];
|
|
87
|
+
}
|
|
88
|
+
return val;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function exec(cmd: string, cwd?: string): { ok: boolean; output: string } {
|
|
92
|
+
try {
|
|
93
|
+
const output = execSync(cmd, { cwd, encoding: "utf-8", timeout: EXEC_TIMEOUT_MS }).trim();
|
|
94
|
+
return { ok: true, output };
|
|
95
|
+
} catch (e: any) {
|
|
96
|
+
return { ok: false, output: (e.stdout ?? "").trim() };
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Argv-form git, for commands carrying a value out of a config file.
|
|
102
|
+
*
|
|
103
|
+
* `exec` above goes through a shell, so `git status --porcelain -- ${paths}`
|
|
104
|
+
* hands consumer-authored strings to `sh`. `codeCommittedPaths` is read from
|
|
105
|
+
* the project's own `.claude/rungate.json`, and a path list is exactly the
|
|
106
|
+
* kind of field somebody pastes into without thinking about quoting. Argv
|
|
107
|
+
* form removes the shell rather than trying to sanitise for it.
|
|
108
|
+
*
|
|
109
|
+
* `--` is passed by the caller, so a pathspec beginning with `-` is a
|
|
110
|
+
* pathspec and not an option.
|
|
111
|
+
*/
|
|
112
|
+
function execArgvOf(bin: string, args: string[], cwd?: string): { ok: boolean; output: string } {
|
|
113
|
+
try {
|
|
114
|
+
const output = execFileSync(bin, args, { cwd, encoding: "utf-8", timeout: EXEC_TIMEOUT_MS }).trim();
|
|
115
|
+
return { ok: true, output };
|
|
116
|
+
} catch (e: any) {
|
|
117
|
+
return { ok: false, output: `${e?.stdout ?? ""}${e?.stderr ?? ""}`.trim() || String(e?.message ?? e) };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function execArgv(args: string[], cwd?: string): { ok: boolean; output: string } {
|
|
122
|
+
try {
|
|
123
|
+
const output = execFileSync("git", args, { cwd, encoding: "utf-8", timeout: EXEC_TIMEOUT_MS }).trim();
|
|
124
|
+
return { ok: true, output };
|
|
125
|
+
} catch (e: any) {
|
|
126
|
+
return { ok: false, output: `${e?.stdout ?? ""}${e?.stderr ?? ""}`.trim() || String(e?.message ?? e) };
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function loadProjectHarness(): any {
|
|
131
|
+
const root = sf("projectRoot");
|
|
132
|
+
if (!root) return null;
|
|
133
|
+
const p = join(root, ".claude", "rungate.json");
|
|
134
|
+
if (!existsSync(p)) return null;
|
|
135
|
+
return JSON.parse(readFileSync(p, "utf-8"));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* #314 — the tree a check MEASURES, as opposed to where config is READ.
|
|
140
|
+
*
|
|
141
|
+
* `projectRoot` is the main checkout. On a worktree-isolated run the change
|
|
142
|
+
* under test is not in it, so a command executed there measures code that is
|
|
143
|
+
* already merged and green. Measured on run `wf_5cdce852-1e2`: `tsc-pass` ran
|
|
144
|
+
* the compiler in `projectRoot`, reported PASS, and CI then caught two real
|
|
145
|
+
* type errors on the branch.
|
|
146
|
+
*
|
|
147
|
+
* Returns `null` when the state does not name the tree. Every caller must
|
|
148
|
+
* refuse on `null` rather than substitute `projectRoot` — that substitution is
|
|
149
|
+
* the defect, and it is invisible because it always produces a pass.
|
|
150
|
+
*/
|
|
151
|
+
export function treeUnderTestViolations(tree: unknown, checkName: string): string[] {
|
|
152
|
+
if (typeof tree !== "string" || tree.trim() === "") {
|
|
153
|
+
return [
|
|
154
|
+
`${checkName}: workflow state does not name treeUnderTest, so there is ` +
|
|
155
|
+
`nothing to measure. This is UNMEASURED, not PASS — measuring ` +
|
|
156
|
+
`projectRoot instead would check the main checkout, which is green by ` +
|
|
157
|
+
`construction and is not the change being shipped (#314).`,
|
|
158
|
+
];
|
|
159
|
+
}
|
|
160
|
+
return [];
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Does the tree have commits its upstream does not (#174)?
|
|
165
|
+
*
|
|
166
|
+
* The reading that matters is NOT just the count. `git rev-list --count
|
|
167
|
+
* @{upstream}..HEAD` fails outright when there is no upstream, and the old
|
|
168
|
+
* body read that failure as nothing-to-report:
|
|
169
|
+
*
|
|
170
|
+
* if (!result.ok) return; // no upstream
|
|
171
|
+
*
|
|
172
|
+
* A branch with no upstream has never been pushed, which is the strongest
|
|
173
|
+
* possible failure of a check named "no unpushed commits" — and it was the one
|
|
174
|
+
* input that passed unconditionally. Fail-open, shape one in
|
|
175
|
+
* .claude/rules/checks-must-be-able-to-fail.md, in the check whose whole job is
|
|
176
|
+
* to notice unpushed work.
|
|
177
|
+
*
|
|
178
|
+
* It is reachable, not theoretical. The happy path sets upstream
|
|
179
|
+
* (`workflows/ship.js:3161`, `git push -u origin ${pushTarget}`), but both
|
|
180
|
+
* remediation re-pushes do not — `:3324` and `:4704` run
|
|
181
|
+
* `git push origin HEAD:${shipBranch}`. So a run that needed remediation
|
|
182
|
+
* arrives here with no upstream and is waved through.
|
|
183
|
+
*
|
|
184
|
+
* Exported as the single binding this check's refusal is routed through, so
|
|
185
|
+
* `return []` is one edit and the mutation is a real measurement rather than a
|
|
186
|
+
* claim in a PR body.
|
|
187
|
+
*/
|
|
188
|
+
export function unpushedViolations(
|
|
189
|
+
count: { ok: boolean; output: string },
|
|
190
|
+
upstream: { ok: boolean; output: string },
|
|
191
|
+
): string[] {
|
|
192
|
+
if (count.ok) {
|
|
193
|
+
return count.output === "0" ? [] : [`code-pushed: ${count.output} commits ahead of upstream`];
|
|
194
|
+
}
|
|
195
|
+
if (!upstream.ok) {
|
|
196
|
+
return [
|
|
197
|
+
"code-pushed: the branch has no upstream, so nothing on it has been " +
|
|
198
|
+
"pushed. This is the WORST case for this check, not an exemption from " +
|
|
199
|
+
"it — the previous code returned here and passed (#174). A remediation " +
|
|
200
|
+
"re-push uses `git push origin HEAD:<branch>`, which does not set " +
|
|
201
|
+
"upstream, so this is reachable on any run that needed one.",
|
|
202
|
+
];
|
|
203
|
+
}
|
|
204
|
+
return [
|
|
205
|
+
`code-pushed: the branch tracks ${upstream.output} but the commit count ` +
|
|
206
|
+
`could not be read. This is UNMEASURED, not PASS — a git invocation that ` +
|
|
207
|
+
`fails for an unknown reason says nothing about whether work is unpushed.`,
|
|
208
|
+
];
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* #315 — `code-committed` has never executed.
|
|
213
|
+
*
|
|
214
|
+
* The body read `harness?.codeCommittedPaths || []` and returned early on an
|
|
215
|
+
* empty list. **Nothing authors that key.** Measured on main: neither
|
|
216
|
+
* rungate's `.claude/rungate/config.json`, nor its legacy
|
|
217
|
+
* `.claude/rungate.json`, nor DailyBriefDashboard's names it. So the check
|
|
218
|
+
* whose entire job is confirming the shipped work is committed hit the early
|
|
219
|
+
* return on every run this repository has ever made — shape one in
|
|
220
|
+
* .claude/rules/checks-must-be-able-to-fail.md, keyed to a config value that
|
|
221
|
+
* does not exist.
|
|
222
|
+
*
|
|
223
|
+
* It compounded with #314: the status also ran in `projectRoot`, the main
|
|
224
|
+
* checkout, which is clean during a worktree run. Both halves had to be
|
|
225
|
+
* fixed or the check means nothing; #314 fixed the tree, this fixes the scope.
|
|
226
|
+
*
|
|
227
|
+
* ## The scope decision (SC-E2), recorded here rather than left to infer
|
|
228
|
+
*
|
|
229
|
+
* The key becomes OPTIONAL WITH A DERIVED DEFAULT rather than required.
|
|
230
|
+
*
|
|
231
|
+
* Requiring it would refuse at scaffold time for every consumer that has
|
|
232
|
+
* never heard of it — an outage for a rule nobody opted into, which is the
|
|
233
|
+
* same reasoning `loadConformityAllowlists` and #302's growth check already
|
|
234
|
+
* apply. And the default is not a guess: for a check named "no uncommitted
|
|
235
|
+
* changes in project", the honest scope is **the whole tree**, and a
|
|
236
|
+
* configured path list is a narrowing a consumer opts into, not the thing
|
|
237
|
+
* that switches the check on.
|
|
238
|
+
*
|
|
239
|
+
* Untracked files are included, because `--porcelain` already honours
|
|
240
|
+
* `.gitignore` and a source file that was never added is exactly the
|
|
241
|
+
* uncommitted work this check exists to catch. Build artefacts that trip it
|
|
242
|
+
* belong in `.gitignore`; that is the right pressure rather than a reason to
|
|
243
|
+
* narrow the check.
|
|
244
|
+
*
|
|
245
|
+
* Exported as the single binding this check's refusal is routed through, so
|
|
246
|
+
* `return []` is one edit and the mutation is a measurement rather than a
|
|
247
|
+
* claim in a PR body.
|
|
248
|
+
*/
|
|
249
|
+
export function codeCommittedViolations(
|
|
250
|
+
status: { ok: boolean; output: string },
|
|
251
|
+
scope: { configured: boolean; paths: string[] },
|
|
252
|
+
): string[] {
|
|
253
|
+
if (!status.ok) {
|
|
254
|
+
return [
|
|
255
|
+
`code-committed: \`git status\` could not be read in the tree under test ` +
|
|
256
|
+
`(${status.output || "no output"}). This is UNMEASURED, not PASS — a git ` +
|
|
257
|
+
`invocation that fails for an unknown reason says nothing about whether the ` +
|
|
258
|
+
`work is committed.`,
|
|
259
|
+
];
|
|
260
|
+
}
|
|
261
|
+
const dirty = status.output.split("\n").map(l => l.trim()).filter(Boolean);
|
|
262
|
+
if (dirty.length === 0) return [];
|
|
263
|
+
const where = scope.configured ? ` within ${scope.paths.join(", ")}` : "";
|
|
264
|
+
const shown = dirty.slice(0, 10).join("; ");
|
|
265
|
+
return [
|
|
266
|
+
`code-committed: ${dirty.length} uncommitted change(s) in the tree under test${where}: ` +
|
|
267
|
+
`${shown}${dirty.length > 10 ? ` … and ${dirty.length - 10} more` : ""}`,
|
|
268
|
+
];
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* #414 — the worktree-repository check, read from the disk rather than from
|
|
273
|
+
* the agent standing in it.
|
|
274
|
+
*
|
|
275
|
+
* #218 refuses a worktree whose `origin` does not match the run's repo, and
|
|
276
|
+
* the remote it compares is SELF-REPORTED: `workflows/ship.js` has no
|
|
277
|
+
* subprocess (#69), so the only remote it can see is the one the agent said
|
|
278
|
+
* it read. An agent that answers from its brief rather than from
|
|
279
|
+
* `git remote get-url origin` produces a reply indistinguishable from a
|
|
280
|
+
* correct one, and `lib/worktree-repo.ts` has exported the unforgeable half
|
|
281
|
+
* since #218 with nothing able to call it. A gate can: gates run as real
|
|
282
|
+
* subprocesses with the worktree on disk.
|
|
283
|
+
*
|
|
284
|
+
* Exported as the single binding this check's refusal is routed through, so
|
|
285
|
+
* `return []` is one edit and the mutation in
|
|
286
|
+
* `test/suite-binding-mutation.test.ts` is a measurement rather than a claim
|
|
287
|
+
* in a PR body. Keep it one function with one body — a second refusal path
|
|
288
|
+
* for the same fact would survive that mutation.
|
|
289
|
+
*
|
|
290
|
+
* AN EMPTY LIST REFUSES. Every run this fires on has at least a tree under
|
|
291
|
+
* test, so zero entries means the state stopped naming them, and a check that
|
|
292
|
+
* scans nothing reports nothing and is indistinguishable from compliance.
|
|
293
|
+
*/
|
|
294
|
+
export function worktreeRepoViolations(entries: unknown, expectedRepo: unknown, read?: (p: string) => RemoteReading): string[] {
|
|
295
|
+
if (!Array.isArray(entries) || entries.length === 0) {
|
|
296
|
+
return [
|
|
297
|
+
"worktree-repo: the workflow state names no worktree to read, so no checkout " +
|
|
298
|
+
"was compared against this run's repository. This is UNMEASURED, not PASS — " +
|
|
299
|
+
"every run reaches this gate with at least a tree under test, so an empty " +
|
|
300
|
+
"list means the state stopped recording them (#414).",
|
|
301
|
+
];
|
|
302
|
+
}
|
|
303
|
+
const out: string[] = [];
|
|
304
|
+
const seen = new Set<string>();
|
|
305
|
+
for (const raw of entries) {
|
|
306
|
+
const e = (raw ?? {}) as { path?: unknown; reportedRepo?: unknown };
|
|
307
|
+
const path = typeof e.path === "string" ? e.path.trim() : "";
|
|
308
|
+
// One worktree, one reading. A worktree that reported in two parts (#228)
|
|
309
|
+
// appears twice, and two identical refusals for it is noise an operator
|
|
310
|
+
// has to deduplicate by hand. Keyed on the empty string too, so a run of
|
|
311
|
+
// pathless rows refuses once.
|
|
312
|
+
if (seen.has(path)) continue;
|
|
313
|
+
seen.add(path);
|
|
314
|
+
const reported = typeof e.reportedRepo === "string" ? e.reportedRepo : null;
|
|
315
|
+
const got = worktreeDiskIdentity(
|
|
316
|
+
path,
|
|
317
|
+
typeof expectedRepo === "string" ? expectedRepo : null,
|
|
318
|
+
reported,
|
|
319
|
+
read,
|
|
320
|
+
);
|
|
321
|
+
if (got.refusal) out.push(`worktree-repo: ${got.refusal}`);
|
|
322
|
+
}
|
|
323
|
+
return out;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/** The tree to run commands in, or null when the state does not name one. */
|
|
327
|
+
function treeRoot(): string | null {
|
|
328
|
+
const t = sf("treeUnderTest");
|
|
329
|
+
return typeof t === "string" && t.trim() !== "" ? t : null;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
const isVerifyPlus = () => { const p = sf("phase"); return ["VERIFY", "SHIP", "DONE"].includes(p); };
|
|
333
|
+
const isShipPlus = () => { const p = sf("phase"); return ["SHIP", "DONE"].includes(p); };
|
|
334
|
+
const isDone = () => sf("phase") === "DONE";
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* #224 — the suite-to-verdict binding.
|
|
338
|
+
*
|
|
339
|
+
* `environments.local.tests` is the only reading this harness has of whether
|
|
340
|
+
* the suite passed on the tree being shipped, and this is the single site that
|
|
341
|
+
* turns that reading into a refusal. Keep it one function with one body: a
|
|
342
|
+
* second refusal path for the same fact would survive the mutation in
|
|
343
|
+
* test/suite-binding-mutation.test.ts and make the proof vacuous, which is why
|
|
344
|
+
* that file asserts this signature appears exactly once and aborts if it moves.
|
|
345
|
+
*
|
|
346
|
+
* Absent is a refusal, not a pass. That is the whole of #224: the run that
|
|
347
|
+
* reported `regressions: 0` over one failing test had measured nothing, and
|
|
348
|
+
* nothing was read as zero.
|
|
349
|
+
*
|
|
350
|
+
* #530 added the second argument. `environments.local.tests` is an enum with
|
|
351
|
+
* no provenance in it, and until #530 nothing in the harness wrote it — the
|
|
352
|
+
* recorder wrote `suiteMeasurement` and the field was left to a sentence in
|
|
353
|
+
* a heal prompt that only runs when the scope gate fails. Now the recorder
|
|
354
|
+
* writes both in one write, and this reads both: a PASS the measurement does
|
|
355
|
+
* not back is refused, which is the hand-written verdict the ship prompt has
|
|
356
|
+
* only ever forbidden in prose.
|
|
357
|
+
*
|
|
358
|
+
* The clauses do not overlap. An absent field, a `FAIL`, and a declared
|
|
359
|
+
* `SKIP` are each decided before the measurement is consulted at all, and
|
|
360
|
+
* `{tests: "PASS"}` with no `suiteMeasurement` is reached by nothing else.
|
|
361
|
+
*
|
|
362
|
+
* The second argument is REQUIRED, and that is the point of it. Optional, it
|
|
363
|
+
* would be a check any future caller could lose by omission — and losing it
|
|
364
|
+
* is silent, because the three clauses above still refuse everything they
|
|
365
|
+
* refused before, so the call reads exactly like a call that is checking.
|
|
366
|
+
* `readSuiteMeasurement` is a total function that returns UNMEASURED for
|
|
367
|
+
* every unusable input, so there is never a caller with nothing to pass.
|
|
368
|
+
*
|
|
369
|
+
* **What the requirement buys is that the absence now fails CLOSED.** With an
|
|
370
|
+
* optional argument, omitting it meant "no objection". Required, the only
|
|
371
|
+
* thing a caller can hand this for a state with no measurement is an
|
|
372
|
+
* UNMEASURED reading, and `UNMEASURED !== "PASS"` refuses. The failure mode
|
|
373
|
+
* inverted: absence used to be silence and is now a refusal. That is
|
|
374
|
+
* `.claude/rules/config-access.md`'s Silent Default Substitution at a second
|
|
375
|
+
* site — `0`, `{}`, `null` and `SKIP` are indistinguishable from success;
|
|
376
|
+
* `UNMEASURED` is not.
|
|
377
|
+
*/
|
|
378
|
+
export function suiteVerdictViolations(val: unknown, measurement: SuiteReading): string[] {
|
|
379
|
+
if (val === undefined || val === null || val === "") {
|
|
380
|
+
return [
|
|
381
|
+
"local.tests not set — PASS or SKIP required when dev.testCmd is configured. " +
|
|
382
|
+
"An unmeasured suite is not a passing suite (#224).",
|
|
383
|
+
];
|
|
384
|
+
}
|
|
385
|
+
if (val === "SKIP") return []; // SKIP is allowed — it says so, rather than implying PASS
|
|
386
|
+
if (val !== "PASS") return [`tests: ${String(val)} — expected PASS`];
|
|
387
|
+
// A PASS has to be backed by the measurement that produced it. No
|
|
388
|
+
// `measurement &&` guard: a branch for an argument that cannot be absent
|
|
389
|
+
// is a branch no test can enter.
|
|
390
|
+
if (measurement.verdict !== "PASS") {
|
|
391
|
+
return [
|
|
392
|
+
`tests: PASS, but the suite measurement behind it reads ${measurement.verdict} — ` +
|
|
393
|
+
`${measurement.reason || "the two disagree"} (#530). A verdict no measurement backs is a ` +
|
|
394
|
+
`verdict somebody typed.`,
|
|
395
|
+
];
|
|
396
|
+
}
|
|
397
|
+
return [];
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* #126 — the Verify fan-out leaves a trace, or the gate refuses.
|
|
402
|
+
*
|
|
403
|
+
* The two checks below used to read their slot and, finding nothing, warn and
|
|
404
|
+
* return:
|
|
405
|
+
*
|
|
406
|
+
* if (!ev) { console.warn("...not populated — B2 agent may not have run"); return; }
|
|
407
|
+
*
|
|
408
|
+
* A test that returns early passes. The only input either could fail on was a
|
|
409
|
+
* slot written WRONG, so the failure mode this exists for — the agent never
|
|
410
|
+
* ran — was the single case they could not catch. "When populated" was in
|
|
411
|
+
* both names.
|
|
412
|
+
*
|
|
413
|
+
* The producer reached not-populated four ways, three of them silent: the
|
|
414
|
+
* `fails === 0` guard in gates/gate-executor.ts (a verify gate with any
|
|
415
|
+
* failure skips the agents and then reports their absence, which reads as the
|
|
416
|
+
* cause when it is the consequence), `RUNGATE_SKIP_AGENTS`, a LIGHT tier, and
|
|
417
|
+
* a missing prompt file. So the fan-out could be absent in full and the gate
|
|
418
|
+
* said PASS, which makes every compliance number recorded for Verify a number
|
|
419
|
+
* for a stage that may not have run.
|
|
420
|
+
*
|
|
421
|
+
* One function, one body, for the same reason `suiteVerdictViolations` above
|
|
422
|
+
* is: a second refusal path for the same fact would survive the mutation in
|
|
423
|
+
* test/suite-binding-mutation.test.ts and make the proof vacuous.
|
|
424
|
+
*
|
|
425
|
+
* A DELIBERATE skip passes, and must be written down as one. Refusing on
|
|
426
|
+
* every absence would brick LIGHT runs, and a fix that brings its own outage
|
|
427
|
+
* is how the next fail-open gets argued for. But a skip with no reason is an
|
|
428
|
+
* absence with extra steps, and it would be the cheapest way to make this
|
|
429
|
+
* whole check green again — so it refuses.
|
|
430
|
+
*/
|
|
431
|
+
export function verifyFanoutViolations(label: string, val: unknown, tier: unknown): string[] {
|
|
432
|
+
// The producer tier-gates B1 at gates/gate-executor.ts:1093, so a LIGHT run
|
|
433
|
+
// legitimately has no record. This is the one absence that is expected.
|
|
434
|
+
if (tier === "LIGHT") return [];
|
|
435
|
+
|
|
436
|
+
if (val === undefined || val === null || val === "") {
|
|
437
|
+
return [
|
|
438
|
+
`gates.verify.${label} not recorded — the agent left no trace, and an absence is ` +
|
|
439
|
+
`not an approval (#126). A deliberate skip must be recorded as ` +
|
|
440
|
+
`{"skipped": "<reason>"} so the artefact can tell "it approved" from "it never ran".`,
|
|
441
|
+
];
|
|
442
|
+
}
|
|
443
|
+
if (typeof val !== "object") {
|
|
444
|
+
return [`gates.verify.${label} is ${typeof val}, expected a record (#126)`];
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
const skipped = (val as { skipped?: unknown }).skipped;
|
|
448
|
+
if (skipped !== undefined) {
|
|
449
|
+
return typeof skipped === "string" && skipped.trim().length > 0
|
|
450
|
+
? []
|
|
451
|
+
: [
|
|
452
|
+
`gates.verify.${label} records a skip with no reason — a skip nobody can read ` +
|
|
453
|
+
`is an absence with extra steps (#126)`,
|
|
454
|
+
];
|
|
455
|
+
}
|
|
456
|
+
return [];
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* #235 — the evidence pre-validation reading reaches the verdict.
|
|
461
|
+
*
|
|
462
|
+
* `lib/evidence-prevalidator.ts` dry-runs every AC evidence command at SCOPE
|
|
463
|
+
* and classifies each one `ok` / `broken` / `empty` / `skipped`. Until this
|
|
464
|
+
* function existed the entire result went to `console.error` and nowhere else:
|
|
465
|
+
* a run could be told, in its own log, that three of its ACs could not be
|
|
466
|
+
* measured, and still ship — the detection was never the gap.
|
|
467
|
+
*
|
|
468
|
+
* This is the single site that turns that reading into a refusal, for the same
|
|
469
|
+
* reason `suiteVerdictViolations` above is: a second refusal path for the same
|
|
470
|
+
* fact would survive the mutation in test/suite-binding-mutation.test.ts and
|
|
471
|
+
* make the proof vacuous. That file asserts this signature appears exactly once
|
|
472
|
+
* and aborts when it moves.
|
|
473
|
+
*
|
|
474
|
+
* Absent is a refusal, not a pass. An unmeasured pre-validation says nothing
|
|
475
|
+
* about the evidence commands, and "nothing" has already been read as "clean"
|
|
476
|
+
* once in this harness (#224).
|
|
477
|
+
*/
|
|
478
|
+
export function prevalidationViolations(val: unknown, gatePhase?: unknown): string[] {
|
|
479
|
+
const unmeasured = [
|
|
480
|
+
"evidencePrevalidation not recorded — a PASS reading is required once any AC " +
|
|
481
|
+
"carries an evidence command. An unmeasured pre-validation is not a clean one (#235).",
|
|
482
|
+
];
|
|
483
|
+
|
|
484
|
+
if (val === undefined || val === null || val === "") return unmeasured;
|
|
485
|
+
if (typeof val !== "object") {
|
|
486
|
+
return [`evidencePrevalidation is ${typeof val}, expected a record with a verdict (#235)`];
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
// #280 — a reading whose own `takenAt` says it describes an earlier tree is
|
|
490
|
+
// not evidence about this one.
|
|
491
|
+
//
|
|
492
|
+
// The producer is supposed to re-measure at ship, and this is what makes a
|
|
493
|
+
// producer that stops doing so a REFUSAL rather than a stale artefact
|
|
494
|
+
// recorded as current. That distinction is the whole of #280: the ship gate
|
|
495
|
+
// still refused, correctly and for the right reason, while filing a block
|
|
496
|
+
// stamped `takenAt: "verify"` and an hour old as its own reading.
|
|
497
|
+
//
|
|
498
|
+
// Only at ship, and deliberately only there. A SCOPE reading survives to
|
|
499
|
+
// verify by design (#264) — re-measuring the run's starting conditions
|
|
500
|
+
// after Marcus has run answers a different question — so there is no
|
|
501
|
+
// general "earlier phase is stale" rule to apply here, and inventing one
|
|
502
|
+
// would turn every LIGHT run red.
|
|
503
|
+
const ACCEPTED_AT_SHIP = "ship";
|
|
504
|
+
if (gatePhase === "SHIP" || gatePhase === "DONE") {
|
|
505
|
+
const source = (val as { takenAt?: unknown }).takenAt;
|
|
506
|
+
if (source !== ACCEPTED_AT_SHIP) {
|
|
507
|
+
return [
|
|
508
|
+
`evidence pre-validation was taken at ${source ? `\`${String(source)}\`` : "an unrecorded phase"}, ` +
|
|
509
|
+
"not at ship — the ship gate must measure its own tree. Between verify and ship the " +
|
|
510
|
+
"run commits, pushes, and may move the branch (#280).",
|
|
511
|
+
];
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
const reading = val as { verdict?: unknown; acs?: unknown };
|
|
516
|
+
const verdict = reading.verdict;
|
|
517
|
+
if (verdict === undefined || verdict === null || verdict === "" || verdict === "UNMEASURED") {
|
|
518
|
+
return unmeasured;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// #295 — derived from `acs`, which is what the producer writes and what
|
|
522
|
+
// `EvidencePrevalidationShape` declares. This read `reading.broken`, a key
|
|
523
|
+
// no writer has ever produced and the schema has never declared, so the list
|
|
524
|
+
// was empty on every run: every FAIL fell through to the "no broken AC ids
|
|
525
|
+
// were recorded" branch below, and the refusal whose job is to say what to
|
|
526
|
+
// fix could not say it.
|
|
527
|
+
//
|
|
528
|
+
// `status === "broken"` and nothing else. `empty` is a warning rather than a
|
|
529
|
+
// refusal (lib/evidence-prevalidator.ts), and `skipped` never ran — folding
|
|
530
|
+
// either in here would turn a FAIL that names two ACs into one that names
|
|
531
|
+
// five, which is the same unusable refusal from the other direction.
|
|
532
|
+
const broken = (Array.isArray(reading.acs) ? reading.acs : [])
|
|
533
|
+
.filter((ac: unknown) => (ac as { status?: unknown })?.status === "broken")
|
|
534
|
+
.map((ac: unknown) => String((ac as { id?: unknown })?.id ?? ""))
|
|
535
|
+
.filter((id: string) => id.length > 0);
|
|
536
|
+
|
|
537
|
+
// #669 — an AC whose PRE-CHANGE output already cleared its own threshold
|
|
538
|
+
// cannot observe the work. It printed a satisfying value before a line was
|
|
539
|
+
// written, so it will print one afterwards whatever Marcus does: a check
|
|
540
|
+
// that cannot fail, arriving in the run's own acceptance criteria.
|
|
541
|
+
//
|
|
542
|
+
// Derived through `alreadySatisfiedIds`, the same binding the writer and
|
|
543
|
+
// the schema use, so the refusal cannot name a different set than the
|
|
544
|
+
// reading carries (#295 is the cost of getting that wrong).
|
|
545
|
+
//
|
|
546
|
+
// ONLY on a reading taken at scope, and the restriction is the point. Scope
|
|
547
|
+
// is the one phase whose tree is the pre-change one. At verify and at ship
|
|
548
|
+
// the work has landed, so an AC that satisfies its threshold there is an AC
|
|
549
|
+
// that PASSED — refusing those would turn every successful run red, which
|
|
550
|
+
// is the fastest way to build a refusal everyone learns to click past. A
|
|
551
|
+
// reading with no `takenAt` predates the field and cannot say which tree it
|
|
552
|
+
// describes, so it is not refused either.
|
|
553
|
+
const vacuous =
|
|
554
|
+
(val as { takenAt?: unknown }).takenAt === "scope" ? alreadySatisfiedIds(reading.acs) : [];
|
|
555
|
+
const vacuousRefusal =
|
|
556
|
+
vacuous.length > 0
|
|
557
|
+
? [
|
|
558
|
+
`evidence pre-validation: ${vacuous.join(", ")} already satisfied their own thresholds ` +
|
|
559
|
+
"on the pre-change tree — an AC that passes before the work is done certifies the " +
|
|
560
|
+
"change without observing it (#669)",
|
|
561
|
+
]
|
|
562
|
+
: [];
|
|
563
|
+
|
|
564
|
+
if (verdict === "FAIL") {
|
|
565
|
+
return [
|
|
566
|
+
...(broken.length > 0
|
|
567
|
+
? [`evidence pre-validation: FAIL — broken evidence commands on ${broken.join(", ")}`]
|
|
568
|
+
: [
|
|
569
|
+
"evidence pre-validation: FAIL — but no broken AC ids were recorded, so the " +
|
|
570
|
+
"refusal cannot name what to fix (#235)",
|
|
571
|
+
]),
|
|
572
|
+
...vacuousRefusal,
|
|
573
|
+
];
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
if (verdict !== "PASS") {
|
|
577
|
+
return [`evidence pre-validation: ${String(verdict)} — expected PASS`];
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// A PASS carrying broken ids is a contradiction, not a pass. Without this the
|
|
581
|
+
// writer could record the broken list faithfully and still be waved through.
|
|
582
|
+
if (broken.length > 0) {
|
|
583
|
+
return [
|
|
584
|
+
`evidence pre-validation: PASS recorded alongside broken evidence commands on ${broken.join(", ")}`,
|
|
585
|
+
...vacuousRefusal,
|
|
586
|
+
];
|
|
587
|
+
}
|
|
588
|
+
return vacuousRefusal;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
// ═══ SCHEMA VALIDATION ════════════════════════════════════════════════
|
|
592
|
+
describe("schema validation", () => {
|
|
593
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
594
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
test("workflow-state.json validates (includes threshold + behavioral superRefine)", () => {
|
|
598
|
+
const raw = loadRaw();
|
|
599
|
+
const result = WorkflowStateSchema.passthrough().safeParse(raw);
|
|
600
|
+
if (!result.success) {
|
|
601
|
+
const issues = result.error.issues.map(i => `${i.path.join(".")}: ${i.message}`);
|
|
602
|
+
// Narrowed to `false` inside this branch — widen so the always-fail
|
|
603
|
+
// assertion still reports the collected issues as its message.
|
|
604
|
+
expect<boolean>(result.success, issues.join("\n")).toBe(true);
|
|
605
|
+
}
|
|
606
|
+
});
|
|
607
|
+
});
|
|
608
|
+
|
|
609
|
+
// ═══ SCOPE CHECKS (11 functions) ══════════════════════════════════════
|
|
610
|
+
describe("scope checks", () => {
|
|
611
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
612
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
// 1. acs-exist
|
|
616
|
+
test("acs-exist: at least one AC", () => {
|
|
617
|
+
expect(sf("acs")?.length ?? 0).toBeGreaterThan(0);
|
|
618
|
+
});
|
|
619
|
+
|
|
620
|
+
// 4. acs-have-evidence-method
|
|
621
|
+
test("acs-have-evidence-method", () => {
|
|
622
|
+
const missing = (sf("acs") || []).filter((ac: any) => !ac.evidenceMethod);
|
|
623
|
+
expect(missing.map((ac: any) => ac.id)).toEqual([]);
|
|
624
|
+
});
|
|
625
|
+
|
|
626
|
+
// 5. acs-measurable
|
|
627
|
+
test("acs-measurable: thresholds have op and value", () => {
|
|
628
|
+
const bad = (sf("acs") || []).filter((ac: any) => ac.threshold && (!ac.threshold.op || ac.threshold.value == null));
|
|
629
|
+
expect(bad.map((ac: any) => ac.id)).toEqual([]);
|
|
630
|
+
});
|
|
631
|
+
|
|
632
|
+
// 6. source-spec-cited (GAP → PORTED)
|
|
633
|
+
test("source-spec-cited: all sourceSpecs have citedInDiscovery=true", () => {
|
|
634
|
+
const specs = sf("sourceSpecs") || [];
|
|
635
|
+
const uncited = specs.filter((s: any) => s.citedInDiscovery === false);
|
|
636
|
+
expect(uncited.map((s: any) => s.path)).toEqual([]);
|
|
637
|
+
});
|
|
638
|
+
|
|
639
|
+
// 7. spec-elements-in-acs (GAP → PORTED)
|
|
640
|
+
test("spec-elements-in-acs: if specElements exist, at least one AC references them", () => {
|
|
641
|
+
const specs = sf("sourceSpecs") || [];
|
|
642
|
+
const elements = specs.flatMap((s: any) => s.specElements || []);
|
|
643
|
+
if (elements.length === 0) return;
|
|
644
|
+
const acsWithSpec = (sf("acs") || []).filter((ac: any) => ac.specElement);
|
|
645
|
+
expect(acsWithSpec.length).toBeGreaterThan(0);
|
|
646
|
+
});
|
|
647
|
+
|
|
648
|
+
// 8. sizing-declared
|
|
649
|
+
test("sizing-declared", () => {
|
|
650
|
+
expect(sf("sizing")?.predicted).toBeDefined();
|
|
651
|
+
});
|
|
652
|
+
|
|
653
|
+
// 9. issue-goal-captured
|
|
654
|
+
test("issue-goal-captured", () => {
|
|
655
|
+
expect((sf("issueGoal") || "").length).toBeGreaterThan(0);
|
|
656
|
+
});
|
|
657
|
+
|
|
658
|
+
// 10. acs-no-skillmd-enforcement (GAP → PORTED)
|
|
659
|
+
test("acs-no-skillmd-enforcement: no ACs target SKILL.md", () => {
|
|
660
|
+
const pattern = /SKILL\.md/i;
|
|
661
|
+
const hits = (sf("acs") || []).filter((ac: any) =>
|
|
662
|
+
pattern.test(ac.statement || "") || pattern.test(ac.evidenceMethod?.command || "")
|
|
663
|
+
);
|
|
664
|
+
expect(hits.map((ac: any) => ac.id)).toEqual([]);
|
|
665
|
+
});
|
|
666
|
+
|
|
667
|
+
// ac-issue-alignment: ACs cover issueGoal keywords (ADR-009 C4)
|
|
668
|
+
//
|
|
669
|
+
// #423: stays a WARNING, and that is now a decision with a number behind it
|
|
670
|
+
// rather than a deferral. Measured over the 153 workflow states under
|
|
671
|
+
// `~/.rungate` carrying a goal, 71% fall under this threshold — 66% of the
|
|
672
|
+
// 131 whose ACs carry real statements, 73% of the 101 with substantial ones.
|
|
673
|
+
// A check that would refuse two runs in three is an outage, not a gate. The
|
|
674
|
+
// reading moved to `goalKeywordCoverage` so the threshold and the vocabulary
|
|
675
|
+
// can be driven from both sides by a test; what did NOT change is that
|
|
676
|
+
// nothing here refuses.
|
|
677
|
+
test("ac-issue-alignment: ACs cover goal keywords", () => {
|
|
678
|
+
const read = goalKeywordCoverage(sf("issueGoal"), sf("acs"));
|
|
679
|
+
if (read === null) return;
|
|
680
|
+
if (read.coverage < GOAL_COVERAGE_WARN_BELOW) {
|
|
681
|
+
console.warn(
|
|
682
|
+
`WARN: ac-issue-alignment: only ${Math.round(read.coverage * 100)}% goal keyword coverage ` +
|
|
683
|
+
`(${read.covered.length}/${read.keywords.length})`,
|
|
684
|
+
);
|
|
685
|
+
}
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
// 11. sourceSpecs-required (#473)
|
|
689
|
+
test("sourceSpecs-required: non-LIGHT tier must have sourceSpecs", () => {
|
|
690
|
+
const tier = sf("sizing.ceremonyTier");
|
|
691
|
+
if (tier === "LIGHT") return;
|
|
692
|
+
const specs = sf("sourceSpecs") || [];
|
|
693
|
+
expect(specs.length, "STANDARD/THOROUGH tier must have at least one sourceSpec").toBeGreaterThanOrEqual(1);
|
|
694
|
+
});
|
|
695
|
+
|
|
696
|
+
// 11b. before-state-captured (#1411)
|
|
697
|
+
test("before-state-captured: STANDARD+ must have beforeState in workflow-state", () => {
|
|
698
|
+
const tier = sf("sizing.ceremonyTier");
|
|
699
|
+
const VALID_TYPES = ["text", "screenshot", "api", "none"];
|
|
700
|
+
const bs = sf("beforeState");
|
|
701
|
+
if (!bs) {
|
|
702
|
+
if (tier === "LIGHT") return;
|
|
703
|
+
console.warn("WARN: before-state-captured: beforeState missing for STANDARD+ tier — goal skill should capture before-state (#1411)");
|
|
704
|
+
return;
|
|
705
|
+
}
|
|
706
|
+
expect(VALID_TYPES.includes(bs.type), `beforeState.type must be text|screenshot|api|none, got "${bs.type}"`).toBe(true);
|
|
707
|
+
if (tier !== "LIGHT" && bs.type === "none") {
|
|
708
|
+
console.warn('WARN: before-state-captured: beforeState.type is "none" for STANDARD+ tier — consider capturing actual before-state');
|
|
709
|
+
}
|
|
710
|
+
});
|
|
711
|
+
|
|
712
|
+
// 12. discovery-evidence (#473)
|
|
713
|
+
test("discovery-evidence: non-LIGHT tiers have cited sourceSpecs", () => {
|
|
714
|
+
const tier = sf("sizing.ceremonyTier");
|
|
715
|
+
if (tier === "LIGHT") return;
|
|
716
|
+
const specs = sf("sourceSpecs") || [];
|
|
717
|
+
const cited = specs.filter((s: any) => s.citedInDiscovery);
|
|
718
|
+
expect(cited.length, "No sourceSpecs with citedInDiscovery — DISCOVERY may have been skipped").toBeGreaterThanOrEqual(1);
|
|
719
|
+
});
|
|
720
|
+
|
|
721
|
+
// 13. governing-spec-exists (#473)
|
|
722
|
+
test("governing-spec-exists: spec file exists on disk", () => {
|
|
723
|
+
const tier = sf("sizing.ceremonyTier");
|
|
724
|
+
if (tier === "LIGHT") return;
|
|
725
|
+
const specs = sf("sourceSpecs") || [];
|
|
726
|
+
if (specs.length === 0) return; // caught by sourceSpecs-required
|
|
727
|
+
const specPath = specs[0].path;
|
|
728
|
+
// #314 — the spec is resolved against the TREE UNDER TEST. A spec added on
|
|
729
|
+
// the branch does not exist in the main checkout, so resolving against
|
|
730
|
+
// projectRoot failed a run for writing its own governing spec; a spec
|
|
731
|
+
// deleted on the branch still exists in main, so the same read passed a
|
|
732
|
+
// run that had removed the thing it claimed to be governed by.
|
|
733
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "governing-spec-exists");
|
|
734
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
735
|
+
const root = treeRoot();
|
|
736
|
+
const resolved = specPath.startsWith("/") ? specPath : join(root!, specPath);
|
|
737
|
+
expect(existsSync(resolved), `Governing spec not found: ${resolved}`).toBe(true);
|
|
738
|
+
});
|
|
739
|
+
|
|
740
|
+
// 14. harness-port-validation: rungate.json ports match Makefile
|
|
741
|
+
test("harness-port-validation: dev/test/prod ports match Makefile", () => {
|
|
742
|
+
// #314 — this check's SUBJECT is whether two files in the tree agree, so
|
|
743
|
+
// it reads both from the tree under test. It is not a config read: a
|
|
744
|
+
// branch that edits a port in either file would otherwise be graded
|
|
745
|
+
// against the main checkout's copies, where the two already agree.
|
|
746
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "harness-port-validation");
|
|
747
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
748
|
+
const projectRoot = treeRoot();
|
|
749
|
+
if (!projectRoot) return;
|
|
750
|
+
const harnessPath = join(projectRoot, ".claude", "rungate.json");
|
|
751
|
+
const makefilePath = join(projectRoot, "Makefile");
|
|
752
|
+
if (!existsSync(harnessPath) || !existsSync(makefilePath)) return;
|
|
753
|
+
// Named treeHarness, not harness: `harness` means the loadProjectHarness()
|
|
754
|
+
// result everywhere else in this file, and test/gate-optional-config-keys.test.ts
|
|
755
|
+
// enumerates reads off it so no gate check can gate on an unexplained config
|
|
756
|
+
// key. A second const with the same name reading a different file would be
|
|
757
|
+
// invisible to that scan — which is how the scan was first written, and what
|
|
758
|
+
// the security review on #315 caught.
|
|
759
|
+
const treeHarness = JSON.parse(readFileSync(harnessPath, "utf-8"));
|
|
760
|
+
const makefile = readFileSync(makefilePath, "utf-8");
|
|
761
|
+
|
|
762
|
+
const warnings: string[] = [];
|
|
763
|
+
|
|
764
|
+
// Check dev port — match PORT= in the dev-all/dev section of Makefile
|
|
765
|
+
const devPort = treeHarness?.dev?.apiBase?.match(/:(\d+)/)?.[1];
|
|
766
|
+
if (devPort) {
|
|
767
|
+
const devSection = makefile.match(/dev.*environment[^]*?PORT=(\d+)/i);
|
|
768
|
+
const makeDevPort = devSection?.[1];
|
|
769
|
+
if (makeDevPort && devPort !== makeDevPort) {
|
|
770
|
+
warnings.push(`dev.apiBase port ${devPort} != Makefile dev PORT ${makeDevPort}`);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
// Check prod port — match PORT= in container run commands
|
|
775
|
+
const prodPort = treeHarness?.prod?.apiBase?.match(/:(\d+)/)?.[1];
|
|
776
|
+
if (prodPort) {
|
|
777
|
+
const makeProdPort = makefile.match(/-e PORT=(\d+)/)?.[1];
|
|
778
|
+
if (makeProdPort && prodPort !== makeProdPort) {
|
|
779
|
+
warnings.push(`prod.apiBase port ${prodPort} != Makefile container PORT ${makeProdPort}`);
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
if (warnings.length > 0) {
|
|
784
|
+
console.warn(`WARN: harness-port-validation: ${warnings.join("; ")}`);
|
|
785
|
+
}
|
|
786
|
+
});
|
|
787
|
+
});
|
|
788
|
+
|
|
789
|
+
// ═══ VERIFY CHECKS (16 functions) ═════════════════════════════════════
|
|
790
|
+
describe("verify checks", () => {
|
|
791
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
792
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
793
|
+
return;
|
|
794
|
+
}
|
|
795
|
+
// 12. local-before-prod
|
|
796
|
+
test("local-before-prod: local validated before prod rebuild", () => {
|
|
797
|
+
const prodRebuild = sf("environments")?.prod?.rebuild;
|
|
798
|
+
if (!prodRebuild || prodRebuild === "SKIP") return;
|
|
799
|
+
const localApi = sf("environments")?.local?.api;
|
|
800
|
+
const localTests = sf("environments")?.local?.tests;
|
|
801
|
+
if (localApi !== "PASS" && localTests !== "PASS") {
|
|
802
|
+
expect("prod deployed without local validation").toBe("local.api or local.tests = PASS first");
|
|
803
|
+
}
|
|
804
|
+
});
|
|
805
|
+
|
|
806
|
+
// 13. container-has-fix — OCI label SHA comparison (#437)
|
|
807
|
+
test("container-has-fix: container image SHA matches HEAD", () => {
|
|
808
|
+
if (!isShipPlus()) return;
|
|
809
|
+
const harness = loadProjectHarness();
|
|
810
|
+
if (!harness?.prod?.rebuild) return;
|
|
811
|
+
const prodRebuild = sf("environments")?.prod?.rebuild;
|
|
812
|
+
// #314: the container SHA is compared against the tree under test's HEAD, not main's.
|
|
813
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "container-has-fix");
|
|
814
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
815
|
+
const root = treeRoot();
|
|
816
|
+
if (!root) return;
|
|
817
|
+
const containerName = harness?.prod?.containerName || harness?.prod?.imageName;
|
|
818
|
+
if (!containerName) return;
|
|
819
|
+
|
|
820
|
+
// #341: four early returns used to share one observable outcome — PASS.
|
|
821
|
+
// Container absent, daemon down, no revision label, no runtime installed.
|
|
822
|
+
// Two of them are MORE likely exactly when the check matters, so it was
|
|
823
|
+
// quietest in the situation it exists to catch.
|
|
824
|
+
//
|
|
825
|
+
// Each is now a named outcome, recorded. Which of them should BLOCK is
|
|
826
|
+
// #341 criterion 1's question — it needs the frequency of each measured on a
|
|
827
|
+
// real consumer first, because refusing on a runtime that happens to be
|
|
828
|
+
// down turns real runs red. Only a measured BEHIND blocks today, which
|
|
829
|
+
// is what blocked before.
|
|
830
|
+
const runtime = resolveContainerRuntime(
|
|
831
|
+
harness?.prod?.containerRuntime,
|
|
832
|
+
name => exec(`command -v ${name}`).ok,
|
|
833
|
+
);
|
|
834
|
+
// Argv form: containerName comes out of the consumer's rungate.json and
|
|
835
|
+
// the old line interpolated it into a shell command (#315 security review).
|
|
836
|
+
const inspect =
|
|
837
|
+
runtime.kind === "resolved"
|
|
838
|
+
? execArgvOf(runtime.runtime, [
|
|
839
|
+
"inspect",
|
|
840
|
+
"--format={{index .Config.Labels \"org.opencontainers.image.revision\"}}",
|
|
841
|
+
"--",
|
|
842
|
+
containerName,
|
|
843
|
+
])
|
|
844
|
+
: undefined;
|
|
845
|
+
const head = exec("git rev-parse HEAD", root);
|
|
846
|
+
|
|
847
|
+
const verdict = classifyContainerProvenance({
|
|
848
|
+
runtime,
|
|
849
|
+
containerName,
|
|
850
|
+
// execArgvOf folds stderr into `output` on failure, which is the text
|
|
851
|
+
// classifyInspectFailure needs to tell absence from a dead daemon.
|
|
852
|
+
inspect: inspect ? { ok: inspect.ok, output: inspect.ok ? inspect.output : "", stderr: inspect.ok ? "" : inspect.output } : undefined,
|
|
853
|
+
head: { ok: head.ok, output: head.output },
|
|
854
|
+
prodRebuild,
|
|
855
|
+
});
|
|
856
|
+
|
|
857
|
+
// criterion 3: surfaced, not swallowed. One line per run naming which of the
|
|
858
|
+
// outcomes was reached, so criterion 1's frequencies are collectable from the
|
|
859
|
+
// logs of real runs rather than needing new instrumentation later.
|
|
860
|
+
console.log(
|
|
861
|
+
`container-has-fix: ${verdict.outcome} (${verdict.measured ? "measured" : "UNMEASURED"}) — ${verdict.detail}`,
|
|
862
|
+
);
|
|
863
|
+
|
|
864
|
+
const violations = containerProvenanceViolations(verdict);
|
|
865
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
866
|
+
});
|
|
867
|
+
|
|
868
|
+
// 14. all-acs-have-evidence (GAP → PORTED)
|
|
869
|
+
// OUTCOME ACs may not have mechanical evidence at ship time — verified at prove via Quinn
|
|
870
|
+
test("all-acs-have-evidence: every AC has evidence in verify+", () => {
|
|
871
|
+
if (!isVerifyPlus()) return;
|
|
872
|
+
const missing = (sf("acs") || []).filter((ac: any) => ac.evidence == null && ac.type !== "OUTCOME");
|
|
873
|
+
expect(missing.map((ac: any) => ac.id)).toEqual([]);
|
|
874
|
+
});
|
|
875
|
+
|
|
876
|
+
// 15. all-acs-pass
|
|
877
|
+
// OUTCOME ACs may be PENDING at ship time — they require Quinn/prove for verification
|
|
878
|
+
test("all-acs-pass: every AC has verdict in ship+", () => {
|
|
879
|
+
if (!isShipPlus()) return;
|
|
880
|
+
const noVerdict = (sf("acs") || []).filter((ac: any) => (!ac.verdict || ac.verdict === "PENDING") && ac.type !== "OUTCOME");
|
|
881
|
+
expect(noVerdict.map((ac: any) => ac.id)).toEqual([]);
|
|
882
|
+
});
|
|
883
|
+
|
|
884
|
+
// 15b. outcome-acs-no-skip: OUTCOME ACs must be verified, not skipped (#90 RCA)
|
|
885
|
+
test("outcome-acs-no-skip: OUTCOME ACs cannot have SKIP verdict at ship+", () => {
|
|
886
|
+
if (!isShipPlus()) return;
|
|
887
|
+
const skipped = (sf("acs") || []).filter((ac: any) => ac.type === "OUTCOME" && ac.verdict === "SKIP");
|
|
888
|
+
expect(skipped.map((ac: any) => ac.id), "OUTCOME ACs require live verification — start dev server and run the feature (#90 RCA)").toEqual([]);
|
|
889
|
+
});
|
|
890
|
+
|
|
891
|
+
// 15c. outcome-evidence-behavioral: OUTCOME evidence is not just a grep count (#90 RCA)
|
|
892
|
+
//
|
|
893
|
+
// #423: this REFUSES now. It was `tsc-pass`'s shape — a name promising a
|
|
894
|
+
// verdict over a body that could only `console.warn` — while its sibling
|
|
895
|
+
// two tests above it has always asserted. An OUTCOME AC records that a
|
|
896
|
+
// user-visible outcome was observed, and a bare number cannot establish
|
|
897
|
+
// one; `0` is indistinguishable from success, which is the anti-pattern
|
|
898
|
+
// `.claude/rules/config-access.md` names as Silent Default Substitution.
|
|
899
|
+
test("outcome-evidence-behavioral: OUTCOME AC evidence is not just a number", () => {
|
|
900
|
+
if (!isShipPlus()) return;
|
|
901
|
+
const violations = outcomeEvidenceViolations(sf("acs"));
|
|
902
|
+
expect(
|
|
903
|
+
violations,
|
|
904
|
+
"OUTCOME ACs need evidence a reader can check — a bare measurement says a number " +
|
|
905
|
+
"was produced, not that the outcome happened (#90 RCA, #423)",
|
|
906
|
+
).toEqual([]);
|
|
907
|
+
});
|
|
908
|
+
|
|
909
|
+
// #339: renamed from the prose title "no FAIL verdicts in DONE". A gate
|
|
910
|
+
// check with a prose name cannot be referenced from a ceremony profile at
|
|
911
|
+
// all, so it was structurally undocumentable rather than merely
|
|
912
|
+
// undocumented. Referenced by nothing else; the rename is local.
|
|
913
|
+
test("no-fail-verdicts-in-done: no AC is FAIL once the run is DONE", () => {
|
|
914
|
+
if (!isDone()) return;
|
|
915
|
+
const fails = (sf("acs") || []).filter((ac: any) => ac.verdict === "FAIL");
|
|
916
|
+
expect(fails.map((ac: any) => ac.id)).toEqual([]);
|
|
917
|
+
});
|
|
918
|
+
|
|
919
|
+
// 16. evidence-type-valid (GAP → PORTED)
|
|
920
|
+
test("evidence-type-valid: evidence.type matches allowed enum", () => {
|
|
921
|
+
if (!isVerifyPlus()) return;
|
|
922
|
+
const VALID = ["file-citation", "grep-output", "api-response", "screenshot", "test-output", "command-output", "manual-attestation", "grep"];
|
|
923
|
+
const bad = (sf("acs") || []).filter((ac: any) => {
|
|
924
|
+
if (!ac.evidence || typeof ac.evidence !== "object") return false;
|
|
925
|
+
return !VALID.includes(ac.evidence.type);
|
|
926
|
+
});
|
|
927
|
+
expect(bad.map((ac: any) => `${ac.id}: ${ac.evidence?.type}`)).toEqual([]);
|
|
928
|
+
});
|
|
929
|
+
|
|
930
|
+
// 17. evidence-type-ratio (GAP → PORTED, ADR-009 A5: >= 50% behavioral evidence required)
|
|
931
|
+
test("evidence-type-ratio: <50% pattern-only evidence (ADR-009 A5)", () => {
|
|
932
|
+
if (!isVerifyPlus()) return;
|
|
933
|
+
const acs = (sf("acs") || []).filter((ac: any) => ac.evidenceMethod);
|
|
934
|
+
if (acs.length === 0) return;
|
|
935
|
+
const patternOnly = acs.filter((ac: any) => {
|
|
936
|
+
const t = ac.evidenceMethod?.type;
|
|
937
|
+
return !t || t === "pattern" || t === "GREP_CHECK" || t === "grep";
|
|
938
|
+
});
|
|
939
|
+
const ratio = patternOnly.length / acs.length;
|
|
940
|
+
// ADR-009 A5: at least 50% behavioral evidence required
|
|
941
|
+
expect(ratio, `${patternOnly.length}/${acs.length} (${Math.round(ratio * 100)}%) pattern-only — must be < 50% (ADR-009 A5)`).toBeLessThanOrEqual(0.5);
|
|
942
|
+
});
|
|
943
|
+
|
|
944
|
+
// 18. evidence-method-executed (GAP → PORTED)
|
|
945
|
+
test("evidence-method-executed: ACs with a threshold have evidence recorded", () => {
|
|
946
|
+
if (!isVerifyPlus()) return;
|
|
947
|
+
const violations = missingEvidenceViolations(sf("acs"));
|
|
948
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
949
|
+
});
|
|
950
|
+
|
|
951
|
+
// 19. threshold-crosscheck (GAP → PORTED)
|
|
952
|
+
test("threshold-crosscheck: evidence values meet threshold", () => {
|
|
953
|
+
if (!isVerifyPlus()) return;
|
|
954
|
+
const violations = thresholdCrosscheckViolations(sf("acs"));
|
|
955
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
956
|
+
});
|
|
957
|
+
|
|
958
|
+
// 20. tests-pass (GAP → PORTED)
|
|
959
|
+
//
|
|
960
|
+
// #224: run wf_7ac5f614-d21 returned `regressions: 0` on a branch running
|
|
961
|
+
// 3984 pass / 1 fail. The suite reading and the verdict were not connected,
|
|
962
|
+
// and an unmeasured suite read as a passing one. The conversion from reading
|
|
963
|
+
// to refusal now lives in `suiteVerdictViolations` above — one function, one
|
|
964
|
+
// body — so test/suite-binding-mutation.test.ts can remove it from a mutant
|
|
965
|
+
// copy of this file and watch the mutant ship a fixture whose failing test
|
|
966
|
+
// was really run. What was broken to prove it: short-circuiting that
|
|
967
|
+
// function to `return []` turns the gate green on a measured FAIL.
|
|
968
|
+
test("tests-pass: test suite passes", () => {
|
|
969
|
+
if (!isVerifyPlus()) return;
|
|
970
|
+
const harness = loadProjectHarness();
|
|
971
|
+
if (!harness?.dev?.testCmd) return;
|
|
972
|
+
// Both halves of one fact, read from one state (#530). The enum is what
|
|
973
|
+
// this check has always read; the measurement is what produced it, and a
|
|
974
|
+
// PASS with nothing behind it is the case the enum alone cannot see.
|
|
975
|
+
const violations = suiteVerdictViolations(
|
|
976
|
+
sf("environments")?.local?.tests,
|
|
977
|
+
readSuiteMeasurement(loadRaw()),
|
|
978
|
+
);
|
|
979
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
980
|
+
});
|
|
981
|
+
|
|
982
|
+
// 20b. evidence-prevalidation (#235)
|
|
983
|
+
//
|
|
984
|
+
// The pre-validator already dry-runs every AC evidence command and prints
|
|
985
|
+
// which ones are broken. Nothing read that print. The conversion from
|
|
986
|
+
// reading to refusal lives in `prevalidationViolations` above — one
|
|
987
|
+
// function, one body — so test/suite-binding-mutation.test.ts can remove it
|
|
988
|
+
// from a mutant copy of this file and watch the mutant ship a fixture whose
|
|
989
|
+
// broken evidence command was really dry-run. What was broken to prove it:
|
|
990
|
+
// short-circuiting `prevalidateEvidence` to `return []` makes the fixture
|
|
991
|
+
// measure PASS, and short-circuiting this function makes the gate green on a
|
|
992
|
+
// measured FAIL.
|
|
993
|
+
test("evidence-prevalidation: AC evidence commands pre-validated before ship", () => {
|
|
994
|
+
if (!isVerifyPlus()) return;
|
|
995
|
+
// An AC with no evidence command has no command that can be broken, so a
|
|
996
|
+
// run with none of them has nothing to pre-validate. That is a real early
|
|
997
|
+
// return, like tsc-pass's missing tsconfig — not a fail-open, because the
|
|
998
|
+
// pre-validator classifies exactly those ACs `skipped` itself.
|
|
999
|
+
const acs = sf("acs") || [];
|
|
1000
|
+
if (!acs.some((ac: any) => ac?.evidenceMethod?.command)) return;
|
|
1001
|
+
const violations = prevalidationViolations(sf("evidencePrevalidation"), sf("phase"));
|
|
1002
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
1003
|
+
});
|
|
1004
|
+
|
|
1005
|
+
// 21. tsc-pass (GAP → PORTED)
|
|
1006
|
+
//
|
|
1007
|
+
// #176: this was two early returns and an empty tail — no expect, no tsc
|
|
1008
|
+
// invocation, no baseline comparison. The second return read
|
|
1009
|
+
// `environments.local.tests` and bailed with a comment deferring to the
|
|
1010
|
+
// engineer, so the gate asked the agent it was gating whether it needed to
|
|
1011
|
+
// run. No gate defers to its subject. It now runs the
|
|
1012
|
+
// compiler and compares the count against the project's declared baseline.
|
|
1013
|
+
// What was broken to prove it fails: a project with one bad assignment and
|
|
1014
|
+
// `.claude/typecheck-baseline.json` at 0 — see
|
|
1015
|
+
// test/gate-vacuous-checks.test.ts, which plants exactly that and watches
|
|
1016
|
+
// this go red, then removes it and watches it go green.
|
|
1017
|
+
test("tsc-pass: no new TypeScript errors above baseline", () => {
|
|
1018
|
+
if (!isVerifyPlus()) return;
|
|
1019
|
+
// #314: the compiler runs on the TREE UNDER TEST. Running it in
|
|
1020
|
+
// projectRoot typechecked the main checkout, which passes by construction,
|
|
1021
|
+
// and let two real type errors through to CI on run wf_5cdce852-1e2.
|
|
1022
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "tsc-pass");
|
|
1023
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
1024
|
+
const root = treeRoot();
|
|
1025
|
+
// No tsconfig means there is nothing for tsc to check; that is a real
|
|
1026
|
+
// early return, not a skipped measurement.
|
|
1027
|
+
if (!root || !existsSync(join(root, "tsconfig.json"))) return;
|
|
1028
|
+
|
|
1029
|
+
const baselineFile = join(root, ".claude", "typecheck-baseline.json");
|
|
1030
|
+
const baselineRaw = existsSync(baselineFile) ? readFileSync(baselineFile, "utf-8") : null;
|
|
1031
|
+
|
|
1032
|
+
const run = spawnSync("bunx", ["tsc", "--noEmit"], {
|
|
1033
|
+
cwd: root,
|
|
1034
|
+
encoding: "utf-8",
|
|
1035
|
+
timeout: TYPECHECK_TIMEOUT_MS,
|
|
1036
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
1037
|
+
});
|
|
1038
|
+
const failures = run.error
|
|
1039
|
+
? [`could not run tsc: ${run.error.message}`]
|
|
1040
|
+
: typecheckFailures(
|
|
1041
|
+
{ status: run.status, output: `${run.stdout ?? ""}${run.stderr ?? ""}` },
|
|
1042
|
+
baselineRaw,
|
|
1043
|
+
);
|
|
1044
|
+
expect(failures, failures.join("\n")).toEqual([]);
|
|
1045
|
+
});
|
|
1046
|
+
|
|
1047
|
+
// 22. code-committed (GAP → PORTED)
|
|
1048
|
+
test("code-committed: no uncommitted changes in project", () => {
|
|
1049
|
+
if (!isShipPlus()) return;
|
|
1050
|
+
// #314: `git status` must run in the tree holding the work. In projectRoot
|
|
1051
|
+
// it inspected the main checkout, which is clean during a worktree run, so
|
|
1052
|
+
// this check — whose whole job is confirming the work is committed —
|
|
1053
|
+
// passed unconditionally on every run this repo has made.
|
|
1054
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "code-committed");
|
|
1055
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
1056
|
+
const root = treeRoot();
|
|
1057
|
+
if (!root) return;
|
|
1058
|
+
// #315: an unset `codeCommittedPaths` used to early-return here, and
|
|
1059
|
+
// nothing authors that key, so this check had never run. The whole tree
|
|
1060
|
+
// is the default scope; a configured list is a narrowing, not a switch.
|
|
1061
|
+
const harness = loadProjectHarness();
|
|
1062
|
+
const paths: string[] = harness?.codeCommittedPaths || [];
|
|
1063
|
+
const configured = paths.length > 0;
|
|
1064
|
+
// Argv form: `paths` comes out of the consumer's own rungate.json and
|
|
1065
|
+
// `exec` goes through a shell. The old line interpolated it straight
|
|
1066
|
+
// into one.
|
|
1067
|
+
const result = execArgv(
|
|
1068
|
+
["status", "--porcelain", ...(configured ? ["--", ...paths] : [])],
|
|
1069
|
+
root,
|
|
1070
|
+
);
|
|
1071
|
+
const violations = codeCommittedViolations(result, { configured, paths });
|
|
1072
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
1073
|
+
});
|
|
1074
|
+
|
|
1075
|
+
// 22b. worktree-repo (#414, follow-up to #218)
|
|
1076
|
+
test("worktree-repo: every worktree is a checkout of this run's repository", () => {
|
|
1077
|
+
if (!isShipPlus()) return;
|
|
1078
|
+
// The entries, not just `treeUnderTest`: a decomposed run has one
|
|
1079
|
+
// worktree per sub-issue and #218's incident was a SECOND agent landing
|
|
1080
|
+
// in the wrong checkout while the first was correct. Checking only the
|
|
1081
|
+
// tree under test would read the right answer off the wrong worktree.
|
|
1082
|
+
//
|
|
1083
|
+
// `treeUnderTest` is included because it is the tree every other check in
|
|
1084
|
+
// this file measures — if it is a worktree of another repository then
|
|
1085
|
+
// tsc-pass, code-committed and code-pushed all measured that repository,
|
|
1086
|
+
// and said PASS.
|
|
1087
|
+
const named = Array.isArray(sf("worktrees")) ? sf("worktrees") : [];
|
|
1088
|
+
const tree = sf("treeUnderTest");
|
|
1089
|
+
const entries = [
|
|
1090
|
+
...(typeof tree === "string" && tree.trim() ? [{ path: tree }] : []),
|
|
1091
|
+
...named,
|
|
1092
|
+
];
|
|
1093
|
+
const violations = worktreeRepoViolations(entries, sf("repo"));
|
|
1094
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
1095
|
+
});
|
|
1096
|
+
|
|
1097
|
+
// 23. code-pushed (GAP → PORTED)
|
|
1098
|
+
test("code-pushed: no unpushed commits", () => {
|
|
1099
|
+
if (!isShipPlus()) return;
|
|
1100
|
+
// #314: in projectRoot this asked whether `main` was ahead of its own
|
|
1101
|
+
// upstream, which it is not, so the answer was always 0. #174 filed the
|
|
1102
|
+
// same defect against this check independently.
|
|
1103
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "code-pushed");
|
|
1104
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
1105
|
+
const root = treeRoot();
|
|
1106
|
+
if (!root) return;
|
|
1107
|
+
const count = exec("git rev-list --count @{upstream}..HEAD", root);
|
|
1108
|
+
const upstream = exec("git rev-parse --abbrev-ref --symbolic-full-name @{upstream}", root);
|
|
1109
|
+
const violations = unpushedViolations(count, upstream);
|
|
1110
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
1111
|
+
});
|
|
1112
|
+
|
|
1113
|
+
// 25. local-api-validated (GAP → PORTED)
|
|
1114
|
+
test("local-api-validated: environments.local.api set", () => {
|
|
1115
|
+
if (!isVerifyPlus()) return;
|
|
1116
|
+
const harness = loadProjectHarness();
|
|
1117
|
+
if (!harness?.dev?.apiBase) return;
|
|
1118
|
+
const val = sf("environments")?.local?.api;
|
|
1119
|
+
if (!val) {
|
|
1120
|
+
expect("local.api not set").toBe("PASS or SKIP — required when dev.apiBase is configured");
|
|
1121
|
+
} else if (val === "SKIP") {
|
|
1122
|
+
// SKIP is allowed
|
|
1123
|
+
} else if (val !== "PASS") {
|
|
1124
|
+
expect(`api: ${val}`).toBe("PASS");
|
|
1125
|
+
}
|
|
1126
|
+
});
|
|
1127
|
+
|
|
1128
|
+
// 26. local-ui-validated (GAP → PORTED)
|
|
1129
|
+
test("local-ui-validated: environments.local.ui set with skipReason", () => {
|
|
1130
|
+
if (!isVerifyPlus()) return;
|
|
1131
|
+
const harness = loadProjectHarness();
|
|
1132
|
+
if (!harness?.dev?.uiBase) return;
|
|
1133
|
+
const val = sf("environments")?.local?.ui;
|
|
1134
|
+
if (!val) {
|
|
1135
|
+
expect("local.ui not set").toBe("PASS or SKIP — required when dev.uiBase is configured");
|
|
1136
|
+
} else if (val === "SKIP") {
|
|
1137
|
+
const reason = sf("environments")?.local?.uiSkipReason;
|
|
1138
|
+
if (!reason) {
|
|
1139
|
+
expect("UI SKIP without reason").toBe("uiSkipReason required when SKIP");
|
|
1140
|
+
}
|
|
1141
|
+
} else if (val !== "PASS") {
|
|
1142
|
+
expect(`ui: ${val}`).toBe("PASS");
|
|
1143
|
+
}
|
|
1144
|
+
});
|
|
1145
|
+
|
|
1146
|
+
// 29. quinn-on-ui-change
|
|
1147
|
+
test("quinn-on-ui-change: quinn spawned when UI files change", () => {
|
|
1148
|
+
if (!isShipPlus()) return;
|
|
1149
|
+
const tier = sf("sizing")?.ceremonyTier;
|
|
1150
|
+
if (tier === "LIGHT") return;
|
|
1151
|
+
const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
|
|
1152
|
+
const UI_SVC = /template|campaign-html|brief-pipeline|email-template|dashboard/i;
|
|
1153
|
+
const allPaths: string[] = [];
|
|
1154
|
+
for (const ac of sf("acs") || []) {
|
|
1155
|
+
if (ac.statement) allPaths.push(ac.statement);
|
|
1156
|
+
if (ac.evidenceMethod?.command) allPaths.push(ac.evidenceMethod.command);
|
|
1157
|
+
for (const cf of ac.contextFiles || []) {
|
|
1158
|
+
if (cf?.path) allPaths.push(cf.path);
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
// Strip absolute project paths to avoid false positives (e.g. /DailyBriefDashboard/ matching "dashboard")
|
|
1162
|
+
//
|
|
1163
|
+
// #314 — BOTH roots, not just projectRoot. The AC paths this reads are
|
|
1164
|
+
// written while the run is in a worktree, so they carry the tree-under-test
|
|
1165
|
+
// path; stripping only projectRoot left that path in the string and handed
|
|
1166
|
+
// the UI_SVC regex the directory name it exists to not match. This is the
|
|
1167
|
+
// one site that legitimately wants projectRoot alongside the tree, because
|
|
1168
|
+
// it is sanitising text rather than measuring a tree.
|
|
1169
|
+
const joined = [sf("projectRoot"), sf("treeUnderTest")]
|
|
1170
|
+
.filter((r): r is string => typeof r === "string" && r !== "")
|
|
1171
|
+
.reduce((text, root) => text.replaceAll(root, ""), allPaths.join(" "));
|
|
1172
|
+
if (!UI_EXT.test(joined) && !UI_SVC.test(joined)) return;
|
|
1173
|
+
const quinn = sf("agents")?.quinn;
|
|
1174
|
+
if (!quinn?.spawned) {
|
|
1175
|
+
expect("quinn not spawned for UI change").toBe("quinn.spawned required");
|
|
1176
|
+
}
|
|
1177
|
+
// Quinn depth check: spawned but no testedPaths = code review only (ADR-009 A3)
|
|
1178
|
+
// If Quinn has verdict PASS, accept that as sufficient evidence (ship.js doesn't write testedPaths yet)
|
|
1179
|
+
if (quinn?.spawned && quinn.verdict !== 'PASS') {
|
|
1180
|
+
const testedPaths = quinn.testedPaths;
|
|
1181
|
+
expect(Array.isArray(testedPaths) && testedPaths.length > 0, "quinn.testedPaths must not be empty — code review only does not satisfy UI testing (ADR-009 A3)").toBe(true);
|
|
1182
|
+
}
|
|
1183
|
+
// Quinn screenshot check: WARN when verdict=PASS but no screenshots (ship.js Quinn doesn't capture yet)
|
|
1184
|
+
if (quinn?.spawned && quinn.verdict !== 'PASS') {
|
|
1185
|
+
const screenshots = quinn.screenshots || [];
|
|
1186
|
+
const hasRealScreenshots = screenshots.length > 0 && screenshots.some((s: string) => existsSync(s));
|
|
1187
|
+
if (!hasRealScreenshots) {
|
|
1188
|
+
expect("quinn.screenshots empty or files missing").toBe("at least 1 screenshot file on disk — browser verification required for UI changes (#90 RCA)");
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
});
|
|
1192
|
+
|
|
1193
|
+
// quinn-container-port: Quinn must test the CONTAINER, not the dev server (#1371)
|
|
1194
|
+
//
|
|
1195
|
+
// #344: this compared against a literal 7776 — one consumer's container
|
|
1196
|
+
// port. Every other consumer got a warning saying their correct port was
|
|
1197
|
+
// the wrong one, naming a port they do not run. The expected port comes
|
|
1198
|
+
// from `container.port`, which was already a declared schema key, and with
|
|
1199
|
+
// none configured the check says it could not look rather than inventing a
|
|
1200
|
+
// number to disagree with.
|
|
1201
|
+
//
|
|
1202
|
+
// Still a warning rather than a failure, deliberately and not by oversight:
|
|
1203
|
+
// turning it into a refusal changes what blocks live runs and needs its own
|
|
1204
|
+
// measurement. What is fixed here is that it stops being wrong.
|
|
1205
|
+
test("quinn-container-port: Quinn tested the container, not the dev server", () => {
|
|
1206
|
+
const quinn = sf("agents")?.quinn;
|
|
1207
|
+
if (!quinn?.spawned) return;
|
|
1208
|
+
const agentPort = quinn.port;
|
|
1209
|
+
const envPort = sf("environments")?.local?.quinn?.port;
|
|
1210
|
+
const port = agentPort || envPort;
|
|
1211
|
+
if (!port) {
|
|
1212
|
+
console.warn("WARN: quinn-container-port: no port recorded in quinn agent data — cannot verify container testing");
|
|
1213
|
+
return;
|
|
1214
|
+
}
|
|
1215
|
+
// Bound to `harness`, not to a name of its own: #315's key registry only
|
|
1216
|
+
// sees reads off a binding called `harness`, so `const expected =
|
|
1217
|
+
// loadProjectHarness()?...` would have been a config key the registry
|
|
1218
|
+
// could not account for. Its guard caught this on the first run.
|
|
1219
|
+
const harness = loadProjectHarness();
|
|
1220
|
+
const expected = harness?.container?.port;
|
|
1221
|
+
if (!expected) {
|
|
1222
|
+
console.warn(
|
|
1223
|
+
"WARN: quinn-container-port: the project sets no `container.port`, so whether Quinn " +
|
|
1224
|
+
`tested the container could not be determined (it used :${port}). UNMEASURED, not PASS.`,
|
|
1225
|
+
);
|
|
1226
|
+
return;
|
|
1227
|
+
}
|
|
1228
|
+
if (port !== expected) {
|
|
1229
|
+
console.warn(
|
|
1230
|
+
`WARN: quinn-container-port: Quinn tested on port ${port}, expected ${expected} ` +
|
|
1231
|
+
`(container.port). Testing the dev server does not validate container behavior.`,
|
|
1232
|
+
);
|
|
1233
|
+
}
|
|
1234
|
+
});
|
|
1235
|
+
|
|
1236
|
+
// brief-ac-alignment: Marcus brief contains all AC ids (ADR-009 C2)
|
|
1237
|
+
//
|
|
1238
|
+
// #700: the decision lives in `lib/brief-ac-alignment.ts` and this is the
|
|
1239
|
+
// call site. It failed on five consecutive pushes to main and nothing in
|
|
1240
|
+
// the log said which of its two branches fired, because neither refusal
|
|
1241
|
+
// named the path it had resolved — and the path is assembled here from
|
|
1242
|
+
// RUNGATE_WORK_DIR, a $HOME/.rungate fallback and the slug, so a wrong
|
|
1243
|
+
// directory and a short brief produced the same sentence. Every refusal
|
|
1244
|
+
// from the binding now carries the resolved path.
|
|
1245
|
+
test("brief-ac-alignment: brief covers all ACs", () => {
|
|
1246
|
+
if (!isVerifyPlus()) return;
|
|
1247
|
+
const slug = sf("slug");
|
|
1248
|
+
if (!slug) return;
|
|
1249
|
+
const briefPath = runArtifact("marcus-brief.md");
|
|
1250
|
+
const exists = existsSync(briefPath);
|
|
1251
|
+
const reading = briefAcAlignment({
|
|
1252
|
+
briefPath,
|
|
1253
|
+
briefExists: exists,
|
|
1254
|
+
briefText: exists ? readFileSync(briefPath, "utf-8") : "",
|
|
1255
|
+
acIds: (sf("acs") || []).map((ac: any) => ac.id),
|
|
1256
|
+
marcusSpawned: sf("agents")?.marcus?.spawned === true,
|
|
1257
|
+
});
|
|
1258
|
+
if (reading.reading === "NOT_APPLICABLE") {
|
|
1259
|
+
console.warn(`WARN: ${reading.declined}`);
|
|
1260
|
+
return;
|
|
1261
|
+
}
|
|
1262
|
+
expect(reading.refusal, reading.refusal ?? "brief-ac-alignment").toBeNull();
|
|
1263
|
+
});
|
|
1264
|
+
|
|
1265
|
+
// ac-spec-traceability (#440)
|
|
1266
|
+
test("ac-spec-traceability: AC specElements exist in governing spec", () => {
|
|
1267
|
+
if (!isVerifyPlus()) return;
|
|
1268
|
+
const specPath = sf("governingSpec")?.path;
|
|
1269
|
+
if (!specPath || !existsSync(specPath)) return;
|
|
1270
|
+
const specContent = readFileSync(specPath, "utf-8");
|
|
1271
|
+
const missing: string[] = [];
|
|
1272
|
+
for (const ac of sf("acs") || []) {
|
|
1273
|
+
if (!ac.specElement) continue;
|
|
1274
|
+
if (!specContent.includes(ac.specElement)) {
|
|
1275
|
+
missing.push(`${ac.id}: specElement "${ac.specElement}" not found in spec`);
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
expect(missing).toEqual([]);
|
|
1279
|
+
});
|
|
1280
|
+
|
|
1281
|
+
// 28. marcus-committed (GAP → PORTED)
|
|
1282
|
+
test("marcus-committed: no uncommitted Marcus changes", () => {
|
|
1283
|
+
if (!isVerifyPlus()) return;
|
|
1284
|
+
const signalDir = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), "signals");
|
|
1285
|
+
const slug = sf("slug") || "";
|
|
1286
|
+
const signalFile = join(signalDir, `marcus-uncommitted-${slug.replace(/\//g, "-")}`);
|
|
1287
|
+
if (existsSync(signalFile)) {
|
|
1288
|
+
expect("marcus left uncommitted files").toBe("marcus committed all changes");
|
|
1289
|
+
}
|
|
1290
|
+
});
|
|
1291
|
+
|
|
1292
|
+
// prove-evidence-type: UI changes require screenshot/api-response evidence (ADR-009 A4)
|
|
1293
|
+
test("prove-evidence-type: UI changes require live evidence", () => {
|
|
1294
|
+
if (!isVerifyPlus()) return;
|
|
1295
|
+
// #314: the changed-file list comes from the branch's history, not main's.
|
|
1296
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "prove-evidence-type");
|
|
1297
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
1298
|
+
const root = treeRoot();
|
|
1299
|
+
if (!root) return;
|
|
1300
|
+
const scopeSha = sf("gates")?.scope?.commitSha;
|
|
1301
|
+
if (!scopeSha) return;
|
|
1302
|
+
// #531: branch-relative, and an unusable diff says so rather than
|
|
1303
|
+
// reading as "nothing changed".
|
|
1304
|
+
const changed = readBranchDiff(scopeSha, a => execArgvOf("git", a, root), "prove-evidence-type");
|
|
1305
|
+
if (changed.files === null) { console.warn(`WARN: ${changed.reason}`); return; }
|
|
1306
|
+
const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
|
|
1307
|
+
const uiChanged = changed.files.filter(f => UI_EXT.test(f));
|
|
1308
|
+
if (uiChanged.length === 0) return;
|
|
1309
|
+
const slug = sf("slug");
|
|
1310
|
+
if (!slug) return;
|
|
1311
|
+
const evidencePath = runArtifact("prove-evidence.json");
|
|
1312
|
+
if (!existsSync(evidencePath)) return;
|
|
1313
|
+
const evidence = JSON.parse(readFileSync(evidencePath, "utf-8"));
|
|
1314
|
+
const afterType = evidence.afterEvidence?.type;
|
|
1315
|
+
const VALID_UI_EVIDENCE = ["screenshot", "api-response"];
|
|
1316
|
+
expect(afterType, "afterEvidence.type required for UI changes (ADR-009 A4)").toBeTruthy();
|
|
1317
|
+
expect(VALID_UI_EVIDENCE.includes(afterType), `afterEvidence.type must be screenshot or api-response for UI changes, got "${afterType}" (ADR-009 A4)`).toBe(true);
|
|
1318
|
+
});
|
|
1319
|
+
|
|
1320
|
+
// acHash consistency: verify gate checks scope's AC hash (ADR-009)
|
|
1321
|
+
// Hash auto-recomputed by writeWorkflowState on authorized modifications (#43)
|
|
1322
|
+
test("acHash-consistency-verify: ACs unchanged since scope", () => {
|
|
1323
|
+
if (!isVerifyPlus()) return;
|
|
1324
|
+
const acHash = sf("gates")?.scope?.acHash;
|
|
1325
|
+
if (!acHash) return;
|
|
1326
|
+
const acDefs = (sf("acs") || []).map((ac: any) => ({
|
|
1327
|
+
id: ac.id, type: ac.type, statement: ac.statement,
|
|
1328
|
+
specElement: ac.specElement, threshold: ac.threshold,
|
|
1329
|
+
evidenceMethod: ac.evidenceMethod,
|
|
1330
|
+
}));
|
|
1331
|
+
const currentHash = require("crypto").createHash("sha256").update(JSON.stringify(acDefs)).digest("hex");
|
|
1332
|
+
if (currentHash !== acHash) {
|
|
1333
|
+
// Log field-level diff for diagnosis before failing
|
|
1334
|
+
const { diffAcHashFields } = require("./orchestrator");
|
|
1335
|
+
const diff = diffAcHashFields(acDefs, (sf("acs") || []));
|
|
1336
|
+
console.error(`acHash field diff: ${JSON.stringify(diff.changedFields)}`);
|
|
1337
|
+
}
|
|
1338
|
+
expect(currentHash).toBe(acHash);
|
|
1339
|
+
});
|
|
1340
|
+
|
|
1341
|
+
// spec-decision-compliance (#409, Decision #9: WARN tier — console.warn only, no expect failures)
|
|
1342
|
+
test("spec-decision-compliance: AC language aligns with spec decisions", () => {
|
|
1343
|
+
if (!isVerifyPlus()) return;
|
|
1344
|
+
const localPolicies = join(__dirname, "spec-policies.json");
|
|
1345
|
+
const homePolicies = join(process.env.HOME || "", ".claude", "skills", "ship", "spec-policies.json");
|
|
1346
|
+
const policiesPath = existsSync(localPolicies) ? localPolicies : homePolicies;
|
|
1347
|
+
if (!existsSync(policiesPath)) return;
|
|
1348
|
+
const policies = JSON.parse(readFileSync(policiesPath, "utf-8"));
|
|
1349
|
+
const patterns = policies.implementationPatterns;
|
|
1350
|
+
if (!patterns) return;
|
|
1351
|
+
const acs = sf("acs") || [];
|
|
1352
|
+
let warnCount = 0;
|
|
1353
|
+
for (const [key, rule] of Object.entries(patterns) as [string, any][]) {
|
|
1354
|
+
if (!rule.antiPatterns || !rule.decision) continue;
|
|
1355
|
+
const scopes: string[] = rule.scope || ["statement"];
|
|
1356
|
+
for (const ac of acs) {
|
|
1357
|
+
for (const scope of scopes) {
|
|
1358
|
+
const text = scope === "statement" ? (ac.statement || "") : (ac.evidenceMethod?.command || "");
|
|
1359
|
+
if (!text) continue;
|
|
1360
|
+
for (const ap of rule.antiPatterns) {
|
|
1361
|
+
if (new RegExp(ap, "i").test(text)) {
|
|
1362
|
+
console.warn(`WARN: spec-decision-compliance [${key}]: AC "${ac.id}" matches anti-pattern "${ap}" — ${rule.message}`);
|
|
1363
|
+
warnCount++;
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
if (warnCount > 0) {
|
|
1370
|
+
console.warn(`WARN: spec-decision-compliance: ${warnCount} anti-pattern match(es) found — review recommended`);
|
|
1371
|
+
}
|
|
1372
|
+
});
|
|
1373
|
+
|
|
1374
|
+
// contextual-skip-validation (#91 → ADR-009 A1: SKIP → FAIL for file-matched gates)
|
|
1375
|
+
test("contextual-skip-validation: SKIP blocked when changed files require gate", () => {
|
|
1376
|
+
if (!isVerifyPlus()) return;
|
|
1377
|
+
// #314: the changed-file list comes from the branch's history, not main's.
|
|
1378
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "contextual-skip-validation");
|
|
1379
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
1380
|
+
const root = treeRoot();
|
|
1381
|
+
if (!root) return;
|
|
1382
|
+
const scopeSha = sf("gates")?.scope?.commitSha;
|
|
1383
|
+
if (!scopeSha) return;
|
|
1384
|
+
// #531: branch-relative, and an unusable diff is UNMEASURED rather than
|
|
1385
|
+
// "no changed files — silently return", which is what the previous line
|
|
1386
|
+
// said and which waves every file-matched gate below through.
|
|
1387
|
+
const changed = readBranchDiff(scopeSha, a => execArgvOf("git", a, root), "contextual-skip-validation");
|
|
1388
|
+
if (changed.files === null) { console.warn(`WARN: ${changed.reason}`); return; }
|
|
1389
|
+
|
|
1390
|
+
const FILE_GATE_MAP: Record<string, string[]> = {
|
|
1391
|
+
"tsx|jsx": ["local-ui-validated", "quinn-on-ui-change"],
|
|
1392
|
+
"css|scss|less|html|svg": ["local-ui-validated"],
|
|
1393
|
+
"sql|migration": ["local-api-validated"],
|
|
1394
|
+
"Dockerfile|docker-compose|Makefile": ["prod-rebuild-pass"],
|
|
1395
|
+
};
|
|
1396
|
+
|
|
1397
|
+
const ENV_GATE_PATH: Record<string, string> = {
|
|
1398
|
+
"local-ui-validated": "environments.local.ui",
|
|
1399
|
+
"local-api-validated": "environments.local.api",
|
|
1400
|
+
"quinn-on-ui-change": "agents.quinn.spawned",
|
|
1401
|
+
"prod-rebuild-pass": "environments.prod.rebuild",
|
|
1402
|
+
};
|
|
1403
|
+
|
|
1404
|
+
const changedFiles = changed.files;
|
|
1405
|
+
let warnCount = 0;
|
|
1406
|
+
for (const file of changedFiles) {
|
|
1407
|
+
for (const [extPattern, gates] of Object.entries(FILE_GATE_MAP)) {
|
|
1408
|
+
const re = new RegExp(`\\.(${extPattern})$`, "i");
|
|
1409
|
+
const nameRe = new RegExp(`(${extPattern})`, "i");
|
|
1410
|
+
if (!re.test(file) && !nameRe.test(file.split("/").pop() || "")) continue;
|
|
1411
|
+
for (const gate of gates) {
|
|
1412
|
+
const envPath = ENV_GATE_PATH[gate];
|
|
1413
|
+
if (!envPath) continue;
|
|
1414
|
+
const val = sf(envPath);
|
|
1415
|
+
if (val === "SKIP") {
|
|
1416
|
+
expect(`contextual-skip-blocked: "${file}" changed but ${gate} is SKIP`).toBe("gate must not be SKIP when matched files changed (ADR-009 A1)");
|
|
1417
|
+
warnCount++;
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
}
|
|
1421
|
+
}
|
|
1422
|
+
if (warnCount > 0) {
|
|
1423
|
+
expect(`contextual-skip-validation: ${warnCount} SKIP(s) on file-matched gates`).toBe("contextual-skip-validation: 0 SKIPs (ADR-009 A1)");
|
|
1424
|
+
}
|
|
1425
|
+
});
|
|
1426
|
+
});
|
|
1427
|
+
|
|
1428
|
+
// ═══ B1/B2 AGENT RESULTS (#1406, #1407) ═════════════════════════════
|
|
1429
|
+
describe("B1/B2 agent results", () => {
|
|
1430
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
1431
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
1432
|
+
return;
|
|
1433
|
+
}
|
|
1434
|
+
// B2: Evidence Validator prompt file exists
|
|
1435
|
+
test("b2-prompt-exists: evidence-validator.md available", () => {
|
|
1436
|
+
const promptPath = join(__dirname, "prompts", "evidence-validator.md");
|
|
1437
|
+
expect(
|
|
1438
|
+
existsSync(promptPath),
|
|
1439
|
+
"evidence-validator.md prompt not found in gates/prompts/",
|
|
1440
|
+
).toBe(true);
|
|
1441
|
+
});
|
|
1442
|
+
|
|
1443
|
+
// B1: AC Adversary prompt file exists
|
|
1444
|
+
test("b1-prompt-exists: ac-adversary.md available", () => {
|
|
1445
|
+
const promptPath = join(__dirname, "prompts", "ac-adversary.md");
|
|
1446
|
+
expect(
|
|
1447
|
+
existsSync(promptPath),
|
|
1448
|
+
"ac-adversary.md prompt not found in gates/prompts/",
|
|
1449
|
+
).toBe(true);
|
|
1450
|
+
});
|
|
1451
|
+
|
|
1452
|
+
// B2: the evidence validator ran, and left something readable (#126)
|
|
1453
|
+
//
|
|
1454
|
+
// isShipPlus, NOT isVerifyPlus, and the contract fixtures are what found it.
|
|
1455
|
+
// gate-executor runs this suite (`runGateTests`) and only THEN spawns the
|
|
1456
|
+
// fan-out and persists its slots — so at the verify gate the slot cannot yet
|
|
1457
|
+
// exist, and a check that refuses its absence refuses every first attempt.
|
|
1458
|
+
// That is the original circularity with the sign flipped: the gate failing
|
|
1459
|
+
// on the fan-out's absence is what skips the fan-out.
|
|
1460
|
+
//
|
|
1461
|
+
// Verify cannot verify its own not-yet-produced output. Ship can: by then
|
|
1462
|
+
// the fan-out has either run or recorded why it did not, and refusing here
|
|
1463
|
+
// still blocks the PR, which is the only thing the refusal needed to do.
|
|
1464
|
+
test("b2-evidence-validator-structure: results valid when populated", () => {
|
|
1465
|
+
if (!isShipPlus()) return;
|
|
1466
|
+
const tier = sf("sizing")?.ceremonyTier;
|
|
1467
|
+
const ev = sf("gates.verify.evidenceValidator");
|
|
1468
|
+
const violations = verifyFanoutViolations("evidenceValidator", ev, tier);
|
|
1469
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
1470
|
+
// Nothing further to validate when the absence was legitimate: a LIGHT run
|
|
1471
|
+
// or a recorded skip has no verdicts, by construction.
|
|
1472
|
+
if (!ev || (ev as { skipped?: unknown }).skipped !== undefined) return;
|
|
1473
|
+
expect(ev.ts, "evidenceValidator.ts must be an ISO timestamp").toBeTruthy();
|
|
1474
|
+
expect(Array.isArray(ev.verdicts), "evidenceValidator.verdicts must be an array").toBe(true);
|
|
1475
|
+
});
|
|
1476
|
+
|
|
1477
|
+
// B1: the AC adversary ran, and left something readable (#126)
|
|
1478
|
+
// isShipPlus for the reason spelled out above B2.
|
|
1479
|
+
test("b1-adversary-verify-structure: results valid when populated", () => {
|
|
1480
|
+
if (!isShipPlus()) return;
|
|
1481
|
+
const tier = sf("sizing")?.ceremonyTier;
|
|
1482
|
+
const adv = sf("gates.verify.adversary");
|
|
1483
|
+
const violations = verifyFanoutViolations("adversary", adv, tier);
|
|
1484
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
1485
|
+
if (!adv || (adv as { skipped?: unknown }).skipped !== undefined) return;
|
|
1486
|
+
expect(adv.ts, "adversary.ts must be an ISO timestamp").toBeTruthy();
|
|
1487
|
+
expect(typeof adv.gameable, "adversary.gameable must be a number").toBe("number");
|
|
1488
|
+
expect(typeof adv.approved, "adversary.approved must be a boolean").toBe("boolean");
|
|
1489
|
+
});
|
|
1490
|
+
});
|
|
1491
|
+
|
|
1492
|
+
// ═══ SPEC TRACEABILITY (2 functions) ══════════════════════════════════
|
|
1493
|
+
describe("spec traceability", () => {
|
|
1494
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
1495
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
1496
|
+
return;
|
|
1497
|
+
}
|
|
1498
|
+
// 31. decision-id-valid (GAP → PORTED)
|
|
1499
|
+
test("decision-id-valid: D-NNN IDs in ACs exist in governing spec", () => {
|
|
1500
|
+
const specPath = sf("sourceSpecs")?.[0]?.path || sf("governingSpec")?.path;
|
|
1501
|
+
if (!specPath || !existsSync(specPath)) return;
|
|
1502
|
+
const spec = readFileSync(specPath, "utf-8");
|
|
1503
|
+
const ids = new Set<string>();
|
|
1504
|
+
for (const ac of sf("acs") || []) {
|
|
1505
|
+
const matches = (ac.statement || "").match(/D-\d+/g) || [];
|
|
1506
|
+
matches.forEach((id: string) => ids.add(id));
|
|
1507
|
+
}
|
|
1508
|
+
if (ids.size === 0) return;
|
|
1509
|
+
const missing = [...ids].filter(id => !spec.includes(id));
|
|
1510
|
+
expect(missing).toEqual([]);
|
|
1511
|
+
});
|
|
1512
|
+
|
|
1513
|
+
// 32. evidence-target-valid (GAP → PORTED)
|
|
1514
|
+
//
|
|
1515
|
+
// #315 found this one while fixing `code-committed`, and it is worse than
|
|
1516
|
+
// the defect it was filed for. It read the same unauthored
|
|
1517
|
+
// `codeCommittedPaths` key and early-returned — but even past that return
|
|
1518
|
+
// it computed an `outside` list and **discarded it**, under a comment
|
|
1519
|
+
// saying outside refs are common. It has no assertion. It is a name in the
|
|
1520
|
+
// gate list and nothing else, and it cannot fail for any input.
|
|
1521
|
+
//
|
|
1522
|
+
// Deleted rather than left standing, because a check that asserts nothing
|
|
1523
|
+
// reads as coverage in every report that counts gate checks. The threshold
|
|
1524
|
+
// it would need — which evidence file references count as out of scope —
|
|
1525
|
+
// does not exist yet and inventing one here would be a guess that goes red
|
|
1526
|
+
// on every run. It is recorded in OPTIONAL_HARNESS_KEYS in
|
|
1527
|
+
// test/gate-optional-config-keys.test.ts so it cannot be quietly re-added,
|
|
1528
|
+
// and #338 tracks building it with a threshold that has evidence behind it.
|
|
1529
|
+
//
|
|
1530
|
+
// #338 measured it, and the original name does not survive the
|
|
1531
|
+
// measurement. Across 82 completed runs, 278 of 542 classifiable evidence
|
|
1532
|
+
// file references fall OUTSIDE the AC's own contextFiles, and the largest
|
|
1533
|
+
// category is the test file that proves the AC. "Within scope" is not a
|
|
1534
|
+
// property real runs have, so no threshold for it exists to state.
|
|
1535
|
+
//
|
|
1536
|
+
// What the same measurement did find is below: 11.1% of evidence commands
|
|
1537
|
+
// cannot run anywhere but the machine that wrote them.
|
|
1538
|
+
|
|
1539
|
+
// 32. evidence-portable (#338 — replaces evidence-target-valid)
|
|
1540
|
+
test("evidence-portable: no evidence command is pinned to one machine", () => {
|
|
1541
|
+
if (!isVerifyPlus()) return;
|
|
1542
|
+
const violations = evidencePortabilityViolations(sf("acs"));
|
|
1543
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
1544
|
+
});
|
|
1545
|
+
});
|
|
1546
|
+
|
|
1547
|
+
// ═══ SHIP CHECKS (7 functions) ════════════════════════════════════════
|
|
1548
|
+
describe("ship checks", () => {
|
|
1549
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
1550
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
1551
|
+
return;
|
|
1552
|
+
}
|
|
1553
|
+
// 35. all-gates-pass (GAP → PORTED)
|
|
1554
|
+
test("all-gates-pass: scope + verify gates PASS", () => {
|
|
1555
|
+
if (!isShipPlus()) return;
|
|
1556
|
+
const scope = sf("gates")?.scope?.result;
|
|
1557
|
+
const verify = sf("gates")?.verify?.result;
|
|
1558
|
+
if (scope && scope !== "PASS") {
|
|
1559
|
+
expect(`scope gate: ${scope}`).toBe("PASS");
|
|
1560
|
+
}
|
|
1561
|
+
if (verify && verify !== "PASS") {
|
|
1562
|
+
expect(`verify gate: ${verify}`).toBe("PASS");
|
|
1563
|
+
}
|
|
1564
|
+
});
|
|
1565
|
+
|
|
1566
|
+
// 36. branch-merged / code-pushed (GAP → PORTED)
|
|
1567
|
+
// Ship gate: code pushed + PR exists = PASS (merge is post-ship, verified at prove)
|
|
1568
|
+
// Prove gate: merged to main = PASS
|
|
1569
|
+
test("branch-merged: fix is on main", () => {
|
|
1570
|
+
if (!isShipPlus()) return;
|
|
1571
|
+
// #314: `git branch --contains HEAD` must ask about the branch's HEAD.
|
|
1572
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "branch-merged");
|
|
1573
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
1574
|
+
const root = treeRoot();
|
|
1575
|
+
if (!root) return;
|
|
1576
|
+
const gate = process.env.GATE || "";
|
|
1577
|
+
if (gate === "ship") {
|
|
1578
|
+
// At ship gate: check code is pushed, not merged
|
|
1579
|
+
const commit = sf("buildCommit");
|
|
1580
|
+
const agents = sf("agents");
|
|
1581
|
+
const pushed = agents?.marcus?.branch;
|
|
1582
|
+
if (!commit && !pushed) {
|
|
1583
|
+
expect("no commit or branch").toBe("buildCommit or marcus.branch required at ship gate");
|
|
1584
|
+
}
|
|
1585
|
+
return;
|
|
1586
|
+
}
|
|
1587
|
+
// At prove gate or verify: check merged to main
|
|
1588
|
+
const result = exec("git branch --contains HEAD --list main", root);
|
|
1589
|
+
if (result.ok && !result.output.includes("main")) {
|
|
1590
|
+
expect("not on main").toBe("merged to main");
|
|
1591
|
+
}
|
|
1592
|
+
});
|
|
1593
|
+
|
|
1594
|
+
// 38. prod-rebuild-pass (GAP → PORTED)
|
|
1595
|
+
test("prod-rebuild-pass: prod rebuild required when configured", () => {
|
|
1596
|
+
if (!isShipPlus()) return;
|
|
1597
|
+
const harness = loadProjectHarness();
|
|
1598
|
+
if (!harness?.prod?.rebuild) return;
|
|
1599
|
+
const val = sf("environments")?.prod?.rebuild;
|
|
1600
|
+
if (!val) {
|
|
1601
|
+
expect("prod.rebuild not set").toBe("PASS or SKIP with reason — required when prod.rebuild is configured");
|
|
1602
|
+
} else if (val === "SKIP") {
|
|
1603
|
+
const reason = sf("environments")?.prod?.rebuildSkipReason;
|
|
1604
|
+
if (!reason) expect("rebuild SKIP without reason").toBe("rebuildSkipReason required");
|
|
1605
|
+
} else if (val !== "PASS") {
|
|
1606
|
+
expect(`prod rebuild: ${val}`).toBe("PASS or SKIP with reason");
|
|
1607
|
+
}
|
|
1608
|
+
});
|
|
1609
|
+
|
|
1610
|
+
// 39. prod-smoke-pass (GAP → PORTED)
|
|
1611
|
+
test("prod-smoke-pass: prod smoke required when configured", () => {
|
|
1612
|
+
if (!isShipPlus()) return;
|
|
1613
|
+
const harness = loadProjectHarness();
|
|
1614
|
+
if (!harness?.prod?.smokeTest) return;
|
|
1615
|
+
const val = sf("environments")?.prod?.smoke;
|
|
1616
|
+
if (!val) {
|
|
1617
|
+
expect("prod.smoke not set").toBe("PASS or SKIP with reason — required when prod.smokeTest is configured");
|
|
1618
|
+
} else if (val === "SKIP") {
|
|
1619
|
+
const reason = sf("environments")?.prod?.smokeSkipReason;
|
|
1620
|
+
if (!reason) expect("smoke SKIP without reason").toBe("smokeSkipReason required");
|
|
1621
|
+
} else if (val !== "PASS") {
|
|
1622
|
+
expect(`prod smoke: ${val}`).toBe("PASS or SKIP with reason");
|
|
1623
|
+
}
|
|
1624
|
+
});
|
|
1625
|
+
|
|
1626
|
+
// 40. prod-quinn-pass (GAP → PORTED)
|
|
1627
|
+
test("prod-quinn-pass: prod quinn required when prod configured", () => {
|
|
1628
|
+
if (!isShipPlus()) return;
|
|
1629
|
+
const harness = loadProjectHarness();
|
|
1630
|
+
if (!harness?.prod?.rebuild) return;
|
|
1631
|
+
const val = sf("environments")?.prod?.quinn;
|
|
1632
|
+
if (!val) {
|
|
1633
|
+
expect("prod.quinn not set").toBe("PASS or SKIP with reason — required when prod.rebuild is configured");
|
|
1634
|
+
} else if (val === "SKIP") {
|
|
1635
|
+
const reason = sf("environments")?.prod?.quinnSkipReason;
|
|
1636
|
+
if (!reason) expect("quinn SKIP without reason").toBe("quinnSkipReason required");
|
|
1637
|
+
} else if (val !== "PASS") {
|
|
1638
|
+
expect(`prod quinn: ${val}`).toBe("PASS or SKIP with reason");
|
|
1639
|
+
}
|
|
1640
|
+
});
|
|
1641
|
+
|
|
1642
|
+
// quinn-tested-changed-files (#441)
|
|
1643
|
+
test("quinn-tested-changed-files: Quinn tested paths cover UI diffs", () => {
|
|
1644
|
+
if (!isShipPlus()) return;
|
|
1645
|
+
// #314: the changed-file list comes from the branch's history, not main's.
|
|
1646
|
+
const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "quinn-tested-changed-files");
|
|
1647
|
+
expect(treeViolations, treeViolations.join("\n")).toEqual([]);
|
|
1648
|
+
const root = treeRoot();
|
|
1649
|
+
if (!root) return;
|
|
1650
|
+
const scopeSha = sf("gates")?.scope?.commitSha;
|
|
1651
|
+
if (!scopeSha) return;
|
|
1652
|
+
// #531: branch-relative. This is the check that refused on seven `.tsx`
|
|
1653
|
+
// files the branch has never contained.
|
|
1654
|
+
const changed = readBranchDiff(scopeSha, a => execArgvOf("git", a, root), "quinn-tested-changed-files");
|
|
1655
|
+
if (changed.files === null) { console.warn(`WARN: ${changed.reason}`); return; }
|
|
1656
|
+
const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
|
|
1657
|
+
const uiChanged = changed.files.filter(f => UI_EXT.test(f));
|
|
1658
|
+
if (uiChanged.length === 0) return;
|
|
1659
|
+
const testedPaths = sf("agents")?.quinn?.testedPaths || [];
|
|
1660
|
+
if (testedPaths.length === 0) {
|
|
1661
|
+
expect(`${uiChanged.length} UI files changed but quinn.testedPaths empty`).toBe("quinn.testedPaths populated");
|
|
1662
|
+
}
|
|
1663
|
+
});
|
|
1664
|
+
|
|
1665
|
+
// 42. verify-blockers-cleared
|
|
1666
|
+
test("verify-blockers-cleared: all Quinn-found bugs re-verified", () => {
|
|
1667
|
+
if (!isShipPlus()) return;
|
|
1668
|
+
const blockers = sf("verifyBlockers") || [];
|
|
1669
|
+
const unresolved = blockers.filter((b: any) => !b.quinnReverified);
|
|
1670
|
+
expect(unresolved.map((b: any) => b.id)).toEqual([]);
|
|
1671
|
+
});
|
|
1672
|
+
|
|
1673
|
+
// 41. prod-quinn-spot (GAP → PORTED)
|
|
1674
|
+
test("prod-quinn-spot: spot check PASS or inherited from full quinn", () => {
|
|
1675
|
+
if (!isShipPlus()) return;
|
|
1676
|
+
const harness = loadProjectHarness();
|
|
1677
|
+
if (!harness?.prod?.rebuild) return;
|
|
1678
|
+
const quinn = sf("environments")?.prod?.quinn;
|
|
1679
|
+
const spot = sf("environments")?.prod?.quinnSpot;
|
|
1680
|
+
if (quinn === "PASS" || spot === "PASS") return; // either is fine
|
|
1681
|
+
if (spot === "SKIP") {
|
|
1682
|
+
const reason = sf("environments")?.prod?.quinnSpotSkipReason;
|
|
1683
|
+
if (!reason) expect("quinnSpot SKIP without reason").toBe("quinnSpotSkipReason required");
|
|
1684
|
+
}
|
|
1685
|
+
});
|
|
1686
|
+
|
|
1687
|
+
// 43. witness-chain (#473)
|
|
1688
|
+
test("witness-chain: witness directory has gate artifacts", () => {
|
|
1689
|
+
if (!isShipPlus()) return;
|
|
1690
|
+
const slug = sf("slug");
|
|
1691
|
+
if (!slug) return;
|
|
1692
|
+
const witnessDir = runArtifact("witnesses");
|
|
1693
|
+
if (!existsSync(witnessDir)) {
|
|
1694
|
+
// WARN — witnesses are new, not all workflows have them yet
|
|
1695
|
+
console.warn("WARN: No witness directory found — witness chain not verified");
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
const witnesses = readdirSync(witnessDir).filter(f => f.endsWith(".json"));
|
|
1699
|
+
expect(witnesses.length, "Witness directory exists but is empty").toBeGreaterThan(0);
|
|
1700
|
+
});
|
|
1701
|
+
|
|
1702
|
+
// 44. witness-ac-verdict: AC PASS verdicts must have corresponding witness chain entry (#521)
|
|
1703
|
+
test("witness-ac-verdict: AC PASS verdicts have matching witness with valid HMAC", () => {
|
|
1704
|
+
if (!isShipPlus()) return;
|
|
1705
|
+
const slug = sf("slug");
|
|
1706
|
+
if (!slug) return;
|
|
1707
|
+
const witnessDir = runArtifact("witnesses");
|
|
1708
|
+
if (!existsSync(witnessDir)) {
|
|
1709
|
+
// No witnesses = cannot verify AC verdicts were set by gate system
|
|
1710
|
+
const passACs = (sf("acs") || []).filter((ac: any) => ac.verdict === "PASS");
|
|
1711
|
+
if (passACs.length > 0) {
|
|
1712
|
+
expect(
|
|
1713
|
+
`${passACs.length} AC(s) have PASS verdict but no witness directory exists`,
|
|
1714
|
+
).toBe("witness directory required when ACs have PASS verdicts (#521 — prevents manual workflow-state.json edits)");
|
|
1715
|
+
}
|
|
1716
|
+
return;
|
|
1717
|
+
}
|
|
1718
|
+
const witnessFiles = readdirSync(witnessDir).filter(f => f.endsWith(".json"));
|
|
1719
|
+
// Load all witness records
|
|
1720
|
+
const witnesses: Array<{ gate: string; result: string; hmac: string; [k: string]: any }> = [];
|
|
1721
|
+
for (const wf of witnessFiles) {
|
|
1722
|
+
try {
|
|
1723
|
+
const record = JSON.parse(readFileSync(join(witnessDir, wf), "utf-8"));
|
|
1724
|
+
witnesses.push(record);
|
|
1725
|
+
} catch { /* skip unparseable */ }
|
|
1726
|
+
}
|
|
1727
|
+
// Build set of gates that have PASS witnesses with HMAC
|
|
1728
|
+
const passedGatesWithHmac = new Set(
|
|
1729
|
+
witnesses
|
|
1730
|
+
.filter(w => w.result === "PASS" && w.hmac && w.hmac.length > 0)
|
|
1731
|
+
.map(w => w.gate),
|
|
1732
|
+
);
|
|
1733
|
+
// Each AC with PASS verdict must be backed by a prior gate witness
|
|
1734
|
+
// The verify gate covers AC evidence checks; scope gate covers AC definition
|
|
1735
|
+
// Ship witness is written AFTER this test suite — requiring it here is circular
|
|
1736
|
+
// Scope is skipped for LIGHT ceremony tier — only require its witness if it ran
|
|
1737
|
+
const scopeRan = sf("gates")?.scope?.ts;
|
|
1738
|
+
const requiredGates = scopeRan ? ["scope", "verify"] : ["verify"];
|
|
1739
|
+
const missingGates = requiredGates.filter(g => !passedGatesWithHmac.has(g));
|
|
1740
|
+
const passACs = (sf("acs") || []).filter((ac: any) => ac.verdict === "PASS");
|
|
1741
|
+
if (passACs.length > 0 && missingGates.length > 0) {
|
|
1742
|
+
expect(
|
|
1743
|
+
`AC PASS verdicts present but missing witness for: ${missingGates.join(", ")}`,
|
|
1744
|
+
).toBe("PASS witnesses with valid HMAC required for verify + ship gates (#521 — manual edits detected)");
|
|
1745
|
+
}
|
|
1746
|
+
});
|
|
1747
|
+
|
|
1748
|
+
// pre-registration: ACs must predate fix (ADR-009 A2)
|
|
1749
|
+
test("pre-registration: scope precedes ship", () => {
|
|
1750
|
+
if (!isShipPlus()) return;
|
|
1751
|
+
const scopeTs = sf("gates")?.scope?.ts;
|
|
1752
|
+
if (!scopeTs) return; // LIGHT tier or pre-ADR-009 — scope gate not run
|
|
1753
|
+
const scopeTime = new Date(scopeTs).getTime();
|
|
1754
|
+
expect(scopeTime, "scope timestamp must be in the past (ADR-009 A2: pre-registration)").toBeLessThan(Date.now());
|
|
1755
|
+
const shipTs = sf("gates")?.ship?.ts;
|
|
1756
|
+
if (shipTs) {
|
|
1757
|
+
expect(scopeTime, "scope must predate ship (ADR-009 A2: pre-registration)").toBeLessThanOrEqual(new Date(shipTs).getTime());
|
|
1758
|
+
}
|
|
1759
|
+
});
|
|
1760
|
+
|
|
1761
|
+
// acHash consistency: ship gate checks scope's AC hash (ADR-009)
|
|
1762
|
+
// Restored to FAIL: writeWorkflowState now recomputes acHash on authorized modifications,
|
|
1763
|
+
// so mismatches indicate unauthorized tampering, not heal agent updates (#43)
|
|
1764
|
+
test("acHash-consistency-ship: ACs unchanged since scope", () => {
|
|
1765
|
+
if (!isShipPlus()) return;
|
|
1766
|
+
const acHash = sf("gates")?.scope?.acHash;
|
|
1767
|
+
if (!acHash) return;
|
|
1768
|
+
const acDefs = (sf("acs") || []).map((ac: any) => ({
|
|
1769
|
+
id: ac.id, type: ac.type, statement: ac.statement,
|
|
1770
|
+
specElement: ac.specElement, threshold: ac.threshold,
|
|
1771
|
+
evidenceMethod: ac.evidenceMethod,
|
|
1772
|
+
}));
|
|
1773
|
+
const currentHash = require("crypto").createHash("sha256").update(JSON.stringify(acDefs)).digest("hex");
|
|
1774
|
+
if (currentHash !== acHash) {
|
|
1775
|
+
// Log field-level diff for diagnosis before failing
|
|
1776
|
+
const { diffAcHashFields } = require("./orchestrator");
|
|
1777
|
+
const scopeAcDefs = acDefs; // current defs — scope defs are lost, but hash mismatch means they differ
|
|
1778
|
+
const diff = diffAcHashFields(acDefs, (sf("acs") || []));
|
|
1779
|
+
console.error(`acHash field diff: ${JSON.stringify(diff.changedFields)}`);
|
|
1780
|
+
}
|
|
1781
|
+
expect(currentHash).toBe(acHash);
|
|
1782
|
+
});
|
|
1783
|
+
|
|
1784
|
+
// prove-label-enforcement: needs-prove label must have matching proven label (#1390)
|
|
1785
|
+
//
|
|
1786
|
+
// The labels come from Octokit, not from a GitHub CLI subprocess (#395).
|
|
1787
|
+
// The old call interpolated `issue` and `issueRepo` — both out of workflow
|
|
1788
|
+
// state — into a shell command string, and needed `gh` installed and
|
|
1789
|
+
// authenticated in whatever tree the gate ran in. `getIssueLabels` takes
|
|
1790
|
+
// them as arguments, so no shell parses them.
|
|
1791
|
+
//
|
|
1792
|
+
// The decision is `proveLabelViolation` and lives in lib/, because it was
|
|
1793
|
+
// entangled with the transport and therefore untestable: nothing could
|
|
1794
|
+
// exercise the rule without a network and a `gh` binary, so nothing did,
|
|
1795
|
+
// and SC-502 was ticked over the subprocess for its whole life.
|
|
1796
|
+
test("prove-label-enforcement: needs-prove requires proven label", async () => {
|
|
1797
|
+
if (!isShipPlus()) return;
|
|
1798
|
+
const issueNumber = sf("issue");
|
|
1799
|
+
const issueRepo = sf("issueRepo") || sf("repo");
|
|
1800
|
+
if (!issueNumber || !issueRepo) return;
|
|
1801
|
+
|
|
1802
|
+
// #405 — the fail-open is gone. This used to be `catch { return }`, so a
|
|
1803
|
+
// missing token, a rate limit, a network blip and a typo'd repo slug all
|
|
1804
|
+
// landed in one place and the gate reported clean: "no blocking label"
|
|
1805
|
+
// and "could not find out" were the same outcome.
|
|
1806
|
+
//
|
|
1807
|
+
// `proveLabelGateVerdict` owns the three-way decision and the reasoning
|
|
1808
|
+
// behind the token-less case. The read is the only thing left here,
|
|
1809
|
+
// because a decision entangled with its transport is a decision nobody
|
|
1810
|
+
// tests — the lesson that produced `proveLabelViolation` in the first
|
|
1811
|
+
// place.
|
|
1812
|
+
const { createGitHubClient, getIssueLabels, resolveGitHubToken } = require("../lib/github");
|
|
1813
|
+
const tokenPresent = Boolean(resolveGitHubToken());
|
|
1814
|
+
let read: LabelRead;
|
|
1815
|
+
try {
|
|
1816
|
+
read = {
|
|
1817
|
+
labels: await getIssueLabels(createGitHubClient(), issueRepo, Number(issueNumber)),
|
|
1818
|
+
tokenPresent,
|
|
1819
|
+
};
|
|
1820
|
+
} catch (e) {
|
|
1821
|
+
read = { error: e, tokenPresent };
|
|
1822
|
+
}
|
|
1823
|
+
|
|
1824
|
+
const outcome = proveLabelGateVerdict(read);
|
|
1825
|
+
if (outcome.verdict === "UNMEASURED") {
|
|
1826
|
+
// Reported, not swallowed. `SKIP` is indistinguishable from success and
|
|
1827
|
+
// `UNMEASURED` is not (`.claude/rules/config-access.md`).
|
|
1828
|
+
console.error(`PROVE_LABEL_UNMEASURED: ${outcome.message}`);
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
expect(outcome.verdict, outcome.message ?? undefined).toBe("ALLOW");
|
|
1832
|
+
});
|
|
1833
|
+
});
|
|
1834
|
+
|
|
1835
|
+
// ═══ PROMPT IMMUTABILITY (#1409, ADR-009) ════════════════════════════
|
|
1836
|
+
//
|
|
1837
|
+
// This asserted the macOS `uchg` flag via `ls -lO`, and was broken in both
|
|
1838
|
+
// directions at once (#92).
|
|
1839
|
+
//
|
|
1840
|
+
// It failed OPEN on CI, which is the only place it runs. `ls -lO` is BSD; the
|
|
1841
|
+
// workflows are ubuntu-latest, where GNU coreutils rejects -O and exits 2. The
|
|
1842
|
+
// check read `if (result.ok && !output.includes("uchg"))`, so a non-zero exit
|
|
1843
|
+
// skipped the body, the unlocked list stayed empty, and the test passed. Every
|
|
1844
|
+
// green CI run in this repo's history passed it without evaluating a file.
|
|
1845
|
+
//
|
|
1846
|
+
// It failed CLOSED anywhere it would have worked. chflags is filesystem
|
|
1847
|
+
// metadata and git does not track it, so a fresh clone or `git worktree add`
|
|
1848
|
+
// starts with no flags — confirmed by a second session seeing 0 of 3 flagged
|
|
1849
|
+
// in its worktree against 3 of 3 in the primary checkout at the same commit.
|
|
1850
|
+
// It failed hardest for a developer doing what the harness tells them to do.
|
|
1851
|
+
//
|
|
1852
|
+
// And the remedy it printed, `gates/lock-prompts.sh`, does not exist. Nothing
|
|
1853
|
+
// in the repo calls chflags at all: the check asserted a property the project
|
|
1854
|
+
// had no mechanism to establish.
|
|
1855
|
+
//
|
|
1856
|
+
// What it actually wants is that the prompts have not been modified. That is
|
|
1857
|
+
// content, git already tracks it, and it means the same thing on every
|
|
1858
|
+
// platform and in every checkout.
|
|
1859
|
+
// A diff against HEAD alone is not enough, and it is worth being explicit
|
|
1860
|
+
// about why rather than shipping the obvious fix. In CI the working tree IS
|
|
1861
|
+
// HEAD, so `git diff HEAD` is empty by construction and the check could never
|
|
1862
|
+
// fail in the one place it runs — the same vacuity, rebuilt. The committed
|
|
1863
|
+
// checksum baseline is what makes it meaningful there: editing a prompt
|
|
1864
|
+
// without deliberately updating gates/prompts.sha256 fails on CI, and updating
|
|
1865
|
+
// the baseline is a reviewable line in the diff. That is also the mechanism
|
|
1866
|
+
// the old check never had — a way to establish the property, not just assert
|
|
1867
|
+
// it.
|
|
1868
|
+
//
|
|
1869
|
+
// Both checks are kept. The baseline catches committed drift anywhere; the
|
|
1870
|
+
// HEAD diff catches local tampering before it is ever committed.
|
|
1871
|
+
describe("prompt immutability", () => {
|
|
1872
|
+
const promptDir = join(__dirname, "prompts");
|
|
1873
|
+
const baselinePath = join(__dirname, "prompts.sha256");
|
|
1874
|
+
|
|
1875
|
+
test("prompt-immutability: prompt files match the committed checksum baseline", () => {
|
|
1876
|
+
if (!existsSync(promptDir)) return;
|
|
1877
|
+
const prompts = readdirSync(promptDir).filter(f => f.endsWith(".md")).sort();
|
|
1878
|
+
if (prompts.length === 0) return;
|
|
1879
|
+
|
|
1880
|
+
expect(existsSync(baselinePath), `missing ${baselinePath} — the baseline IS the immutability record`).toBe(true);
|
|
1881
|
+
|
|
1882
|
+
const baseline = new Map<string, string>();
|
|
1883
|
+
for (const line of readFileSync(baselinePath, "utf-8").split("\n")) {
|
|
1884
|
+
const m = line.trim().match(/^([0-9a-f]{64})\s+(.+)$/);
|
|
1885
|
+
if (m) baseline.set(m[2], m[1]);
|
|
1886
|
+
}
|
|
1887
|
+
// A baseline that parsed to nothing would make every comparison below
|
|
1888
|
+
// vacuous, so reject it rather than report success.
|
|
1889
|
+
expect(baseline.size, `${baselinePath} contains no usable checksum lines`).toBeGreaterThan(0);
|
|
1890
|
+
|
|
1891
|
+
const drifted: string[] = [];
|
|
1892
|
+
for (const f of prompts) {
|
|
1893
|
+
const want = baseline.get(f);
|
|
1894
|
+
const got = createHash("sha256").update(readFileSync(join(promptDir, f))).digest("hex");
|
|
1895
|
+
if (want !== got) drifted.push(f);
|
|
1896
|
+
}
|
|
1897
|
+
// A prompt added without a baseline entry is drift too — otherwise the
|
|
1898
|
+
// way to evade the check is to add a file rather than edit one.
|
|
1899
|
+
for (const f of baseline.keys()) {
|
|
1900
|
+
if (!prompts.includes(f)) drifted.push(`${f} (in baseline, missing on disk)`);
|
|
1901
|
+
}
|
|
1902
|
+
|
|
1903
|
+
expect(
|
|
1904
|
+
drifted,
|
|
1905
|
+
`Prompt files are immutable (#1409, ADR-009). If a change is intended, update gates/prompts.sha256 in the same commit so it shows up in review.`,
|
|
1906
|
+
).toEqual([]);
|
|
1907
|
+
});
|
|
1908
|
+
|
|
1909
|
+
test("prompt-immutability: prompt files are unmodified against HEAD", () => {
|
|
1910
|
+
if (!existsSync(promptDir)) return;
|
|
1911
|
+
const prompts = readdirSync(promptDir).filter(f => f.endsWith(".md"));
|
|
1912
|
+
if (prompts.length === 0) return;
|
|
1913
|
+
|
|
1914
|
+
const repoRoot = join(__dirname, "..");
|
|
1915
|
+
const rel = "gates/prompts";
|
|
1916
|
+
const result = exec(`git -C "${repoRoot}" diff --name-only HEAD -- "${rel}"`);
|
|
1917
|
+
|
|
1918
|
+
// Fail closed. The predecessor treated a failed command as "nothing to
|
|
1919
|
+
// report", which is precisely how it passed vacuously for its whole life.
|
|
1920
|
+
expect(result.ok, `could not diff ${rel} against HEAD: ${result.output}`).toBe(true);
|
|
1921
|
+
|
|
1922
|
+
const modified = result.output.split("\n").map(l => l.trim()).filter(Boolean);
|
|
1923
|
+
expect(modified, `Prompt files are immutable (#1409, ADR-009) — revert with: git checkout HEAD -- ${rel}`).toEqual([]);
|
|
1924
|
+
});
|
|
1925
|
+
});
|
|
1926
|
+
|
|
1927
|
+
// ═══ PROVE CHECKS (ADR-009 B3) ═══════════════════════════════════════
|
|
1928
|
+
describe("prove checks", () => {
|
|
1929
|
+
if (STANDALONE || !existsSync(SF)) {
|
|
1930
|
+
test("skipped — no active workflow (standalone run)", () => {});
|
|
1931
|
+
return;
|
|
1932
|
+
}
|
|
1933
|
+
// prove-evidence exists when prove gate runs
|
|
1934
|
+
test("prove-evidence-exists: prove gate produces evidence file", () => {
|
|
1935
|
+
const currentGate = process.env.GATE;
|
|
1936
|
+
if (currentGate !== "prove") return;
|
|
1937
|
+
const slug = sf("slug");
|
|
1938
|
+
if (!slug) return;
|
|
1939
|
+
const evidencePath = runArtifact("prove-evidence.json");
|
|
1940
|
+
expect(existsSync(evidencePath), "prove-evidence.json must exist after prove gate runs (ADR-009 B3)").toBe(true);
|
|
1941
|
+
});
|
|
1942
|
+
|
|
1943
|
+
// prove-evidence has valid verdict enum
|
|
1944
|
+
test("prove-evidence-verdict: verdict is PROVEN|UNPROVEN|INCONCLUSIVE", () => {
|
|
1945
|
+
const currentGate = process.env.GATE;
|
|
1946
|
+
if (currentGate !== "prove") return;
|
|
1947
|
+
const slug = sf("slug");
|
|
1948
|
+
if (!slug) return;
|
|
1949
|
+
const evidencePath = runArtifact("prove-evidence.json");
|
|
1950
|
+
if (!existsSync(evidencePath)) return;
|
|
1951
|
+
const evidence = JSON.parse(readFileSync(evidencePath, "utf-8"));
|
|
1952
|
+
const VALID_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
|
|
1953
|
+
expect(VALID_VERDICTS.includes(evidence.verdict), `prove-evidence verdict must be PROVEN|UNPROVEN|INCONCLUSIVE, got "${evidence.verdict}" (ADR-009 B3)`).toBe(true);
|
|
1954
|
+
});
|
|
1955
|
+
});
|
|
1956
|
+
|
|
1957
|
+
// ═══ TEST BASELINE SUBTRACTION (#599) ═══════════════════════════════
|
|
1958
|
+
describe("test-baseline-subtraction", () => {
|
|
1959
|
+
// AC-2: subtracts baseline from total test failures
|
|
1960
|
+
test("subtracts baseline from total test failures", () => {
|
|
1961
|
+
expect(subtractTestBaseline(5, 3)).toBe(2);
|
|
1962
|
+
expect(subtractTestBaseline(10, 0)).toBe(10);
|
|
1963
|
+
expect(subtractTestBaseline(0, 0)).toBe(0);
|
|
1964
|
+
});
|
|
1965
|
+
|
|
1966
|
+
test("baseline diffing clamps to zero for negative values", () => {
|
|
1967
|
+
// If baseline is higher than total (tests were fixed), new failures = 0
|
|
1968
|
+
expect(subtractTestBaseline(2, 5)).toBe(0);
|
|
1969
|
+
expect(subtractTestBaseline(0, 3)).toBe(0);
|
|
1970
|
+
});
|
|
1971
|
+
|
|
1972
|
+
// AC-3: zero new failures = PASS
|
|
1973
|
+
test("zero new failures when total equals baseline", () => {
|
|
1974
|
+
const newFailures = subtractTestBaseline(3, 3);
|
|
1975
|
+
expect(newFailures).toBe(0);
|
|
1976
|
+
// 0 new failures should mean PASS
|
|
1977
|
+
const resultVal: "PASS" | "FAIL" = newFailures > 0 ? "FAIL" : "PASS";
|
|
1978
|
+
expect(resultVal).toBe("PASS");
|
|
1979
|
+
});
|
|
1980
|
+
|
|
1981
|
+
test("pre-existing failures do not block when no new failures introduced", () => {
|
|
1982
|
+
// Scenario: 7 pre-existing failures, Marcus introduces 0 new ones
|
|
1983
|
+
const baseline = 7;
|
|
1984
|
+
const totalAfterMarcus = 7;
|
|
1985
|
+
const newFailures = subtractTestBaseline(totalAfterMarcus, baseline);
|
|
1986
|
+
expect(newFailures).toBe(0);
|
|
1987
|
+
const resultVal: "PASS" | "FAIL" = newFailures > 0 ? "FAIL" : "PASS";
|
|
1988
|
+
expect(resultVal).toBe("PASS");
|
|
1989
|
+
});
|
|
1990
|
+
|
|
1991
|
+
// extractTestFailureCount unit tests
|
|
1992
|
+
test("extractTestFailureCount: parses failure count from bun test output", () => {
|
|
1993
|
+
const output = `
|
|
1994
|
+
42 pass
|
|
1995
|
+
3 fail
|
|
1996
|
+
`;
|
|
1997
|
+
expect(extractTestFailureCount(output)).toBe(3);
|
|
1998
|
+
});
|
|
1999
|
+
|
|
2000
|
+
test("extractTestFailureCount: returns 0 when no failures in output", () => {
|
|
2001
|
+
const output = `
|
|
2002
|
+
42 pass
|
|
2003
|
+
`;
|
|
2004
|
+
expect(extractTestFailureCount(output)).toBe(0);
|
|
2005
|
+
});
|
|
2006
|
+
|
|
2007
|
+
test("extractTestFailureCount: uses last fail line when multiple present", () => {
|
|
2008
|
+
// bun test can print intermediate summaries
|
|
2009
|
+
const output = `
|
|
2010
|
+
10 pass
|
|
2011
|
+
2 fail
|
|
2012
|
+
|
|
2013
|
+
52 pass
|
|
2014
|
+
5 fail
|
|
2015
|
+
`;
|
|
2016
|
+
expect(extractTestFailureCount(output)).toBe(5);
|
|
2017
|
+
});
|
|
2018
|
+
});
|