rungate 0.8.0 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/gates/.gate-salt +1 -0
  2. package/gates/SCHEMA-GUIDE.md +485 -0
  3. package/gates/ac-quality.test.ts +106 -0
  4. package/gates/adversarial.test.ts +283 -0
  5. package/gates/brief-assembler.test.ts +487 -0
  6. package/gates/brief-assembler.ts +448 -0
  7. package/gates/bunfig.toml +2 -0
  8. package/gates/chain.test.ts +91 -0
  9. package/gates/e2e-smoke.test.ts +234 -0
  10. package/gates/error-classifier.test.ts +250 -0
  11. package/gates/error-classifier.ts +99 -0
  12. package/gates/gate-executor.ts +3071 -0
  13. package/gates/orchestrator.test.ts +968 -0
  14. package/gates/orchestrator.ts +827 -0
  15. package/gates/preload.ts +89 -0
  16. package/gates/prompts/ac-adversary.md +20 -0
  17. package/gates/prompts/evidence-validator.md +22 -0
  18. package/gates/prompts/prove-reproducer.md +53 -0
  19. package/gates/prompts.sha256 +3 -0
  20. package/gates/prove.test.ts +164 -0
  21. package/gates/rook-state.test.ts +108 -0
  22. package/gates/run-attribution.ts +290 -0
  23. package/gates/run-gate.ts +163 -0
  24. package/gates/schema-parity.test.ts +378 -0
  25. package/gates/schema.ts +1209 -0
  26. package/gates/self-heal.test.ts +511 -0
  27. package/gates/self-heal.ts +125 -0
  28. package/gates/ship-orchestrator.test.ts +399 -0
  29. package/gates/ship-orchestrator.ts +384 -0
  30. package/gates/spec-compliance-probe.ts +159 -0
  31. package/gates/test-fixtures/broken-no-environments.json +110 -0
  32. package/gates/test-fixtures/consumer-directory-layout.json +90 -0
  33. package/gates/test-fixtures/valid-standard.json +91 -0
  34. package/gates/test-utils.ts +43 -0
  35. package/gates/typecheck-gate.ts +111 -0
  36. package/gates/witness.ts +349 -0
  37. package/gates/workflow.test.ts +2018 -0
  38. package/hooks/BashWriteGuard.hook.ts +19 -0
  39. package/hooks/GateResultProtection.hook.ts +27 -1
  40. package/hooks/VerifyPhaseLock.hook.ts +51 -11
  41. package/hooks/WorkflowStateGuard.hook.ts +24 -0
  42. package/lib/ai-review-rounds.ts +462 -0
  43. package/lib/bash-file-read.ts +17 -3
  44. package/lib/bash-write-guard.ts +1285 -50
  45. package/lib/distribution-standing.ts +75 -4
  46. package/lib/failure-intake.ts +12 -4
  47. package/lib/file-extensions.ts +81 -0
  48. package/lib/finding-arbiter.ts +1015 -0
  49. package/lib/finding-jury.ts +537 -0
  50. package/lib/gate-enforcement.ts +171 -43
  51. package/lib/gate-result-protection.ts +12 -2
  52. package/lib/generators/code-map.ts +55 -4
  53. package/lib/generators/types.ts +27 -1
  54. package/lib/github.ts +55 -0
  55. package/lib/interpreters.ts +64 -0
  56. package/lib/issue-create-override.ts +69 -8
  57. package/lib/issue-filing-gate.ts +605 -0
  58. package/lib/js-top-level-code.ts +20 -2
  59. package/lib/junit-checks.ts +239 -1
  60. package/lib/package-closure.ts +114 -0
  61. package/lib/publish-window.ts +287 -0
  62. package/lib/scaffold/ai-review-findings.ts +142 -3
  63. package/lib/scaffold/ai-review-step.ts +1012 -18
  64. package/lib/scaffold/managed-workflow.ts +163 -4
  65. package/lib/scaffold/steps.ts +498 -22
  66. package/lib/scanner.ts +13 -4
  67. package/lib/ticked-sc-symbols.ts +361 -0
  68. package/lib/transcript-checker.ts +6 -0
  69. package/lib/workflow-actions.ts +18 -0
  70. package/package.json +3 -1
  71. package/scripts/arbitrate-finding.ts +224 -0
  72. package/scripts/generate-code-map.ts +18 -10
  73. package/scripts/sync-sc-status.ts +82 -8
@@ -0,0 +1,2018 @@
1
+ import { test, expect, describe, setDefaultTimeout } from "bun:test";
2
+ import { readFileSync, existsSync, readdirSync } from "fs";
3
+ import { execSync, execFileSync, spawnSync } from "child_process";
4
+ import { createHash } from "crypto";
5
+ import { join } from "path";
6
+ import { WorkflowStateSchema } from "./schema";
7
+ import { extractTestFailureCount, subtractTestBaseline } from "./gate-executor";
8
+ import { typecheckFailures } from "./typecheck-gate";
9
+ import {
10
+ missingEvidenceViolations,
11
+ thresholdCrosscheckViolations,
12
+ } from "../lib/evidence-required";
13
+ import {
14
+ classifyContainerProvenance,
15
+ containerProvenanceViolations,
16
+ resolveContainerRuntime,
17
+ } from "../lib/container-provenance";
18
+ import { evidencePortabilityViolations } from "../lib/evidence-portability";
19
+ import { alreadySatisfiedIds } from "../lib/evidence-prevalidator";
20
+ import { readBranchDiff } from "../lib/branch-diff";
21
+ import { proveLabelGateVerdict, type LabelRead } from "../lib/prove-label-gate";
22
+ import { worktreeDiskIdentity, type RemoteReading } from "../lib/worktree-repo";
23
+ import { readSuiteMeasurement, type SuiteReading } from "../lib/suite-measurement";
24
+ import { briefAcAlignment } from "../lib/brief-ac-alignment";
25
+ import {
26
+ EXEC_TIMEOUT_MS,
27
+ GATE_TEST_TIMEOUT_MS,
28
+ TYPECHECK_TIMEOUT_MS,
29
+ } from "../lib/gate-timeouts";
30
+ import {
31
+ outcomeEvidenceViolations,
32
+ goalKeywordCoverage,
33
+ GOAL_COVERAGE_WARN_BELOW,
34
+ } from "../lib/ac-quality";
35
+
36
+ /**
37
+ * Every check in this file gets longer than the processes it starts (#550).
38
+ *
39
+ * File level and not per-test on purpose: `}, 120000);` on `tsc-pass` alone
40
+ * fixes one check, and the exec helpers below are called from dozens. The
41
+ * ordering against the spawns is owned by lib/gate-timeouts.ts and asserted
42
+ * in test/gate-suite-timeouts.test.ts.
43
+ */
44
+ setDefaultTimeout(GATE_TEST_TIMEOUT_MS);
45
+
46
+ const STANDALONE = !process.env.TEST_WORK_DIR;
47
+ const TEST_DIR = process.env.TEST_WORK_DIR || "/tmp/gate-viability-test";
48
+ const SF = `${TEST_DIR}/workflow-state.json`;
49
+
50
+ /**
51
+ * A path inside THIS run's directory — the one the gate handed us (#700).
52
+ *
53
+ * Six checks used to rebuild this directory a second way, from
54
+ * `process.env.RUNGATE_WORK_DIR` (or a `$HOME/.rungate` fallback) joined to the
55
+ * state's slug. That is the same answer derived twice, and the two can
56
+ * disagree: `bun test gates/` runs every file in this directory in ONE process,
57
+ * so a module-scope `process.env.RUNGATE_WORK_DIR = …` in any sibling file is a
58
+ * write to this one's environment, and which of them lands first is a
59
+ * filesystem walk order rather than a decision anybody made.
60
+ *
61
+ * `gate-executor.ts:1523` already passes the resolved directory as
62
+ * `TEST_WORK_DIR`, and `SF` above reads the state out of it. So the directory
63
+ * is not in doubt here; it only became doubtful when a check went and asked a
64
+ * global for it again.
65
+ *
66
+ * One binding, so there is one thing to mutate
67
+ * (`.claude/rules/checks-must-be-able-to-fail.md`). Shared, cross-run paths —
68
+ * `signals` — are deliberately NOT routed through this; they want the base.
69
+ */
70
+ function runArtifact(...parts: string[]): string {
71
+ return join(TEST_DIR, ...parts);
72
+ }
73
+
74
+ function loadRaw(): any | null {
75
+ if (!existsSync(SF)) return null;
76
+ return JSON.parse(readFileSync(SF, "utf-8"));
77
+ }
78
+
79
+ function sf(path: string): any {
80
+ const wf = loadRaw();
81
+ if (!wf) return undefined;
82
+ const parts = path.replace(/^\./, "").split(".").filter(Boolean);
83
+ let val: any = wf;
84
+ for (const p of parts) {
85
+ if (val == null) return undefined;
86
+ val = val[p];
87
+ }
88
+ return val;
89
+ }
90
+
91
+ function exec(cmd: string, cwd?: string): { ok: boolean; output: string } {
92
+ try {
93
+ const output = execSync(cmd, { cwd, encoding: "utf-8", timeout: EXEC_TIMEOUT_MS }).trim();
94
+ return { ok: true, output };
95
+ } catch (e: any) {
96
+ return { ok: false, output: (e.stdout ?? "").trim() };
97
+ }
98
+ }
99
+
100
+ /**
101
+ * Argv-form git, for commands carrying a value out of a config file.
102
+ *
103
+ * `exec` above goes through a shell, so `git status --porcelain -- ${paths}`
104
+ * hands consumer-authored strings to `sh`. `codeCommittedPaths` is read from
105
+ * the project's own `.claude/rungate.json`, and a path list is exactly the
106
+ * kind of field somebody pastes into without thinking about quoting. Argv
107
+ * form removes the shell rather than trying to sanitise for it.
108
+ *
109
+ * `--` is passed by the caller, so a pathspec beginning with `-` is a
110
+ * pathspec and not an option.
111
+ */
112
+ function execArgvOf(bin: string, args: string[], cwd?: string): { ok: boolean; output: string } {
113
+ try {
114
+ const output = execFileSync(bin, args, { cwd, encoding: "utf-8", timeout: EXEC_TIMEOUT_MS }).trim();
115
+ return { ok: true, output };
116
+ } catch (e: any) {
117
+ return { ok: false, output: `${e?.stdout ?? ""}${e?.stderr ?? ""}`.trim() || String(e?.message ?? e) };
118
+ }
119
+ }
120
+
121
+ function execArgv(args: string[], cwd?: string): { ok: boolean; output: string } {
122
+ try {
123
+ const output = execFileSync("git", args, { cwd, encoding: "utf-8", timeout: EXEC_TIMEOUT_MS }).trim();
124
+ return { ok: true, output };
125
+ } catch (e: any) {
126
+ return { ok: false, output: `${e?.stdout ?? ""}${e?.stderr ?? ""}`.trim() || String(e?.message ?? e) };
127
+ }
128
+ }
129
+
130
+ function loadProjectHarness(): any {
131
+ const root = sf("projectRoot");
132
+ if (!root) return null;
133
+ const p = join(root, ".claude", "rungate.json");
134
+ if (!existsSync(p)) return null;
135
+ return JSON.parse(readFileSync(p, "utf-8"));
136
+ }
137
+
138
+ /**
139
+ * #314 — the tree a check MEASURES, as opposed to where config is READ.
140
+ *
141
+ * `projectRoot` is the main checkout. On a worktree-isolated run the change
142
+ * under test is not in it, so a command executed there measures code that is
143
+ * already merged and green. Measured on run `wf_5cdce852-1e2`: `tsc-pass` ran
144
+ * the compiler in `projectRoot`, reported PASS, and CI then caught two real
145
+ * type errors on the branch.
146
+ *
147
+ * Returns `null` when the state does not name the tree. Every caller must
148
+ * refuse on `null` rather than substitute `projectRoot` — that substitution is
149
+ * the defect, and it is invisible because it always produces a pass.
150
+ */
151
+ export function treeUnderTestViolations(tree: unknown, checkName: string): string[] {
152
+ if (typeof tree !== "string" || tree.trim() === "") {
153
+ return [
154
+ `${checkName}: workflow state does not name treeUnderTest, so there is ` +
155
+ `nothing to measure. This is UNMEASURED, not PASS — measuring ` +
156
+ `projectRoot instead would check the main checkout, which is green by ` +
157
+ `construction and is not the change being shipped (#314).`,
158
+ ];
159
+ }
160
+ return [];
161
+ }
162
+
163
+ /**
164
+ * Does the tree have commits its upstream does not (#174)?
165
+ *
166
+ * The reading that matters is NOT just the count. `git rev-list --count
167
+ * @{upstream}..HEAD` fails outright when there is no upstream, and the old
168
+ * body read that failure as nothing-to-report:
169
+ *
170
+ * if (!result.ok) return; // no upstream
171
+ *
172
+ * A branch with no upstream has never been pushed, which is the strongest
173
+ * possible failure of a check named "no unpushed commits" — and it was the one
174
+ * input that passed unconditionally. Fail-open, shape one in
175
+ * .claude/rules/checks-must-be-able-to-fail.md, in the check whose whole job is
176
+ * to notice unpushed work.
177
+ *
178
+ * It is reachable, not theoretical. The happy path sets upstream
179
+ * (`workflows/ship.js:3161`, `git push -u origin ${pushTarget}`), but both
180
+ * remediation re-pushes do not — `:3324` and `:4704` run
181
+ * `git push origin HEAD:${shipBranch}`. So a run that needed remediation
182
+ * arrives here with no upstream and is waved through.
183
+ *
184
+ * Exported as the single binding this check's refusal is routed through, so
185
+ * `return []` is one edit and the mutation is a real measurement rather than a
186
+ * claim in a PR body.
187
+ */
188
+ export function unpushedViolations(
189
+ count: { ok: boolean; output: string },
190
+ upstream: { ok: boolean; output: string },
191
+ ): string[] {
192
+ if (count.ok) {
193
+ return count.output === "0" ? [] : [`code-pushed: ${count.output} commits ahead of upstream`];
194
+ }
195
+ if (!upstream.ok) {
196
+ return [
197
+ "code-pushed: the branch has no upstream, so nothing on it has been " +
198
+ "pushed. This is the WORST case for this check, not an exemption from " +
199
+ "it — the previous code returned here and passed (#174). A remediation " +
200
+ "re-push uses `git push origin HEAD:<branch>`, which does not set " +
201
+ "upstream, so this is reachable on any run that needed one.",
202
+ ];
203
+ }
204
+ return [
205
+ `code-pushed: the branch tracks ${upstream.output} but the commit count ` +
206
+ `could not be read. This is UNMEASURED, not PASS — a git invocation that ` +
207
+ `fails for an unknown reason says nothing about whether work is unpushed.`,
208
+ ];
209
+ }
210
+
211
+ /**
212
+ * #315 — `code-committed` has never executed.
213
+ *
214
+ * The body read `harness?.codeCommittedPaths || []` and returned early on an
215
+ * empty list. **Nothing authors that key.** Measured on main: neither
216
+ * rungate's `.claude/rungate/config.json`, nor its legacy
217
+ * `.claude/rungate.json`, nor DailyBriefDashboard's names it. So the check
218
+ * whose entire job is confirming the shipped work is committed hit the early
219
+ * return on every run this repository has ever made — shape one in
220
+ * .claude/rules/checks-must-be-able-to-fail.md, keyed to a config value that
221
+ * does not exist.
222
+ *
223
+ * It compounded with #314: the status also ran in `projectRoot`, the main
224
+ * checkout, which is clean during a worktree run. Both halves had to be
225
+ * fixed or the check means nothing; #314 fixed the tree, this fixes the scope.
226
+ *
227
+ * ## The scope decision (SC-E2), recorded here rather than left to infer
228
+ *
229
+ * The key becomes OPTIONAL WITH A DERIVED DEFAULT rather than required.
230
+ *
231
+ * Requiring it would refuse at scaffold time for every consumer that has
232
+ * never heard of it — an outage for a rule nobody opted into, which is the
233
+ * same reasoning `loadConformityAllowlists` and #302's growth check already
234
+ * apply. And the default is not a guess: for a check named "no uncommitted
235
+ * changes in project", the honest scope is **the whole tree**, and a
236
+ * configured path list is a narrowing a consumer opts into, not the thing
237
+ * that switches the check on.
238
+ *
239
+ * Untracked files are included, because `--porcelain` already honours
240
+ * `.gitignore` and a source file that was never added is exactly the
241
+ * uncommitted work this check exists to catch. Build artefacts that trip it
242
+ * belong in `.gitignore`; that is the right pressure rather than a reason to
243
+ * narrow the check.
244
+ *
245
+ * Exported as the single binding this check's refusal is routed through, so
246
+ * `return []` is one edit and the mutation is a measurement rather than a
247
+ * claim in a PR body.
248
+ */
249
+ export function codeCommittedViolations(
250
+ status: { ok: boolean; output: string },
251
+ scope: { configured: boolean; paths: string[] },
252
+ ): string[] {
253
+ if (!status.ok) {
254
+ return [
255
+ `code-committed: \`git status\` could not be read in the tree under test ` +
256
+ `(${status.output || "no output"}). This is UNMEASURED, not PASS — a git ` +
257
+ `invocation that fails for an unknown reason says nothing about whether the ` +
258
+ `work is committed.`,
259
+ ];
260
+ }
261
+ const dirty = status.output.split("\n").map(l => l.trim()).filter(Boolean);
262
+ if (dirty.length === 0) return [];
263
+ const where = scope.configured ? ` within ${scope.paths.join(", ")}` : "";
264
+ const shown = dirty.slice(0, 10).join("; ");
265
+ return [
266
+ `code-committed: ${dirty.length} uncommitted change(s) in the tree under test${where}: ` +
267
+ `${shown}${dirty.length > 10 ? ` … and ${dirty.length - 10} more` : ""}`,
268
+ ];
269
+ }
270
+
271
+ /**
272
+ * #414 — the worktree-repository check, read from the disk rather than from
273
+ * the agent standing in it.
274
+ *
275
+ * #218 refuses a worktree whose `origin` does not match the run's repo, and
276
+ * the remote it compares is SELF-REPORTED: `workflows/ship.js` has no
277
+ * subprocess (#69), so the only remote it can see is the one the agent said
278
+ * it read. An agent that answers from its brief rather than from
279
+ * `git remote get-url origin` produces a reply indistinguishable from a
280
+ * correct one, and `lib/worktree-repo.ts` has exported the unforgeable half
281
+ * since #218 with nothing able to call it. A gate can: gates run as real
282
+ * subprocesses with the worktree on disk.
283
+ *
284
+ * Exported as the single binding this check's refusal is routed through, so
285
+ * `return []` is one edit and the mutation in
286
+ * `test/suite-binding-mutation.test.ts` is a measurement rather than a claim
287
+ * in a PR body. Keep it one function with one body — a second refusal path
288
+ * for the same fact would survive that mutation.
289
+ *
290
+ * AN EMPTY LIST REFUSES. Every run this fires on has at least a tree under
291
+ * test, so zero entries means the state stopped naming them, and a check that
292
+ * scans nothing reports nothing and is indistinguishable from compliance.
293
+ */
294
+ export function worktreeRepoViolations(entries: unknown, expectedRepo: unknown, read?: (p: string) => RemoteReading): string[] {
295
+ if (!Array.isArray(entries) || entries.length === 0) {
296
+ return [
297
+ "worktree-repo: the workflow state names no worktree to read, so no checkout " +
298
+ "was compared against this run's repository. This is UNMEASURED, not PASS — " +
299
+ "every run reaches this gate with at least a tree under test, so an empty " +
300
+ "list means the state stopped recording them (#414).",
301
+ ];
302
+ }
303
+ const out: string[] = [];
304
+ const seen = new Set<string>();
305
+ for (const raw of entries) {
306
+ const e = (raw ?? {}) as { path?: unknown; reportedRepo?: unknown };
307
+ const path = typeof e.path === "string" ? e.path.trim() : "";
308
+ // One worktree, one reading. A worktree that reported in two parts (#228)
309
+ // appears twice, and two identical refusals for it is noise an operator
310
+ // has to deduplicate by hand. Keyed on the empty string too, so a run of
311
+ // pathless rows refuses once.
312
+ if (seen.has(path)) continue;
313
+ seen.add(path);
314
+ const reported = typeof e.reportedRepo === "string" ? e.reportedRepo : null;
315
+ const got = worktreeDiskIdentity(
316
+ path,
317
+ typeof expectedRepo === "string" ? expectedRepo : null,
318
+ reported,
319
+ read,
320
+ );
321
+ if (got.refusal) out.push(`worktree-repo: ${got.refusal}`);
322
+ }
323
+ return out;
324
+ }
325
+
326
+ /** The tree to run commands in, or null when the state does not name one. */
327
+ function treeRoot(): string | null {
328
+ const t = sf("treeUnderTest");
329
+ return typeof t === "string" && t.trim() !== "" ? t : null;
330
+ }
331
+
332
+ const isVerifyPlus = () => { const p = sf("phase"); return ["VERIFY", "SHIP", "DONE"].includes(p); };
333
+ const isShipPlus = () => { const p = sf("phase"); return ["SHIP", "DONE"].includes(p); };
334
+ const isDone = () => sf("phase") === "DONE";
335
+
336
+ /**
337
+ * #224 — the suite-to-verdict binding.
338
+ *
339
+ * `environments.local.tests` is the only reading this harness has of whether
340
+ * the suite passed on the tree being shipped, and this is the single site that
341
+ * turns that reading into a refusal. Keep it one function with one body: a
342
+ * second refusal path for the same fact would survive the mutation in
343
+ * test/suite-binding-mutation.test.ts and make the proof vacuous, which is why
344
+ * that file asserts this signature appears exactly once and aborts if it moves.
345
+ *
346
+ * Absent is a refusal, not a pass. That is the whole of #224: the run that
347
+ * reported `regressions: 0` over one failing test had measured nothing, and
348
+ * nothing was read as zero.
349
+ *
350
+ * #530 added the second argument. `environments.local.tests` is an enum with
351
+ * no provenance in it, and until #530 nothing in the harness wrote it — the
352
+ * recorder wrote `suiteMeasurement` and the field was left to a sentence in
353
+ * a heal prompt that only runs when the scope gate fails. Now the recorder
354
+ * writes both in one write, and this reads both: a PASS the measurement does
355
+ * not back is refused, which is the hand-written verdict the ship prompt has
356
+ * only ever forbidden in prose.
357
+ *
358
+ * The clauses do not overlap. An absent field, a `FAIL`, and a declared
359
+ * `SKIP` are each decided before the measurement is consulted at all, and
360
+ * `{tests: "PASS"}` with no `suiteMeasurement` is reached by nothing else.
361
+ *
362
+ * The second argument is REQUIRED, and that is the point of it. Optional, it
363
+ * would be a check any future caller could lose by omission — and losing it
364
+ * is silent, because the three clauses above still refuse everything they
365
+ * refused before, so the call reads exactly like a call that is checking.
366
+ * `readSuiteMeasurement` is a total function that returns UNMEASURED for
367
+ * every unusable input, so there is never a caller with nothing to pass.
368
+ *
369
+ * **What the requirement buys is that the absence now fails CLOSED.** With an
370
+ * optional argument, omitting it meant "no objection". Required, the only
371
+ * thing a caller can hand this for a state with no measurement is an
372
+ * UNMEASURED reading, and `UNMEASURED !== "PASS"` refuses. The failure mode
373
+ * inverted: absence used to be silence and is now a refusal. That is
374
+ * `.claude/rules/config-access.md`'s Silent Default Substitution at a second
375
+ * site — `0`, `{}`, `null` and `SKIP` are indistinguishable from success;
376
+ * `UNMEASURED` is not.
377
+ */
378
+ export function suiteVerdictViolations(val: unknown, measurement: SuiteReading): string[] {
379
+ if (val === undefined || val === null || val === "") {
380
+ return [
381
+ "local.tests not set — PASS or SKIP required when dev.testCmd is configured. " +
382
+ "An unmeasured suite is not a passing suite (#224).",
383
+ ];
384
+ }
385
+ if (val === "SKIP") return []; // SKIP is allowed — it says so, rather than implying PASS
386
+ if (val !== "PASS") return [`tests: ${String(val)} — expected PASS`];
387
+ // A PASS has to be backed by the measurement that produced it. No
388
+ // `measurement &&` guard: a branch for an argument that cannot be absent
389
+ // is a branch no test can enter.
390
+ if (measurement.verdict !== "PASS") {
391
+ return [
392
+ `tests: PASS, but the suite measurement behind it reads ${measurement.verdict} — ` +
393
+ `${measurement.reason || "the two disagree"} (#530). A verdict no measurement backs is a ` +
394
+ `verdict somebody typed.`,
395
+ ];
396
+ }
397
+ return [];
398
+ }
399
+
400
+ /**
401
+ * #126 — the Verify fan-out leaves a trace, or the gate refuses.
402
+ *
403
+ * The two checks below used to read their slot and, finding nothing, warn and
404
+ * return:
405
+ *
406
+ * if (!ev) { console.warn("...not populated — B2 agent may not have run"); return; }
407
+ *
408
+ * A test that returns early passes. The only input either could fail on was a
409
+ * slot written WRONG, so the failure mode this exists for — the agent never
410
+ * ran — was the single case they could not catch. "When populated" was in
411
+ * both names.
412
+ *
413
+ * The producer reached not-populated four ways, three of them silent: the
414
+ * `fails === 0` guard in gates/gate-executor.ts (a verify gate with any
415
+ * failure skips the agents and then reports their absence, which reads as the
416
+ * cause when it is the consequence), `RUNGATE_SKIP_AGENTS`, a LIGHT tier, and
417
+ * a missing prompt file. So the fan-out could be absent in full and the gate
418
+ * said PASS, which makes every compliance number recorded for Verify a number
419
+ * for a stage that may not have run.
420
+ *
421
+ * One function, one body, for the same reason `suiteVerdictViolations` above
422
+ * is: a second refusal path for the same fact would survive the mutation in
423
+ * test/suite-binding-mutation.test.ts and make the proof vacuous.
424
+ *
425
+ * A DELIBERATE skip passes, and must be written down as one. Refusing on
426
+ * every absence would brick LIGHT runs, and a fix that brings its own outage
427
+ * is how the next fail-open gets argued for. But a skip with no reason is an
428
+ * absence with extra steps, and it would be the cheapest way to make this
429
+ * whole check green again — so it refuses.
430
+ */
431
+ export function verifyFanoutViolations(label: string, val: unknown, tier: unknown): string[] {
432
+ // The producer tier-gates B1 at gates/gate-executor.ts:1093, so a LIGHT run
433
+ // legitimately has no record. This is the one absence that is expected.
434
+ if (tier === "LIGHT") return [];
435
+
436
+ if (val === undefined || val === null || val === "") {
437
+ return [
438
+ `gates.verify.${label} not recorded — the agent left no trace, and an absence is ` +
439
+ `not an approval (#126). A deliberate skip must be recorded as ` +
440
+ `{"skipped": "<reason>"} so the artefact can tell "it approved" from "it never ran".`,
441
+ ];
442
+ }
443
+ if (typeof val !== "object") {
444
+ return [`gates.verify.${label} is ${typeof val}, expected a record (#126)`];
445
+ }
446
+
447
+ const skipped = (val as { skipped?: unknown }).skipped;
448
+ if (skipped !== undefined) {
449
+ return typeof skipped === "string" && skipped.trim().length > 0
450
+ ? []
451
+ : [
452
+ `gates.verify.${label} records a skip with no reason — a skip nobody can read ` +
453
+ `is an absence with extra steps (#126)`,
454
+ ];
455
+ }
456
+ return [];
457
+ }
458
+
459
+ /**
460
+ * #235 — the evidence pre-validation reading reaches the verdict.
461
+ *
462
+ * `lib/evidence-prevalidator.ts` dry-runs every AC evidence command at SCOPE
463
+ * and classifies each one `ok` / `broken` / `empty` / `skipped`. Until this
464
+ * function existed the entire result went to `console.error` and nowhere else:
465
+ * a run could be told, in its own log, that three of its ACs could not be
466
+ * measured, and still ship — the detection was never the gap.
467
+ *
468
+ * This is the single site that turns that reading into a refusal, for the same
469
+ * reason `suiteVerdictViolations` above is: a second refusal path for the same
470
+ * fact would survive the mutation in test/suite-binding-mutation.test.ts and
471
+ * make the proof vacuous. That file asserts this signature appears exactly once
472
+ * and aborts when it moves.
473
+ *
474
+ * Absent is a refusal, not a pass. An unmeasured pre-validation says nothing
475
+ * about the evidence commands, and "nothing" has already been read as "clean"
476
+ * once in this harness (#224).
477
+ */
478
+ export function prevalidationViolations(val: unknown, gatePhase?: unknown): string[] {
479
+ const unmeasured = [
480
+ "evidencePrevalidation not recorded — a PASS reading is required once any AC " +
481
+ "carries an evidence command. An unmeasured pre-validation is not a clean one (#235).",
482
+ ];
483
+
484
+ if (val === undefined || val === null || val === "") return unmeasured;
485
+ if (typeof val !== "object") {
486
+ return [`evidencePrevalidation is ${typeof val}, expected a record with a verdict (#235)`];
487
+ }
488
+
489
+ // #280 — a reading whose own `takenAt` says it describes an earlier tree is
490
+ // not evidence about this one.
491
+ //
492
+ // The producer is supposed to re-measure at ship, and this is what makes a
493
+ // producer that stops doing so a REFUSAL rather than a stale artefact
494
+ // recorded as current. That distinction is the whole of #280: the ship gate
495
+ // still refused, correctly and for the right reason, while filing a block
496
+ // stamped `takenAt: "verify"` and an hour old as its own reading.
497
+ //
498
+ // Only at ship, and deliberately only there. A SCOPE reading survives to
499
+ // verify by design (#264) — re-measuring the run's starting conditions
500
+ // after Marcus has run answers a different question — so there is no
501
+ // general "earlier phase is stale" rule to apply here, and inventing one
502
+ // would turn every LIGHT run red.
503
+ const ACCEPTED_AT_SHIP = "ship";
504
+ if (gatePhase === "SHIP" || gatePhase === "DONE") {
505
+ const source = (val as { takenAt?: unknown }).takenAt;
506
+ if (source !== ACCEPTED_AT_SHIP) {
507
+ return [
508
+ `evidence pre-validation was taken at ${source ? `\`${String(source)}\`` : "an unrecorded phase"}, ` +
509
+ "not at ship — the ship gate must measure its own tree. Between verify and ship the " +
510
+ "run commits, pushes, and may move the branch (#280).",
511
+ ];
512
+ }
513
+ }
514
+
515
+ const reading = val as { verdict?: unknown; acs?: unknown };
516
+ const verdict = reading.verdict;
517
+ if (verdict === undefined || verdict === null || verdict === "" || verdict === "UNMEASURED") {
518
+ return unmeasured;
519
+ }
520
+
521
+ // #295 — derived from `acs`, which is what the producer writes and what
522
+ // `EvidencePrevalidationShape` declares. This read `reading.broken`, a key
523
+ // no writer has ever produced and the schema has never declared, so the list
524
+ // was empty on every run: every FAIL fell through to the "no broken AC ids
525
+ // were recorded" branch below, and the refusal whose job is to say what to
526
+ // fix could not say it.
527
+ //
528
+ // `status === "broken"` and nothing else. `empty` is a warning rather than a
529
+ // refusal (lib/evidence-prevalidator.ts), and `skipped` never ran — folding
530
+ // either in here would turn a FAIL that names two ACs into one that names
531
+ // five, which is the same unusable refusal from the other direction.
532
+ const broken = (Array.isArray(reading.acs) ? reading.acs : [])
533
+ .filter((ac: unknown) => (ac as { status?: unknown })?.status === "broken")
534
+ .map((ac: unknown) => String((ac as { id?: unknown })?.id ?? ""))
535
+ .filter((id: string) => id.length > 0);
536
+
537
+ // #669 — an AC whose PRE-CHANGE output already cleared its own threshold
538
+ // cannot observe the work. It printed a satisfying value before a line was
539
+ // written, so it will print one afterwards whatever Marcus does: a check
540
+ // that cannot fail, arriving in the run's own acceptance criteria.
541
+ //
542
+ // Derived through `alreadySatisfiedIds`, the same binding the writer and
543
+ // the schema use, so the refusal cannot name a different set than the
544
+ // reading carries (#295 is the cost of getting that wrong).
545
+ //
546
+ // ONLY on a reading taken at scope, and the restriction is the point. Scope
547
+ // is the one phase whose tree is the pre-change one. At verify and at ship
548
+ // the work has landed, so an AC that satisfies its threshold there is an AC
549
+ // that PASSED — refusing those would turn every successful run red, which
550
+ // is the fastest way to build a refusal everyone learns to click past. A
551
+ // reading with no `takenAt` predates the field and cannot say which tree it
552
+ // describes, so it is not refused either.
553
+ const vacuous =
554
+ (val as { takenAt?: unknown }).takenAt === "scope" ? alreadySatisfiedIds(reading.acs) : [];
555
+ const vacuousRefusal =
556
+ vacuous.length > 0
557
+ ? [
558
+ `evidence pre-validation: ${vacuous.join(", ")} already satisfied their own thresholds ` +
559
+ "on the pre-change tree — an AC that passes before the work is done certifies the " +
560
+ "change without observing it (#669)",
561
+ ]
562
+ : [];
563
+
564
+ if (verdict === "FAIL") {
565
+ return [
566
+ ...(broken.length > 0
567
+ ? [`evidence pre-validation: FAIL — broken evidence commands on ${broken.join(", ")}`]
568
+ : [
569
+ "evidence pre-validation: FAIL — but no broken AC ids were recorded, so the " +
570
+ "refusal cannot name what to fix (#235)",
571
+ ]),
572
+ ...vacuousRefusal,
573
+ ];
574
+ }
575
+
576
+ if (verdict !== "PASS") {
577
+ return [`evidence pre-validation: ${String(verdict)} — expected PASS`];
578
+ }
579
+
580
+ // A PASS carrying broken ids is a contradiction, not a pass. Without this the
581
+ // writer could record the broken list faithfully and still be waved through.
582
+ if (broken.length > 0) {
583
+ return [
584
+ `evidence pre-validation: PASS recorded alongside broken evidence commands on ${broken.join(", ")}`,
585
+ ...vacuousRefusal,
586
+ ];
587
+ }
588
+ return vacuousRefusal;
589
+ }
590
+
591
+ // ═══ SCHEMA VALIDATION ════════════════════════════════════════════════
592
+ describe("schema validation", () => {
593
+ if (STANDALONE || !existsSync(SF)) {
594
+ test("skipped — no active workflow (standalone run)", () => {});
595
+ return;
596
+ }
597
+ test("workflow-state.json validates (includes threshold + behavioral superRefine)", () => {
598
+ const raw = loadRaw();
599
+ const result = WorkflowStateSchema.passthrough().safeParse(raw);
600
+ if (!result.success) {
601
+ const issues = result.error.issues.map(i => `${i.path.join(".")}: ${i.message}`);
602
+ // Narrowed to `false` inside this branch — widen so the always-fail
603
+ // assertion still reports the collected issues as its message.
604
+ expect<boolean>(result.success, issues.join("\n")).toBe(true);
605
+ }
606
+ });
607
+ });
608
+
609
+ // ═══ SCOPE CHECKS (11 functions) ══════════════════════════════════════
610
+ describe("scope checks", () => {
611
+ if (STANDALONE || !existsSync(SF)) {
612
+ test("skipped — no active workflow (standalone run)", () => {});
613
+ return;
614
+ }
615
+ // 1. acs-exist
616
+ test("acs-exist: at least one AC", () => {
617
+ expect(sf("acs")?.length ?? 0).toBeGreaterThan(0);
618
+ });
619
+
620
+ // 4. acs-have-evidence-method
621
+ test("acs-have-evidence-method", () => {
622
+ const missing = (sf("acs") || []).filter((ac: any) => !ac.evidenceMethod);
623
+ expect(missing.map((ac: any) => ac.id)).toEqual([]);
624
+ });
625
+
626
+ // 5. acs-measurable
627
+ test("acs-measurable: thresholds have op and value", () => {
628
+ const bad = (sf("acs") || []).filter((ac: any) => ac.threshold && (!ac.threshold.op || ac.threshold.value == null));
629
+ expect(bad.map((ac: any) => ac.id)).toEqual([]);
630
+ });
631
+
632
+ // 6. source-spec-cited (GAP → PORTED)
633
+ test("source-spec-cited: all sourceSpecs have citedInDiscovery=true", () => {
634
+ const specs = sf("sourceSpecs") || [];
635
+ const uncited = specs.filter((s: any) => s.citedInDiscovery === false);
636
+ expect(uncited.map((s: any) => s.path)).toEqual([]);
637
+ });
638
+
639
+ // 7. spec-elements-in-acs (GAP → PORTED)
640
+ test("spec-elements-in-acs: if specElements exist, at least one AC references them", () => {
641
+ const specs = sf("sourceSpecs") || [];
642
+ const elements = specs.flatMap((s: any) => s.specElements || []);
643
+ if (elements.length === 0) return;
644
+ const acsWithSpec = (sf("acs") || []).filter((ac: any) => ac.specElement);
645
+ expect(acsWithSpec.length).toBeGreaterThan(0);
646
+ });
647
+
648
+ // 8. sizing-declared
649
+ test("sizing-declared", () => {
650
+ expect(sf("sizing")?.predicted).toBeDefined();
651
+ });
652
+
653
+ // 9. issue-goal-captured
654
+ test("issue-goal-captured", () => {
655
+ expect((sf("issueGoal") || "").length).toBeGreaterThan(0);
656
+ });
657
+
658
+ // 10. acs-no-skillmd-enforcement (GAP → PORTED)
659
+ test("acs-no-skillmd-enforcement: no ACs target SKILL.md", () => {
660
+ const pattern = /SKILL\.md/i;
661
+ const hits = (sf("acs") || []).filter((ac: any) =>
662
+ pattern.test(ac.statement || "") || pattern.test(ac.evidenceMethod?.command || "")
663
+ );
664
+ expect(hits.map((ac: any) => ac.id)).toEqual([]);
665
+ });
666
+
667
+ // ac-issue-alignment: ACs cover issueGoal keywords (ADR-009 C4)
668
+ //
669
+ // #423: stays a WARNING, and that is now a decision with a number behind it
670
+ // rather than a deferral. Measured over the 153 workflow states under
671
+ // `~/.rungate` carrying a goal, 71% fall under this threshold — 66% of the
672
+ // 131 whose ACs carry real statements, 73% of the 101 with substantial ones.
673
+ // A check that would refuse two runs in three is an outage, not a gate. The
674
+ // reading moved to `goalKeywordCoverage` so the threshold and the vocabulary
675
+ // can be driven from both sides by a test; what did NOT change is that
676
+ // nothing here refuses.
677
+ test("ac-issue-alignment: ACs cover goal keywords", () => {
678
+ const read = goalKeywordCoverage(sf("issueGoal"), sf("acs"));
679
+ if (read === null) return;
680
+ if (read.coverage < GOAL_COVERAGE_WARN_BELOW) {
681
+ console.warn(
682
+ `WARN: ac-issue-alignment: only ${Math.round(read.coverage * 100)}% goal keyword coverage ` +
683
+ `(${read.covered.length}/${read.keywords.length})`,
684
+ );
685
+ }
686
+ });
687
+
688
+ // 11. sourceSpecs-required (#473)
689
+ test("sourceSpecs-required: non-LIGHT tier must have sourceSpecs", () => {
690
+ const tier = sf("sizing.ceremonyTier");
691
+ if (tier === "LIGHT") return;
692
+ const specs = sf("sourceSpecs") || [];
693
+ expect(specs.length, "STANDARD/THOROUGH tier must have at least one sourceSpec").toBeGreaterThanOrEqual(1);
694
+ });
695
+
696
+ // 11b. before-state-captured (#1411)
697
+ test("before-state-captured: STANDARD+ must have beforeState in workflow-state", () => {
698
+ const tier = sf("sizing.ceremonyTier");
699
+ const VALID_TYPES = ["text", "screenshot", "api", "none"];
700
+ const bs = sf("beforeState");
701
+ if (!bs) {
702
+ if (tier === "LIGHT") return;
703
+ console.warn("WARN: before-state-captured: beforeState missing for STANDARD+ tier — goal skill should capture before-state (#1411)");
704
+ return;
705
+ }
706
+ expect(VALID_TYPES.includes(bs.type), `beforeState.type must be text|screenshot|api|none, got "${bs.type}"`).toBe(true);
707
+ if (tier !== "LIGHT" && bs.type === "none") {
708
+ console.warn('WARN: before-state-captured: beforeState.type is "none" for STANDARD+ tier — consider capturing actual before-state');
709
+ }
710
+ });
711
+
712
+ // 12. discovery-evidence (#473)
713
+ test("discovery-evidence: non-LIGHT tiers have cited sourceSpecs", () => {
714
+ const tier = sf("sizing.ceremonyTier");
715
+ if (tier === "LIGHT") return;
716
+ const specs = sf("sourceSpecs") || [];
717
+ const cited = specs.filter((s: any) => s.citedInDiscovery);
718
+ expect(cited.length, "No sourceSpecs with citedInDiscovery — DISCOVERY may have been skipped").toBeGreaterThanOrEqual(1);
719
+ });
720
+
721
+ // 13. governing-spec-exists (#473)
722
+ test("governing-spec-exists: spec file exists on disk", () => {
723
+ const tier = sf("sizing.ceremonyTier");
724
+ if (tier === "LIGHT") return;
725
+ const specs = sf("sourceSpecs") || [];
726
+ if (specs.length === 0) return; // caught by sourceSpecs-required
727
+ const specPath = specs[0].path;
728
+ // #314 — the spec is resolved against the TREE UNDER TEST. A spec added on
729
+ // the branch does not exist in the main checkout, so resolving against
730
+ // projectRoot failed a run for writing its own governing spec; a spec
731
+ // deleted on the branch still exists in main, so the same read passed a
732
+ // run that had removed the thing it claimed to be governed by.
733
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "governing-spec-exists");
734
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
735
+ const root = treeRoot();
736
+ const resolved = specPath.startsWith("/") ? specPath : join(root!, specPath);
737
+ expect(existsSync(resolved), `Governing spec not found: ${resolved}`).toBe(true);
738
+ });
739
+
740
+ // 14. harness-port-validation: rungate.json ports match Makefile
741
+ test("harness-port-validation: dev/test/prod ports match Makefile", () => {
742
+ // #314 — this check's SUBJECT is whether two files in the tree agree, so
743
+ // it reads both from the tree under test. It is not a config read: a
744
+ // branch that edits a port in either file would otherwise be graded
745
+ // against the main checkout's copies, where the two already agree.
746
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "harness-port-validation");
747
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
748
+ const projectRoot = treeRoot();
749
+ if (!projectRoot) return;
750
+ const harnessPath = join(projectRoot, ".claude", "rungate.json");
751
+ const makefilePath = join(projectRoot, "Makefile");
752
+ if (!existsSync(harnessPath) || !existsSync(makefilePath)) return;
753
+ // Named treeHarness, not harness: `harness` means the loadProjectHarness()
754
+ // result everywhere else in this file, and test/gate-optional-config-keys.test.ts
755
+ // enumerates reads off it so no gate check can gate on an unexplained config
756
+ // key. A second const with the same name reading a different file would be
757
+ // invisible to that scan — which is how the scan was first written, and what
758
+ // the security review on #315 caught.
759
+ const treeHarness = JSON.parse(readFileSync(harnessPath, "utf-8"));
760
+ const makefile = readFileSync(makefilePath, "utf-8");
761
+
762
+ const warnings: string[] = [];
763
+
764
+ // Check dev port — match PORT= in the dev-all/dev section of Makefile
765
+ const devPort = treeHarness?.dev?.apiBase?.match(/:(\d+)/)?.[1];
766
+ if (devPort) {
767
+ const devSection = makefile.match(/dev.*environment[^]*?PORT=(\d+)/i);
768
+ const makeDevPort = devSection?.[1];
769
+ if (makeDevPort && devPort !== makeDevPort) {
770
+ warnings.push(`dev.apiBase port ${devPort} != Makefile dev PORT ${makeDevPort}`);
771
+ }
772
+ }
773
+
774
+ // Check prod port — match PORT= in container run commands
775
+ const prodPort = treeHarness?.prod?.apiBase?.match(/:(\d+)/)?.[1];
776
+ if (prodPort) {
777
+ const makeProdPort = makefile.match(/-e PORT=(\d+)/)?.[1];
778
+ if (makeProdPort && prodPort !== makeProdPort) {
779
+ warnings.push(`prod.apiBase port ${prodPort} != Makefile container PORT ${makeProdPort}`);
780
+ }
781
+ }
782
+
783
+ if (warnings.length > 0) {
784
+ console.warn(`WARN: harness-port-validation: ${warnings.join("; ")}`);
785
+ }
786
+ });
787
+ });
788
+
789
+ // ═══ VERIFY CHECKS (16 functions) ═════════════════════════════════════
790
+ describe("verify checks", () => {
791
+ if (STANDALONE || !existsSync(SF)) {
792
+ test("skipped — no active workflow (standalone run)", () => {});
793
+ return;
794
+ }
795
+ // 12. local-before-prod
796
+ test("local-before-prod: local validated before prod rebuild", () => {
797
+ const prodRebuild = sf("environments")?.prod?.rebuild;
798
+ if (!prodRebuild || prodRebuild === "SKIP") return;
799
+ const localApi = sf("environments")?.local?.api;
800
+ const localTests = sf("environments")?.local?.tests;
801
+ if (localApi !== "PASS" && localTests !== "PASS") {
802
+ expect("prod deployed without local validation").toBe("local.api or local.tests = PASS first");
803
+ }
804
+ });
805
+
806
+ // 13. container-has-fix — OCI label SHA comparison (#437)
807
+ test("container-has-fix: container image SHA matches HEAD", () => {
808
+ if (!isShipPlus()) return;
809
+ const harness = loadProjectHarness();
810
+ if (!harness?.prod?.rebuild) return;
811
+ const prodRebuild = sf("environments")?.prod?.rebuild;
812
+ // #314: the container SHA is compared against the tree under test's HEAD, not main's.
813
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "container-has-fix");
814
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
815
+ const root = treeRoot();
816
+ if (!root) return;
817
+ const containerName = harness?.prod?.containerName || harness?.prod?.imageName;
818
+ if (!containerName) return;
819
+
820
+ // #341: four early returns used to share one observable outcome — PASS.
821
+ // Container absent, daemon down, no revision label, no runtime installed.
822
+ // Two of them are MORE likely exactly when the check matters, so it was
823
+ // quietest in the situation it exists to catch.
824
+ //
825
+ // Each is now a named outcome, recorded. Which of them should BLOCK is
826
+ // #341 criterion 1's question — it needs the frequency of each measured on a
827
+ // real consumer first, because refusing on a runtime that happens to be
828
+ // down turns real runs red. Only a measured BEHIND blocks today, which
829
+ // is what blocked before.
830
+ const runtime = resolveContainerRuntime(
831
+ harness?.prod?.containerRuntime,
832
+ name => exec(`command -v ${name}`).ok,
833
+ );
834
+ // Argv form: containerName comes out of the consumer's rungate.json and
835
+ // the old line interpolated it into a shell command (#315 security review).
836
+ const inspect =
837
+ runtime.kind === "resolved"
838
+ ? execArgvOf(runtime.runtime, [
839
+ "inspect",
840
+ "--format={{index .Config.Labels \"org.opencontainers.image.revision\"}}",
841
+ "--",
842
+ containerName,
843
+ ])
844
+ : undefined;
845
+ const head = exec("git rev-parse HEAD", root);
846
+
847
+ const verdict = classifyContainerProvenance({
848
+ runtime,
849
+ containerName,
850
+ // execArgvOf folds stderr into `output` on failure, which is the text
851
+ // classifyInspectFailure needs to tell absence from a dead daemon.
852
+ inspect: inspect ? { ok: inspect.ok, output: inspect.ok ? inspect.output : "", stderr: inspect.ok ? "" : inspect.output } : undefined,
853
+ head: { ok: head.ok, output: head.output },
854
+ prodRebuild,
855
+ });
856
+
857
+ // criterion 3: surfaced, not swallowed. One line per run naming which of the
858
+ // outcomes was reached, so criterion 1's frequencies are collectable from the
859
+ // logs of real runs rather than needing new instrumentation later.
860
+ console.log(
861
+ `container-has-fix: ${verdict.outcome} (${verdict.measured ? "measured" : "UNMEASURED"}) — ${verdict.detail}`,
862
+ );
863
+
864
+ const violations = containerProvenanceViolations(verdict);
865
+ expect(violations, violations.join("\n")).toEqual([]);
866
+ });
867
+
868
+ // 14. all-acs-have-evidence (GAP → PORTED)
869
+ // OUTCOME ACs may not have mechanical evidence at ship time — verified at prove via Quinn
870
+ test("all-acs-have-evidence: every AC has evidence in verify+", () => {
871
+ if (!isVerifyPlus()) return;
872
+ const missing = (sf("acs") || []).filter((ac: any) => ac.evidence == null && ac.type !== "OUTCOME");
873
+ expect(missing.map((ac: any) => ac.id)).toEqual([]);
874
+ });
875
+
876
+ // 15. all-acs-pass
877
+ // OUTCOME ACs may be PENDING at ship time — they require Quinn/prove for verification
878
+ test("all-acs-pass: every AC has verdict in ship+", () => {
879
+ if (!isShipPlus()) return;
880
+ const noVerdict = (sf("acs") || []).filter((ac: any) => (!ac.verdict || ac.verdict === "PENDING") && ac.type !== "OUTCOME");
881
+ expect(noVerdict.map((ac: any) => ac.id)).toEqual([]);
882
+ });
883
+
884
+ // 15b. outcome-acs-no-skip: OUTCOME ACs must be verified, not skipped (#90 RCA)
885
+ test("outcome-acs-no-skip: OUTCOME ACs cannot have SKIP verdict at ship+", () => {
886
+ if (!isShipPlus()) return;
887
+ const skipped = (sf("acs") || []).filter((ac: any) => ac.type === "OUTCOME" && ac.verdict === "SKIP");
888
+ expect(skipped.map((ac: any) => ac.id), "OUTCOME ACs require live verification — start dev server and run the feature (#90 RCA)").toEqual([]);
889
+ });
890
+
891
+ // 15c. outcome-evidence-behavioral: OUTCOME evidence is not just a grep count (#90 RCA)
892
+ //
893
+ // #423: this REFUSES now. It was `tsc-pass`'s shape — a name promising a
894
+ // verdict over a body that could only `console.warn` — while its sibling
895
+ // two tests above it has always asserted. An OUTCOME AC records that a
896
+ // user-visible outcome was observed, and a bare number cannot establish
897
+ // one; `0` is indistinguishable from success, which is the anti-pattern
898
+ // `.claude/rules/config-access.md` names as Silent Default Substitution.
899
+ test("outcome-evidence-behavioral: OUTCOME AC evidence is not just a number", () => {
900
+ if (!isShipPlus()) return;
901
+ const violations = outcomeEvidenceViolations(sf("acs"));
902
+ expect(
903
+ violations,
904
+ "OUTCOME ACs need evidence a reader can check — a bare measurement says a number " +
905
+ "was produced, not that the outcome happened (#90 RCA, #423)",
906
+ ).toEqual([]);
907
+ });
908
+
909
+ // #339: renamed from the prose title "no FAIL verdicts in DONE". A gate
910
+ // check with a prose name cannot be referenced from a ceremony profile at
911
+ // all, so it was structurally undocumentable rather than merely
912
+ // undocumented. Referenced by nothing else; the rename is local.
913
+ test("no-fail-verdicts-in-done: no AC is FAIL once the run is DONE", () => {
914
+ if (!isDone()) return;
915
+ const fails = (sf("acs") || []).filter((ac: any) => ac.verdict === "FAIL");
916
+ expect(fails.map((ac: any) => ac.id)).toEqual([]);
917
+ });
918
+
919
+ // 16. evidence-type-valid (GAP → PORTED)
920
+ test("evidence-type-valid: evidence.type matches allowed enum", () => {
921
+ if (!isVerifyPlus()) return;
922
+ const VALID = ["file-citation", "grep-output", "api-response", "screenshot", "test-output", "command-output", "manual-attestation", "grep"];
923
+ const bad = (sf("acs") || []).filter((ac: any) => {
924
+ if (!ac.evidence || typeof ac.evidence !== "object") return false;
925
+ return !VALID.includes(ac.evidence.type);
926
+ });
927
+ expect(bad.map((ac: any) => `${ac.id}: ${ac.evidence?.type}`)).toEqual([]);
928
+ });
929
+
930
+ // 17. evidence-type-ratio (GAP → PORTED, ADR-009 A5: >= 50% behavioral evidence required)
931
+ test("evidence-type-ratio: <50% pattern-only evidence (ADR-009 A5)", () => {
932
+ if (!isVerifyPlus()) return;
933
+ const acs = (sf("acs") || []).filter((ac: any) => ac.evidenceMethod);
934
+ if (acs.length === 0) return;
935
+ const patternOnly = acs.filter((ac: any) => {
936
+ const t = ac.evidenceMethod?.type;
937
+ return !t || t === "pattern" || t === "GREP_CHECK" || t === "grep";
938
+ });
939
+ const ratio = patternOnly.length / acs.length;
940
+ // ADR-009 A5: at least 50% behavioral evidence required
941
+ expect(ratio, `${patternOnly.length}/${acs.length} (${Math.round(ratio * 100)}%) pattern-only — must be < 50% (ADR-009 A5)`).toBeLessThanOrEqual(0.5);
942
+ });
943
+
944
+ // 18. evidence-method-executed (GAP → PORTED)
945
+ test("evidence-method-executed: ACs with a threshold have evidence recorded", () => {
946
+ if (!isVerifyPlus()) return;
947
+ const violations = missingEvidenceViolations(sf("acs"));
948
+ expect(violations, violations.join("\n")).toEqual([]);
949
+ });
950
+
951
+ // 19. threshold-crosscheck (GAP → PORTED)
952
+ test("threshold-crosscheck: evidence values meet threshold", () => {
953
+ if (!isVerifyPlus()) return;
954
+ const violations = thresholdCrosscheckViolations(sf("acs"));
955
+ expect(violations, violations.join("\n")).toEqual([]);
956
+ });
957
+
958
+ // 20. tests-pass (GAP → PORTED)
959
+ //
960
+ // #224: run wf_7ac5f614-d21 returned `regressions: 0` on a branch running
961
+ // 3984 pass / 1 fail. The suite reading and the verdict were not connected,
962
+ // and an unmeasured suite read as a passing one. The conversion from reading
963
+ // to refusal now lives in `suiteVerdictViolations` above — one function, one
964
+ // body — so test/suite-binding-mutation.test.ts can remove it from a mutant
965
+ // copy of this file and watch the mutant ship a fixture whose failing test
966
+ // was really run. What was broken to prove it: short-circuiting that
967
+ // function to `return []` turns the gate green on a measured FAIL.
968
+ test("tests-pass: test suite passes", () => {
969
+ if (!isVerifyPlus()) return;
970
+ const harness = loadProjectHarness();
971
+ if (!harness?.dev?.testCmd) return;
972
+ // Both halves of one fact, read from one state (#530). The enum is what
973
+ // this check has always read; the measurement is what produced it, and a
974
+ // PASS with nothing behind it is the case the enum alone cannot see.
975
+ const violations = suiteVerdictViolations(
976
+ sf("environments")?.local?.tests,
977
+ readSuiteMeasurement(loadRaw()),
978
+ );
979
+ expect(violations, violations.join("\n")).toEqual([]);
980
+ });
981
+
982
+ // 20b. evidence-prevalidation (#235)
983
+ //
984
+ // The pre-validator already dry-runs every AC evidence command and prints
985
+ // which ones are broken. Nothing read that print. The conversion from
986
+ // reading to refusal lives in `prevalidationViolations` above — one
987
+ // function, one body — so test/suite-binding-mutation.test.ts can remove it
988
+ // from a mutant copy of this file and watch the mutant ship a fixture whose
989
+ // broken evidence command was really dry-run. What was broken to prove it:
990
+ // short-circuiting `prevalidateEvidence` to `return []` makes the fixture
991
+ // measure PASS, and short-circuiting this function makes the gate green on a
992
+ // measured FAIL.
993
+ test("evidence-prevalidation: AC evidence commands pre-validated before ship", () => {
994
+ if (!isVerifyPlus()) return;
995
+ // An AC with no evidence command has no command that can be broken, so a
996
+ // run with none of them has nothing to pre-validate. That is a real early
997
+ // return, like tsc-pass's missing tsconfig — not a fail-open, because the
998
+ // pre-validator classifies exactly those ACs `skipped` itself.
999
+ const acs = sf("acs") || [];
1000
+ if (!acs.some((ac: any) => ac?.evidenceMethod?.command)) return;
1001
+ const violations = prevalidationViolations(sf("evidencePrevalidation"), sf("phase"));
1002
+ expect(violations, violations.join("\n")).toEqual([]);
1003
+ });
1004
+
1005
+ // 21. tsc-pass (GAP → PORTED)
1006
+ //
1007
+ // #176: this was two early returns and an empty tail — no expect, no tsc
1008
+ // invocation, no baseline comparison. The second return read
1009
+ // `environments.local.tests` and bailed with a comment deferring to the
1010
+ // engineer, so the gate asked the agent it was gating whether it needed to
1011
+ // run. No gate defers to its subject. It now runs the
1012
+ // compiler and compares the count against the project's declared baseline.
1013
+ // What was broken to prove it fails: a project with one bad assignment and
1014
+ // `.claude/typecheck-baseline.json` at 0 — see
1015
+ // test/gate-vacuous-checks.test.ts, which plants exactly that and watches
1016
+ // this go red, then removes it and watches it go green.
1017
+ test("tsc-pass: no new TypeScript errors above baseline", () => {
1018
+ if (!isVerifyPlus()) return;
1019
+ // #314: the compiler runs on the TREE UNDER TEST. Running it in
1020
+ // projectRoot typechecked the main checkout, which passes by construction,
1021
+ // and let two real type errors through to CI on run wf_5cdce852-1e2.
1022
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "tsc-pass");
1023
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
1024
+ const root = treeRoot();
1025
+ // No tsconfig means there is nothing for tsc to check; that is a real
1026
+ // early return, not a skipped measurement.
1027
+ if (!root || !existsSync(join(root, "tsconfig.json"))) return;
1028
+
1029
+ const baselineFile = join(root, ".claude", "typecheck-baseline.json");
1030
+ const baselineRaw = existsSync(baselineFile) ? readFileSync(baselineFile, "utf-8") : null;
1031
+
1032
+ const run = spawnSync("bunx", ["tsc", "--noEmit"], {
1033
+ cwd: root,
1034
+ encoding: "utf-8",
1035
+ timeout: TYPECHECK_TIMEOUT_MS,
1036
+ stdio: ["ignore", "pipe", "pipe"],
1037
+ });
1038
+ const failures = run.error
1039
+ ? [`could not run tsc: ${run.error.message}`]
1040
+ : typecheckFailures(
1041
+ { status: run.status, output: `${run.stdout ?? ""}${run.stderr ?? ""}` },
1042
+ baselineRaw,
1043
+ );
1044
+ expect(failures, failures.join("\n")).toEqual([]);
1045
+ });
1046
+
1047
+ // 22. code-committed (GAP → PORTED)
1048
+ test("code-committed: no uncommitted changes in project", () => {
1049
+ if (!isShipPlus()) return;
1050
+ // #314: `git status` must run in the tree holding the work. In projectRoot
1051
+ // it inspected the main checkout, which is clean during a worktree run, so
1052
+ // this check — whose whole job is confirming the work is committed —
1053
+ // passed unconditionally on every run this repo has made.
1054
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "code-committed");
1055
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
1056
+ const root = treeRoot();
1057
+ if (!root) return;
1058
+ // #315: an unset `codeCommittedPaths` used to early-return here, and
1059
+ // nothing authors that key, so this check had never run. The whole tree
1060
+ // is the default scope; a configured list is a narrowing, not a switch.
1061
+ const harness = loadProjectHarness();
1062
+ const paths: string[] = harness?.codeCommittedPaths || [];
1063
+ const configured = paths.length > 0;
1064
+ // Argv form: `paths` comes out of the consumer's own rungate.json and
1065
+ // `exec` goes through a shell. The old line interpolated it straight
1066
+ // into one.
1067
+ const result = execArgv(
1068
+ ["status", "--porcelain", ...(configured ? ["--", ...paths] : [])],
1069
+ root,
1070
+ );
1071
+ const violations = codeCommittedViolations(result, { configured, paths });
1072
+ expect(violations, violations.join("\n")).toEqual([]);
1073
+ });
1074
+
1075
+ // 22b. worktree-repo (#414, follow-up to #218)
1076
+ test("worktree-repo: every worktree is a checkout of this run's repository", () => {
1077
+ if (!isShipPlus()) return;
1078
+ // The entries, not just `treeUnderTest`: a decomposed run has one
1079
+ // worktree per sub-issue and #218's incident was a SECOND agent landing
1080
+ // in the wrong checkout while the first was correct. Checking only the
1081
+ // tree under test would read the right answer off the wrong worktree.
1082
+ //
1083
+ // `treeUnderTest` is included because it is the tree every other check in
1084
+ // this file measures — if it is a worktree of another repository then
1085
+ // tsc-pass, code-committed and code-pushed all measured that repository,
1086
+ // and said PASS.
1087
+ const named = Array.isArray(sf("worktrees")) ? sf("worktrees") : [];
1088
+ const tree = sf("treeUnderTest");
1089
+ const entries = [
1090
+ ...(typeof tree === "string" && tree.trim() ? [{ path: tree }] : []),
1091
+ ...named,
1092
+ ];
1093
+ const violations = worktreeRepoViolations(entries, sf("repo"));
1094
+ expect(violations, violations.join("\n")).toEqual([]);
1095
+ });
1096
+
1097
+ // 23. code-pushed (GAP → PORTED)
1098
+ test("code-pushed: no unpushed commits", () => {
1099
+ if (!isShipPlus()) return;
1100
+ // #314: in projectRoot this asked whether `main` was ahead of its own
1101
+ // upstream, which it is not, so the answer was always 0. #174 filed the
1102
+ // same defect against this check independently.
1103
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "code-pushed");
1104
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
1105
+ const root = treeRoot();
1106
+ if (!root) return;
1107
+ const count = exec("git rev-list --count @{upstream}..HEAD", root);
1108
+ const upstream = exec("git rev-parse --abbrev-ref --symbolic-full-name @{upstream}", root);
1109
+ const violations = unpushedViolations(count, upstream);
1110
+ expect(violations, violations.join("\n")).toEqual([]);
1111
+ });
1112
+
1113
+ // 25. local-api-validated (GAP → PORTED)
1114
+ test("local-api-validated: environments.local.api set", () => {
1115
+ if (!isVerifyPlus()) return;
1116
+ const harness = loadProjectHarness();
1117
+ if (!harness?.dev?.apiBase) return;
1118
+ const val = sf("environments")?.local?.api;
1119
+ if (!val) {
1120
+ expect("local.api not set").toBe("PASS or SKIP — required when dev.apiBase is configured");
1121
+ } else if (val === "SKIP") {
1122
+ // SKIP is allowed
1123
+ } else if (val !== "PASS") {
1124
+ expect(`api: ${val}`).toBe("PASS");
1125
+ }
1126
+ });
1127
+
1128
+ // 26. local-ui-validated (GAP → PORTED)
1129
+ test("local-ui-validated: environments.local.ui set with skipReason", () => {
1130
+ if (!isVerifyPlus()) return;
1131
+ const harness = loadProjectHarness();
1132
+ if (!harness?.dev?.uiBase) return;
1133
+ const val = sf("environments")?.local?.ui;
1134
+ if (!val) {
1135
+ expect("local.ui not set").toBe("PASS or SKIP — required when dev.uiBase is configured");
1136
+ } else if (val === "SKIP") {
1137
+ const reason = sf("environments")?.local?.uiSkipReason;
1138
+ if (!reason) {
1139
+ expect("UI SKIP without reason").toBe("uiSkipReason required when SKIP");
1140
+ }
1141
+ } else if (val !== "PASS") {
1142
+ expect(`ui: ${val}`).toBe("PASS");
1143
+ }
1144
+ });
1145
+
1146
+ // 29. quinn-on-ui-change
1147
+ test("quinn-on-ui-change: quinn spawned when UI files change", () => {
1148
+ if (!isShipPlus()) return;
1149
+ const tier = sf("sizing")?.ceremonyTier;
1150
+ if (tier === "LIGHT") return;
1151
+ const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
1152
+ const UI_SVC = /template|campaign-html|brief-pipeline|email-template|dashboard/i;
1153
+ const allPaths: string[] = [];
1154
+ for (const ac of sf("acs") || []) {
1155
+ if (ac.statement) allPaths.push(ac.statement);
1156
+ if (ac.evidenceMethod?.command) allPaths.push(ac.evidenceMethod.command);
1157
+ for (const cf of ac.contextFiles || []) {
1158
+ if (cf?.path) allPaths.push(cf.path);
1159
+ }
1160
+ }
1161
+ // Strip absolute project paths to avoid false positives (e.g. /DailyBriefDashboard/ matching "dashboard")
1162
+ //
1163
+ // #314 — BOTH roots, not just projectRoot. The AC paths this reads are
1164
+ // written while the run is in a worktree, so they carry the tree-under-test
1165
+ // path; stripping only projectRoot left that path in the string and handed
1166
+ // the UI_SVC regex the directory name it exists to not match. This is the
1167
+ // one site that legitimately wants projectRoot alongside the tree, because
1168
+ // it is sanitising text rather than measuring a tree.
1169
+ const joined = [sf("projectRoot"), sf("treeUnderTest")]
1170
+ .filter((r): r is string => typeof r === "string" && r !== "")
1171
+ .reduce((text, root) => text.replaceAll(root, ""), allPaths.join(" "));
1172
+ if (!UI_EXT.test(joined) && !UI_SVC.test(joined)) return;
1173
+ const quinn = sf("agents")?.quinn;
1174
+ if (!quinn?.spawned) {
1175
+ expect("quinn not spawned for UI change").toBe("quinn.spawned required");
1176
+ }
1177
+ // Quinn depth check: spawned but no testedPaths = code review only (ADR-009 A3)
1178
+ // If Quinn has verdict PASS, accept that as sufficient evidence (ship.js doesn't write testedPaths yet)
1179
+ if (quinn?.spawned && quinn.verdict !== 'PASS') {
1180
+ const testedPaths = quinn.testedPaths;
1181
+ expect(Array.isArray(testedPaths) && testedPaths.length > 0, "quinn.testedPaths must not be empty — code review only does not satisfy UI testing (ADR-009 A3)").toBe(true);
1182
+ }
1183
+ // Quinn screenshot check: WARN when verdict=PASS but no screenshots (ship.js Quinn doesn't capture yet)
1184
+ if (quinn?.spawned && quinn.verdict !== 'PASS') {
1185
+ const screenshots = quinn.screenshots || [];
1186
+ const hasRealScreenshots = screenshots.length > 0 && screenshots.some((s: string) => existsSync(s));
1187
+ if (!hasRealScreenshots) {
1188
+ expect("quinn.screenshots empty or files missing").toBe("at least 1 screenshot file on disk — browser verification required for UI changes (#90 RCA)");
1189
+ }
1190
+ }
1191
+ });
1192
+
1193
+ // quinn-container-port: Quinn must test the CONTAINER, not the dev server (#1371)
1194
+ //
1195
+ // #344: this compared against a literal 7776 — one consumer's container
1196
+ // port. Every other consumer got a warning saying their correct port was
1197
+ // the wrong one, naming a port they do not run. The expected port comes
1198
+ // from `container.port`, which was already a declared schema key, and with
1199
+ // none configured the check says it could not look rather than inventing a
1200
+ // number to disagree with.
1201
+ //
1202
+ // Still a warning rather than a failure, deliberately and not by oversight:
1203
+ // turning it into a refusal changes what blocks live runs and needs its own
1204
+ // measurement. What is fixed here is that it stops being wrong.
1205
+ test("quinn-container-port: Quinn tested the container, not the dev server", () => {
1206
+ const quinn = sf("agents")?.quinn;
1207
+ if (!quinn?.spawned) return;
1208
+ const agentPort = quinn.port;
1209
+ const envPort = sf("environments")?.local?.quinn?.port;
1210
+ const port = agentPort || envPort;
1211
+ if (!port) {
1212
+ console.warn("WARN: quinn-container-port: no port recorded in quinn agent data — cannot verify container testing");
1213
+ return;
1214
+ }
1215
+ // Bound to `harness`, not to a name of its own: #315's key registry only
1216
+ // sees reads off a binding called `harness`, so `const expected =
1217
+ // loadProjectHarness()?...` would have been a config key the registry
1218
+ // could not account for. Its guard caught this on the first run.
1219
+ const harness = loadProjectHarness();
1220
+ const expected = harness?.container?.port;
1221
+ if (!expected) {
1222
+ console.warn(
1223
+ "WARN: quinn-container-port: the project sets no `container.port`, so whether Quinn " +
1224
+ `tested the container could not be determined (it used :${port}). UNMEASURED, not PASS.`,
1225
+ );
1226
+ return;
1227
+ }
1228
+ if (port !== expected) {
1229
+ console.warn(
1230
+ `WARN: quinn-container-port: Quinn tested on port ${port}, expected ${expected} ` +
1231
+ `(container.port). Testing the dev server does not validate container behavior.`,
1232
+ );
1233
+ }
1234
+ });
1235
+
1236
+ // brief-ac-alignment: Marcus brief contains all AC ids (ADR-009 C2)
1237
+ //
1238
+ // #700: the decision lives in `lib/brief-ac-alignment.ts` and this is the
1239
+ // call site. It failed on five consecutive pushes to main and nothing in
1240
+ // the log said which of its two branches fired, because neither refusal
1241
+ // named the path it had resolved — and the path is assembled here from
1242
+ // RUNGATE_WORK_DIR, a $HOME/.rungate fallback and the slug, so a wrong
1243
+ // directory and a short brief produced the same sentence. Every refusal
1244
+ // from the binding now carries the resolved path.
1245
+ test("brief-ac-alignment: brief covers all ACs", () => {
1246
+ if (!isVerifyPlus()) return;
1247
+ const slug = sf("slug");
1248
+ if (!slug) return;
1249
+ const briefPath = runArtifact("marcus-brief.md");
1250
+ const exists = existsSync(briefPath);
1251
+ const reading = briefAcAlignment({
1252
+ briefPath,
1253
+ briefExists: exists,
1254
+ briefText: exists ? readFileSync(briefPath, "utf-8") : "",
1255
+ acIds: (sf("acs") || []).map((ac: any) => ac.id),
1256
+ marcusSpawned: sf("agents")?.marcus?.spawned === true,
1257
+ });
1258
+ if (reading.reading === "NOT_APPLICABLE") {
1259
+ console.warn(`WARN: ${reading.declined}`);
1260
+ return;
1261
+ }
1262
+ expect(reading.refusal, reading.refusal ?? "brief-ac-alignment").toBeNull();
1263
+ });
1264
+
1265
+ // ac-spec-traceability (#440)
1266
+ test("ac-spec-traceability: AC specElements exist in governing spec", () => {
1267
+ if (!isVerifyPlus()) return;
1268
+ const specPath = sf("governingSpec")?.path;
1269
+ if (!specPath || !existsSync(specPath)) return;
1270
+ const specContent = readFileSync(specPath, "utf-8");
1271
+ const missing: string[] = [];
1272
+ for (const ac of sf("acs") || []) {
1273
+ if (!ac.specElement) continue;
1274
+ if (!specContent.includes(ac.specElement)) {
1275
+ missing.push(`${ac.id}: specElement "${ac.specElement}" not found in spec`);
1276
+ }
1277
+ }
1278
+ expect(missing).toEqual([]);
1279
+ });
1280
+
1281
+ // 28. marcus-committed (GAP → PORTED)
1282
+ test("marcus-committed: no uncommitted Marcus changes", () => {
1283
+ if (!isVerifyPlus()) return;
1284
+ const signalDir = join(process.env.RUNGATE_WORK_DIR || join(process.env.HOME || "", ".rungate"), "signals");
1285
+ const slug = sf("slug") || "";
1286
+ const signalFile = join(signalDir, `marcus-uncommitted-${slug.replace(/\//g, "-")}`);
1287
+ if (existsSync(signalFile)) {
1288
+ expect("marcus left uncommitted files").toBe("marcus committed all changes");
1289
+ }
1290
+ });
1291
+
1292
+ // prove-evidence-type: UI changes require screenshot/api-response evidence (ADR-009 A4)
1293
+ test("prove-evidence-type: UI changes require live evidence", () => {
1294
+ if (!isVerifyPlus()) return;
1295
+ // #314: the changed-file list comes from the branch's history, not main's.
1296
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "prove-evidence-type");
1297
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
1298
+ const root = treeRoot();
1299
+ if (!root) return;
1300
+ const scopeSha = sf("gates")?.scope?.commitSha;
1301
+ if (!scopeSha) return;
1302
+ // #531: branch-relative, and an unusable diff says so rather than
1303
+ // reading as "nothing changed".
1304
+ const changed = readBranchDiff(scopeSha, a => execArgvOf("git", a, root), "prove-evidence-type");
1305
+ if (changed.files === null) { console.warn(`WARN: ${changed.reason}`); return; }
1306
+ const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
1307
+ const uiChanged = changed.files.filter(f => UI_EXT.test(f));
1308
+ if (uiChanged.length === 0) return;
1309
+ const slug = sf("slug");
1310
+ if (!slug) return;
1311
+ const evidencePath = runArtifact("prove-evidence.json");
1312
+ if (!existsSync(evidencePath)) return;
1313
+ const evidence = JSON.parse(readFileSync(evidencePath, "utf-8"));
1314
+ const afterType = evidence.afterEvidence?.type;
1315
+ const VALID_UI_EVIDENCE = ["screenshot", "api-response"];
1316
+ expect(afterType, "afterEvidence.type required for UI changes (ADR-009 A4)").toBeTruthy();
1317
+ expect(VALID_UI_EVIDENCE.includes(afterType), `afterEvidence.type must be screenshot or api-response for UI changes, got "${afterType}" (ADR-009 A4)`).toBe(true);
1318
+ });
1319
+
1320
+ // acHash consistency: verify gate checks scope's AC hash (ADR-009)
1321
+ // Hash auto-recomputed by writeWorkflowState on authorized modifications (#43)
1322
+ test("acHash-consistency-verify: ACs unchanged since scope", () => {
1323
+ if (!isVerifyPlus()) return;
1324
+ const acHash = sf("gates")?.scope?.acHash;
1325
+ if (!acHash) return;
1326
+ const acDefs = (sf("acs") || []).map((ac: any) => ({
1327
+ id: ac.id, type: ac.type, statement: ac.statement,
1328
+ specElement: ac.specElement, threshold: ac.threshold,
1329
+ evidenceMethod: ac.evidenceMethod,
1330
+ }));
1331
+ const currentHash = require("crypto").createHash("sha256").update(JSON.stringify(acDefs)).digest("hex");
1332
+ if (currentHash !== acHash) {
1333
+ // Log field-level diff for diagnosis before failing
1334
+ const { diffAcHashFields } = require("./orchestrator");
1335
+ const diff = diffAcHashFields(acDefs, (sf("acs") || []));
1336
+ console.error(`acHash field diff: ${JSON.stringify(diff.changedFields)}`);
1337
+ }
1338
+ expect(currentHash).toBe(acHash);
1339
+ });
1340
+
1341
+ // spec-decision-compliance (#409, Decision #9: WARN tier — console.warn only, no expect failures)
1342
+ test("spec-decision-compliance: AC language aligns with spec decisions", () => {
1343
+ if (!isVerifyPlus()) return;
1344
+ const localPolicies = join(__dirname, "spec-policies.json");
1345
+ const homePolicies = join(process.env.HOME || "", ".claude", "skills", "ship", "spec-policies.json");
1346
+ const policiesPath = existsSync(localPolicies) ? localPolicies : homePolicies;
1347
+ if (!existsSync(policiesPath)) return;
1348
+ const policies = JSON.parse(readFileSync(policiesPath, "utf-8"));
1349
+ const patterns = policies.implementationPatterns;
1350
+ if (!patterns) return;
1351
+ const acs = sf("acs") || [];
1352
+ let warnCount = 0;
1353
+ for (const [key, rule] of Object.entries(patterns) as [string, any][]) {
1354
+ if (!rule.antiPatterns || !rule.decision) continue;
1355
+ const scopes: string[] = rule.scope || ["statement"];
1356
+ for (const ac of acs) {
1357
+ for (const scope of scopes) {
1358
+ const text = scope === "statement" ? (ac.statement || "") : (ac.evidenceMethod?.command || "");
1359
+ if (!text) continue;
1360
+ for (const ap of rule.antiPatterns) {
1361
+ if (new RegExp(ap, "i").test(text)) {
1362
+ console.warn(`WARN: spec-decision-compliance [${key}]: AC "${ac.id}" matches anti-pattern "${ap}" — ${rule.message}`);
1363
+ warnCount++;
1364
+ }
1365
+ }
1366
+ }
1367
+ }
1368
+ }
1369
+ if (warnCount > 0) {
1370
+ console.warn(`WARN: spec-decision-compliance: ${warnCount} anti-pattern match(es) found — review recommended`);
1371
+ }
1372
+ });
1373
+
1374
+ // contextual-skip-validation (#91 → ADR-009 A1: SKIP → FAIL for file-matched gates)
1375
+ test("contextual-skip-validation: SKIP blocked when changed files require gate", () => {
1376
+ if (!isVerifyPlus()) return;
1377
+ // #314: the changed-file list comes from the branch's history, not main's.
1378
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "contextual-skip-validation");
1379
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
1380
+ const root = treeRoot();
1381
+ if (!root) return;
1382
+ const scopeSha = sf("gates")?.scope?.commitSha;
1383
+ if (!scopeSha) return;
1384
+ // #531: branch-relative, and an unusable diff is UNMEASURED rather than
1385
+ // "no changed files — silently return", which is what the previous line
1386
+ // said and which waves every file-matched gate below through.
1387
+ const changed = readBranchDiff(scopeSha, a => execArgvOf("git", a, root), "contextual-skip-validation");
1388
+ if (changed.files === null) { console.warn(`WARN: ${changed.reason}`); return; }
1389
+
1390
+ const FILE_GATE_MAP: Record<string, string[]> = {
1391
+ "tsx|jsx": ["local-ui-validated", "quinn-on-ui-change"],
1392
+ "css|scss|less|html|svg": ["local-ui-validated"],
1393
+ "sql|migration": ["local-api-validated"],
1394
+ "Dockerfile|docker-compose|Makefile": ["prod-rebuild-pass"],
1395
+ };
1396
+
1397
+ const ENV_GATE_PATH: Record<string, string> = {
1398
+ "local-ui-validated": "environments.local.ui",
1399
+ "local-api-validated": "environments.local.api",
1400
+ "quinn-on-ui-change": "agents.quinn.spawned",
1401
+ "prod-rebuild-pass": "environments.prod.rebuild",
1402
+ };
1403
+
1404
+ const changedFiles = changed.files;
1405
+ let warnCount = 0;
1406
+ for (const file of changedFiles) {
1407
+ for (const [extPattern, gates] of Object.entries(FILE_GATE_MAP)) {
1408
+ const re = new RegExp(`\\.(${extPattern})$`, "i");
1409
+ const nameRe = new RegExp(`(${extPattern})`, "i");
1410
+ if (!re.test(file) && !nameRe.test(file.split("/").pop() || "")) continue;
1411
+ for (const gate of gates) {
1412
+ const envPath = ENV_GATE_PATH[gate];
1413
+ if (!envPath) continue;
1414
+ const val = sf(envPath);
1415
+ if (val === "SKIP") {
1416
+ expect(`contextual-skip-blocked: "${file}" changed but ${gate} is SKIP`).toBe("gate must not be SKIP when matched files changed (ADR-009 A1)");
1417
+ warnCount++;
1418
+ }
1419
+ }
1420
+ }
1421
+ }
1422
+ if (warnCount > 0) {
1423
+ expect(`contextual-skip-validation: ${warnCount} SKIP(s) on file-matched gates`).toBe("contextual-skip-validation: 0 SKIPs (ADR-009 A1)");
1424
+ }
1425
+ });
1426
+ });
1427
+
1428
+ // ═══ B1/B2 AGENT RESULTS (#1406, #1407) ═════════════════════════════
1429
+ describe("B1/B2 agent results", () => {
1430
+ if (STANDALONE || !existsSync(SF)) {
1431
+ test("skipped — no active workflow (standalone run)", () => {});
1432
+ return;
1433
+ }
1434
+ // B2: Evidence Validator prompt file exists
1435
+ test("b2-prompt-exists: evidence-validator.md available", () => {
1436
+ const promptPath = join(__dirname, "prompts", "evidence-validator.md");
1437
+ expect(
1438
+ existsSync(promptPath),
1439
+ "evidence-validator.md prompt not found in gates/prompts/",
1440
+ ).toBe(true);
1441
+ });
1442
+
1443
+ // B1: AC Adversary prompt file exists
1444
+ test("b1-prompt-exists: ac-adversary.md available", () => {
1445
+ const promptPath = join(__dirname, "prompts", "ac-adversary.md");
1446
+ expect(
1447
+ existsSync(promptPath),
1448
+ "ac-adversary.md prompt not found in gates/prompts/",
1449
+ ).toBe(true);
1450
+ });
1451
+
1452
+ // B2: the evidence validator ran, and left something readable (#126)
1453
+ //
1454
+ // isShipPlus, NOT isVerifyPlus, and the contract fixtures are what found it.
1455
+ // gate-executor runs this suite (`runGateTests`) and only THEN spawns the
1456
+ // fan-out and persists its slots — so at the verify gate the slot cannot yet
1457
+ // exist, and a check that refuses its absence refuses every first attempt.
1458
+ // That is the original circularity with the sign flipped: the gate failing
1459
+ // on the fan-out's absence is what skips the fan-out.
1460
+ //
1461
+ // Verify cannot verify its own not-yet-produced output. Ship can: by then
1462
+ // the fan-out has either run or recorded why it did not, and refusing here
1463
+ // still blocks the PR, which is the only thing the refusal needed to do.
1464
+ test("b2-evidence-validator-structure: results valid when populated", () => {
1465
+ if (!isShipPlus()) return;
1466
+ const tier = sf("sizing")?.ceremonyTier;
1467
+ const ev = sf("gates.verify.evidenceValidator");
1468
+ const violations = verifyFanoutViolations("evidenceValidator", ev, tier);
1469
+ expect(violations, violations.join("\n")).toEqual([]);
1470
+ // Nothing further to validate when the absence was legitimate: a LIGHT run
1471
+ // or a recorded skip has no verdicts, by construction.
1472
+ if (!ev || (ev as { skipped?: unknown }).skipped !== undefined) return;
1473
+ expect(ev.ts, "evidenceValidator.ts must be an ISO timestamp").toBeTruthy();
1474
+ expect(Array.isArray(ev.verdicts), "evidenceValidator.verdicts must be an array").toBe(true);
1475
+ });
1476
+
1477
+ // B1: the AC adversary ran, and left something readable (#126)
1478
+ // isShipPlus for the reason spelled out above B2.
1479
+ test("b1-adversary-verify-structure: results valid when populated", () => {
1480
+ if (!isShipPlus()) return;
1481
+ const tier = sf("sizing")?.ceremonyTier;
1482
+ const adv = sf("gates.verify.adversary");
1483
+ const violations = verifyFanoutViolations("adversary", adv, tier);
1484
+ expect(violations, violations.join("\n")).toEqual([]);
1485
+ if (!adv || (adv as { skipped?: unknown }).skipped !== undefined) return;
1486
+ expect(adv.ts, "adversary.ts must be an ISO timestamp").toBeTruthy();
1487
+ expect(typeof adv.gameable, "adversary.gameable must be a number").toBe("number");
1488
+ expect(typeof adv.approved, "adversary.approved must be a boolean").toBe("boolean");
1489
+ });
1490
+ });
1491
+
1492
+ // ═══ SPEC TRACEABILITY (2 functions) ══════════════════════════════════
1493
+ describe("spec traceability", () => {
1494
+ if (STANDALONE || !existsSync(SF)) {
1495
+ test("skipped — no active workflow (standalone run)", () => {});
1496
+ return;
1497
+ }
1498
+ // 31. decision-id-valid (GAP → PORTED)
1499
+ test("decision-id-valid: D-NNN IDs in ACs exist in governing spec", () => {
1500
+ const specPath = sf("sourceSpecs")?.[0]?.path || sf("governingSpec")?.path;
1501
+ if (!specPath || !existsSync(specPath)) return;
1502
+ const spec = readFileSync(specPath, "utf-8");
1503
+ const ids = new Set<string>();
1504
+ for (const ac of sf("acs") || []) {
1505
+ const matches = (ac.statement || "").match(/D-\d+/g) || [];
1506
+ matches.forEach((id: string) => ids.add(id));
1507
+ }
1508
+ if (ids.size === 0) return;
1509
+ const missing = [...ids].filter(id => !spec.includes(id));
1510
+ expect(missing).toEqual([]);
1511
+ });
1512
+
1513
+ // 32. evidence-target-valid (GAP → PORTED)
1514
+ //
1515
+ // #315 found this one while fixing `code-committed`, and it is worse than
1516
+ // the defect it was filed for. It read the same unauthored
1517
+ // `codeCommittedPaths` key and early-returned — but even past that return
1518
+ // it computed an `outside` list and **discarded it**, under a comment
1519
+ // saying outside refs are common. It has no assertion. It is a name in the
1520
+ // gate list and nothing else, and it cannot fail for any input.
1521
+ //
1522
+ // Deleted rather than left standing, because a check that asserts nothing
1523
+ // reads as coverage in every report that counts gate checks. The threshold
1524
+ // it would need — which evidence file references count as out of scope —
1525
+ // does not exist yet and inventing one here would be a guess that goes red
1526
+ // on every run. It is recorded in OPTIONAL_HARNESS_KEYS in
1527
+ // test/gate-optional-config-keys.test.ts so it cannot be quietly re-added,
1528
+ // and #338 tracks building it with a threshold that has evidence behind it.
1529
+ //
1530
+ // #338 measured it, and the original name does not survive the
1531
+ // measurement. Across 82 completed runs, 278 of 542 classifiable evidence
1532
+ // file references fall OUTSIDE the AC's own contextFiles, and the largest
1533
+ // category is the test file that proves the AC. "Within scope" is not a
1534
+ // property real runs have, so no threshold for it exists to state.
1535
+ //
1536
+ // What the same measurement did find is below: 11.1% of evidence commands
1537
+ // cannot run anywhere but the machine that wrote them.
1538
+
1539
+ // 32. evidence-portable (#338 — replaces evidence-target-valid)
1540
+ test("evidence-portable: no evidence command is pinned to one machine", () => {
1541
+ if (!isVerifyPlus()) return;
1542
+ const violations = evidencePortabilityViolations(sf("acs"));
1543
+ expect(violations, violations.join("\n")).toEqual([]);
1544
+ });
1545
+ });
1546
+
1547
+ // ═══ SHIP CHECKS (7 functions) ════════════════════════════════════════
1548
+ describe("ship checks", () => {
1549
+ if (STANDALONE || !existsSync(SF)) {
1550
+ test("skipped — no active workflow (standalone run)", () => {});
1551
+ return;
1552
+ }
1553
+ // 35. all-gates-pass (GAP → PORTED)
1554
+ test("all-gates-pass: scope + verify gates PASS", () => {
1555
+ if (!isShipPlus()) return;
1556
+ const scope = sf("gates")?.scope?.result;
1557
+ const verify = sf("gates")?.verify?.result;
1558
+ if (scope && scope !== "PASS") {
1559
+ expect(`scope gate: ${scope}`).toBe("PASS");
1560
+ }
1561
+ if (verify && verify !== "PASS") {
1562
+ expect(`verify gate: ${verify}`).toBe("PASS");
1563
+ }
1564
+ });
1565
+
1566
+ // 36. branch-merged / code-pushed (GAP → PORTED)
1567
+ // Ship gate: code pushed + PR exists = PASS (merge is post-ship, verified at prove)
1568
+ // Prove gate: merged to main = PASS
1569
+ test("branch-merged: fix is on main", () => {
1570
+ if (!isShipPlus()) return;
1571
+ // #314: `git branch --contains HEAD` must ask about the branch's HEAD.
1572
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "branch-merged");
1573
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
1574
+ const root = treeRoot();
1575
+ if (!root) return;
1576
+ const gate = process.env.GATE || "";
1577
+ if (gate === "ship") {
1578
+ // At ship gate: check code is pushed, not merged
1579
+ const commit = sf("buildCommit");
1580
+ const agents = sf("agents");
1581
+ const pushed = agents?.marcus?.branch;
1582
+ if (!commit && !pushed) {
1583
+ expect("no commit or branch").toBe("buildCommit or marcus.branch required at ship gate");
1584
+ }
1585
+ return;
1586
+ }
1587
+ // At prove gate or verify: check merged to main
1588
+ const result = exec("git branch --contains HEAD --list main", root);
1589
+ if (result.ok && !result.output.includes("main")) {
1590
+ expect("not on main").toBe("merged to main");
1591
+ }
1592
+ });
1593
+
1594
+ // 38. prod-rebuild-pass (GAP → PORTED)
1595
+ test("prod-rebuild-pass: prod rebuild required when configured", () => {
1596
+ if (!isShipPlus()) return;
1597
+ const harness = loadProjectHarness();
1598
+ if (!harness?.prod?.rebuild) return;
1599
+ const val = sf("environments")?.prod?.rebuild;
1600
+ if (!val) {
1601
+ expect("prod.rebuild not set").toBe("PASS or SKIP with reason — required when prod.rebuild is configured");
1602
+ } else if (val === "SKIP") {
1603
+ const reason = sf("environments")?.prod?.rebuildSkipReason;
1604
+ if (!reason) expect("rebuild SKIP without reason").toBe("rebuildSkipReason required");
1605
+ } else if (val !== "PASS") {
1606
+ expect(`prod rebuild: ${val}`).toBe("PASS or SKIP with reason");
1607
+ }
1608
+ });
1609
+
1610
+ // 39. prod-smoke-pass (GAP → PORTED)
1611
+ test("prod-smoke-pass: prod smoke required when configured", () => {
1612
+ if (!isShipPlus()) return;
1613
+ const harness = loadProjectHarness();
1614
+ if (!harness?.prod?.smokeTest) return;
1615
+ const val = sf("environments")?.prod?.smoke;
1616
+ if (!val) {
1617
+ expect("prod.smoke not set").toBe("PASS or SKIP with reason — required when prod.smokeTest is configured");
1618
+ } else if (val === "SKIP") {
1619
+ const reason = sf("environments")?.prod?.smokeSkipReason;
1620
+ if (!reason) expect("smoke SKIP without reason").toBe("smokeSkipReason required");
1621
+ } else if (val !== "PASS") {
1622
+ expect(`prod smoke: ${val}`).toBe("PASS or SKIP with reason");
1623
+ }
1624
+ });
1625
+
1626
+ // 40. prod-quinn-pass (GAP → PORTED)
1627
+ test("prod-quinn-pass: prod quinn required when prod configured", () => {
1628
+ if (!isShipPlus()) return;
1629
+ const harness = loadProjectHarness();
1630
+ if (!harness?.prod?.rebuild) return;
1631
+ const val = sf("environments")?.prod?.quinn;
1632
+ if (!val) {
1633
+ expect("prod.quinn not set").toBe("PASS or SKIP with reason — required when prod.rebuild is configured");
1634
+ } else if (val === "SKIP") {
1635
+ const reason = sf("environments")?.prod?.quinnSkipReason;
1636
+ if (!reason) expect("quinn SKIP without reason").toBe("quinnSkipReason required");
1637
+ } else if (val !== "PASS") {
1638
+ expect(`prod quinn: ${val}`).toBe("PASS or SKIP with reason");
1639
+ }
1640
+ });
1641
+
1642
+ // quinn-tested-changed-files (#441)
1643
+ test("quinn-tested-changed-files: Quinn tested paths cover UI diffs", () => {
1644
+ if (!isShipPlus()) return;
1645
+ // #314: the changed-file list comes from the branch's history, not main's.
1646
+ const treeViolations = treeUnderTestViolations(sf("treeUnderTest"), "quinn-tested-changed-files");
1647
+ expect(treeViolations, treeViolations.join("\n")).toEqual([]);
1648
+ const root = treeRoot();
1649
+ if (!root) return;
1650
+ const scopeSha = sf("gates")?.scope?.commitSha;
1651
+ if (!scopeSha) return;
1652
+ // #531: branch-relative. This is the check that refused on seven `.tsx`
1653
+ // files the branch has never contained.
1654
+ const changed = readBranchDiff(scopeSha, a => execArgvOf("git", a, root), "quinn-tested-changed-files");
1655
+ if (changed.files === null) { console.warn(`WARN: ${changed.reason}`); return; }
1656
+ const UI_EXT = /\.(tsx|jsx|css|scss|html|svg|less)$/i;
1657
+ const uiChanged = changed.files.filter(f => UI_EXT.test(f));
1658
+ if (uiChanged.length === 0) return;
1659
+ const testedPaths = sf("agents")?.quinn?.testedPaths || [];
1660
+ if (testedPaths.length === 0) {
1661
+ expect(`${uiChanged.length} UI files changed but quinn.testedPaths empty`).toBe("quinn.testedPaths populated");
1662
+ }
1663
+ });
1664
+
1665
+ // 42. verify-blockers-cleared
1666
+ test("verify-blockers-cleared: all Quinn-found bugs re-verified", () => {
1667
+ if (!isShipPlus()) return;
1668
+ const blockers = sf("verifyBlockers") || [];
1669
+ const unresolved = blockers.filter((b: any) => !b.quinnReverified);
1670
+ expect(unresolved.map((b: any) => b.id)).toEqual([]);
1671
+ });
1672
+
1673
+ // 41. prod-quinn-spot (GAP → PORTED)
1674
+ test("prod-quinn-spot: spot check PASS or inherited from full quinn", () => {
1675
+ if (!isShipPlus()) return;
1676
+ const harness = loadProjectHarness();
1677
+ if (!harness?.prod?.rebuild) return;
1678
+ const quinn = sf("environments")?.prod?.quinn;
1679
+ const spot = sf("environments")?.prod?.quinnSpot;
1680
+ if (quinn === "PASS" || spot === "PASS") return; // either is fine
1681
+ if (spot === "SKIP") {
1682
+ const reason = sf("environments")?.prod?.quinnSpotSkipReason;
1683
+ if (!reason) expect("quinnSpot SKIP without reason").toBe("quinnSpotSkipReason required");
1684
+ }
1685
+ });
1686
+
1687
+ // 43. witness-chain (#473)
1688
+ test("witness-chain: witness directory has gate artifacts", () => {
1689
+ if (!isShipPlus()) return;
1690
+ const slug = sf("slug");
1691
+ if (!slug) return;
1692
+ const witnessDir = runArtifact("witnesses");
1693
+ if (!existsSync(witnessDir)) {
1694
+ // WARN — witnesses are new, not all workflows have them yet
1695
+ console.warn("WARN: No witness directory found — witness chain not verified");
1696
+ return;
1697
+ }
1698
+ const witnesses = readdirSync(witnessDir).filter(f => f.endsWith(".json"));
1699
+ expect(witnesses.length, "Witness directory exists but is empty").toBeGreaterThan(0);
1700
+ });
1701
+
1702
+ // 44. witness-ac-verdict: AC PASS verdicts must have corresponding witness chain entry (#521)
1703
+ test("witness-ac-verdict: AC PASS verdicts have matching witness with valid HMAC", () => {
1704
+ if (!isShipPlus()) return;
1705
+ const slug = sf("slug");
1706
+ if (!slug) return;
1707
+ const witnessDir = runArtifact("witnesses");
1708
+ if (!existsSync(witnessDir)) {
1709
+ // No witnesses = cannot verify AC verdicts were set by gate system
1710
+ const passACs = (sf("acs") || []).filter((ac: any) => ac.verdict === "PASS");
1711
+ if (passACs.length > 0) {
1712
+ expect(
1713
+ `${passACs.length} AC(s) have PASS verdict but no witness directory exists`,
1714
+ ).toBe("witness directory required when ACs have PASS verdicts (#521 — prevents manual workflow-state.json edits)");
1715
+ }
1716
+ return;
1717
+ }
1718
+ const witnessFiles = readdirSync(witnessDir).filter(f => f.endsWith(".json"));
1719
+ // Load all witness records
1720
+ const witnesses: Array<{ gate: string; result: string; hmac: string; [k: string]: any }> = [];
1721
+ for (const wf of witnessFiles) {
1722
+ try {
1723
+ const record = JSON.parse(readFileSync(join(witnessDir, wf), "utf-8"));
1724
+ witnesses.push(record);
1725
+ } catch { /* skip unparseable */ }
1726
+ }
1727
+ // Build set of gates that have PASS witnesses with HMAC
1728
+ const passedGatesWithHmac = new Set(
1729
+ witnesses
1730
+ .filter(w => w.result === "PASS" && w.hmac && w.hmac.length > 0)
1731
+ .map(w => w.gate),
1732
+ );
1733
+ // Each AC with PASS verdict must be backed by a prior gate witness
1734
+ // The verify gate covers AC evidence checks; scope gate covers AC definition
1735
+ // Ship witness is written AFTER this test suite — requiring it here is circular
1736
+ // Scope is skipped for LIGHT ceremony tier — only require its witness if it ran
1737
+ const scopeRan = sf("gates")?.scope?.ts;
1738
+ const requiredGates = scopeRan ? ["scope", "verify"] : ["verify"];
1739
+ const missingGates = requiredGates.filter(g => !passedGatesWithHmac.has(g));
1740
+ const passACs = (sf("acs") || []).filter((ac: any) => ac.verdict === "PASS");
1741
+ if (passACs.length > 0 && missingGates.length > 0) {
1742
+ expect(
1743
+ `AC PASS verdicts present but missing witness for: ${missingGates.join(", ")}`,
1744
+ ).toBe("PASS witnesses with valid HMAC required for verify + ship gates (#521 — manual edits detected)");
1745
+ }
1746
+ });
1747
+
1748
+ // pre-registration: ACs must predate fix (ADR-009 A2)
1749
+ test("pre-registration: scope precedes ship", () => {
1750
+ if (!isShipPlus()) return;
1751
+ const scopeTs = sf("gates")?.scope?.ts;
1752
+ if (!scopeTs) return; // LIGHT tier or pre-ADR-009 — scope gate not run
1753
+ const scopeTime = new Date(scopeTs).getTime();
1754
+ expect(scopeTime, "scope timestamp must be in the past (ADR-009 A2: pre-registration)").toBeLessThan(Date.now());
1755
+ const shipTs = sf("gates")?.ship?.ts;
1756
+ if (shipTs) {
1757
+ expect(scopeTime, "scope must predate ship (ADR-009 A2: pre-registration)").toBeLessThanOrEqual(new Date(shipTs).getTime());
1758
+ }
1759
+ });
1760
+
1761
+ // acHash consistency: ship gate checks scope's AC hash (ADR-009)
1762
+ // Restored to FAIL: writeWorkflowState now recomputes acHash on authorized modifications,
1763
+ // so mismatches indicate unauthorized tampering, not heal agent updates (#43)
1764
+ test("acHash-consistency-ship: ACs unchanged since scope", () => {
1765
+ if (!isShipPlus()) return;
1766
+ const acHash = sf("gates")?.scope?.acHash;
1767
+ if (!acHash) return;
1768
+ const acDefs = (sf("acs") || []).map((ac: any) => ({
1769
+ id: ac.id, type: ac.type, statement: ac.statement,
1770
+ specElement: ac.specElement, threshold: ac.threshold,
1771
+ evidenceMethod: ac.evidenceMethod,
1772
+ }));
1773
+ const currentHash = require("crypto").createHash("sha256").update(JSON.stringify(acDefs)).digest("hex");
1774
+ if (currentHash !== acHash) {
1775
+ // Log field-level diff for diagnosis before failing
1776
+ const { diffAcHashFields } = require("./orchestrator");
1777
+ const scopeAcDefs = acDefs; // current defs — scope defs are lost, but hash mismatch means they differ
1778
+ const diff = diffAcHashFields(acDefs, (sf("acs") || []));
1779
+ console.error(`acHash field diff: ${JSON.stringify(diff.changedFields)}`);
1780
+ }
1781
+ expect(currentHash).toBe(acHash);
1782
+ });
1783
+
1784
+ // prove-label-enforcement: needs-prove label must have matching proven label (#1390)
1785
+ //
1786
+ // The labels come from Octokit, not from a GitHub CLI subprocess (#395).
1787
+ // The old call interpolated `issue` and `issueRepo` — both out of workflow
1788
+ // state — into a shell command string, and needed `gh` installed and
1789
+ // authenticated in whatever tree the gate ran in. `getIssueLabels` takes
1790
+ // them as arguments, so no shell parses them.
1791
+ //
1792
+ // The decision is `proveLabelViolation` and lives in lib/, because it was
1793
+ // entangled with the transport and therefore untestable: nothing could
1794
+ // exercise the rule without a network and a `gh` binary, so nothing did,
1795
+ // and SC-502 was ticked over the subprocess for its whole life.
1796
+ test("prove-label-enforcement: needs-prove requires proven label", async () => {
1797
+ if (!isShipPlus()) return;
1798
+ const issueNumber = sf("issue");
1799
+ const issueRepo = sf("issueRepo") || sf("repo");
1800
+ if (!issueNumber || !issueRepo) return;
1801
+
1802
+ // #405 — the fail-open is gone. This used to be `catch { return }`, so a
1803
+ // missing token, a rate limit, a network blip and a typo'd repo slug all
1804
+ // landed in one place and the gate reported clean: "no blocking label"
1805
+ // and "could not find out" were the same outcome.
1806
+ //
1807
+ // `proveLabelGateVerdict` owns the three-way decision and the reasoning
1808
+ // behind the token-less case. The read is the only thing left here,
1809
+ // because a decision entangled with its transport is a decision nobody
1810
+ // tests — the lesson that produced `proveLabelViolation` in the first
1811
+ // place.
1812
+ const { createGitHubClient, getIssueLabels, resolveGitHubToken } = require("../lib/github");
1813
+ const tokenPresent = Boolean(resolveGitHubToken());
1814
+ let read: LabelRead;
1815
+ try {
1816
+ read = {
1817
+ labels: await getIssueLabels(createGitHubClient(), issueRepo, Number(issueNumber)),
1818
+ tokenPresent,
1819
+ };
1820
+ } catch (e) {
1821
+ read = { error: e, tokenPresent };
1822
+ }
1823
+
1824
+ const outcome = proveLabelGateVerdict(read);
1825
+ if (outcome.verdict === "UNMEASURED") {
1826
+ // Reported, not swallowed. `SKIP` is indistinguishable from success and
1827
+ // `UNMEASURED` is not (`.claude/rules/config-access.md`).
1828
+ console.error(`PROVE_LABEL_UNMEASURED: ${outcome.message}`);
1829
+ return;
1830
+ }
1831
+ expect(outcome.verdict, outcome.message ?? undefined).toBe("ALLOW");
1832
+ });
1833
+ });
1834
+
1835
+ // ═══ PROMPT IMMUTABILITY (#1409, ADR-009) ════════════════════════════
1836
+ //
1837
+ // This asserted the macOS `uchg` flag via `ls -lO`, and was broken in both
1838
+ // directions at once (#92).
1839
+ //
1840
+ // It failed OPEN on CI, which is the only place it runs. `ls -lO` is BSD; the
1841
+ // workflows are ubuntu-latest, where GNU coreutils rejects -O and exits 2. The
1842
+ // check read `if (result.ok && !output.includes("uchg"))`, so a non-zero exit
1843
+ // skipped the body, the unlocked list stayed empty, and the test passed. Every
1844
+ // green CI run in this repo's history passed it without evaluating a file.
1845
+ //
1846
+ // It failed CLOSED anywhere it would have worked. chflags is filesystem
1847
+ // metadata and git does not track it, so a fresh clone or `git worktree add`
1848
+ // starts with no flags — confirmed by a second session seeing 0 of 3 flagged
1849
+ // in its worktree against 3 of 3 in the primary checkout at the same commit.
1850
+ // It failed hardest for a developer doing what the harness tells them to do.
1851
+ //
1852
+ // And the remedy it printed, `gates/lock-prompts.sh`, does not exist. Nothing
1853
+ // in the repo calls chflags at all: the check asserted a property the project
1854
+ // had no mechanism to establish.
1855
+ //
1856
+ // What it actually wants is that the prompts have not been modified. That is
1857
+ // content, git already tracks it, and it means the same thing on every
1858
+ // platform and in every checkout.
1859
+ // A diff against HEAD alone is not enough, and it is worth being explicit
1860
+ // about why rather than shipping the obvious fix. In CI the working tree IS
1861
+ // HEAD, so `git diff HEAD` is empty by construction and the check could never
1862
+ // fail in the one place it runs — the same vacuity, rebuilt. The committed
1863
+ // checksum baseline is what makes it meaningful there: editing a prompt
1864
+ // without deliberately updating gates/prompts.sha256 fails on CI, and updating
1865
+ // the baseline is a reviewable line in the diff. That is also the mechanism
1866
+ // the old check never had — a way to establish the property, not just assert
1867
+ // it.
1868
+ //
1869
+ // Both checks are kept. The baseline catches committed drift anywhere; the
1870
+ // HEAD diff catches local tampering before it is ever committed.
1871
+ describe("prompt immutability", () => {
1872
+ const promptDir = join(__dirname, "prompts");
1873
+ const baselinePath = join(__dirname, "prompts.sha256");
1874
+
1875
+ test("prompt-immutability: prompt files match the committed checksum baseline", () => {
1876
+ if (!existsSync(promptDir)) return;
1877
+ const prompts = readdirSync(promptDir).filter(f => f.endsWith(".md")).sort();
1878
+ if (prompts.length === 0) return;
1879
+
1880
+ expect(existsSync(baselinePath), `missing ${baselinePath} — the baseline IS the immutability record`).toBe(true);
1881
+
1882
+ const baseline = new Map<string, string>();
1883
+ for (const line of readFileSync(baselinePath, "utf-8").split("\n")) {
1884
+ const m = line.trim().match(/^([0-9a-f]{64})\s+(.+)$/);
1885
+ if (m) baseline.set(m[2], m[1]);
1886
+ }
1887
+ // A baseline that parsed to nothing would make every comparison below
1888
+ // vacuous, so reject it rather than report success.
1889
+ expect(baseline.size, `${baselinePath} contains no usable checksum lines`).toBeGreaterThan(0);
1890
+
1891
+ const drifted: string[] = [];
1892
+ for (const f of prompts) {
1893
+ const want = baseline.get(f);
1894
+ const got = createHash("sha256").update(readFileSync(join(promptDir, f))).digest("hex");
1895
+ if (want !== got) drifted.push(f);
1896
+ }
1897
+ // A prompt added without a baseline entry is drift too — otherwise the
1898
+ // way to evade the check is to add a file rather than edit one.
1899
+ for (const f of baseline.keys()) {
1900
+ if (!prompts.includes(f)) drifted.push(`${f} (in baseline, missing on disk)`);
1901
+ }
1902
+
1903
+ expect(
1904
+ drifted,
1905
+ `Prompt files are immutable (#1409, ADR-009). If a change is intended, update gates/prompts.sha256 in the same commit so it shows up in review.`,
1906
+ ).toEqual([]);
1907
+ });
1908
+
1909
+ test("prompt-immutability: prompt files are unmodified against HEAD", () => {
1910
+ if (!existsSync(promptDir)) return;
1911
+ const prompts = readdirSync(promptDir).filter(f => f.endsWith(".md"));
1912
+ if (prompts.length === 0) return;
1913
+
1914
+ const repoRoot = join(__dirname, "..");
1915
+ const rel = "gates/prompts";
1916
+ const result = exec(`git -C "${repoRoot}" diff --name-only HEAD -- "${rel}"`);
1917
+
1918
+ // Fail closed. The predecessor treated a failed command as "nothing to
1919
+ // report", which is precisely how it passed vacuously for its whole life.
1920
+ expect(result.ok, `could not diff ${rel} against HEAD: ${result.output}`).toBe(true);
1921
+
1922
+ const modified = result.output.split("\n").map(l => l.trim()).filter(Boolean);
1923
+ expect(modified, `Prompt files are immutable (#1409, ADR-009) — revert with: git checkout HEAD -- ${rel}`).toEqual([]);
1924
+ });
1925
+ });
1926
+
1927
+ // ═══ PROVE CHECKS (ADR-009 B3) ═══════════════════════════════════════
1928
+ describe("prove checks", () => {
1929
+ if (STANDALONE || !existsSync(SF)) {
1930
+ test("skipped — no active workflow (standalone run)", () => {});
1931
+ return;
1932
+ }
1933
+ // prove-evidence exists when prove gate runs
1934
+ test("prove-evidence-exists: prove gate produces evidence file", () => {
1935
+ const currentGate = process.env.GATE;
1936
+ if (currentGate !== "prove") return;
1937
+ const slug = sf("slug");
1938
+ if (!slug) return;
1939
+ const evidencePath = runArtifact("prove-evidence.json");
1940
+ expect(existsSync(evidencePath), "prove-evidence.json must exist after prove gate runs (ADR-009 B3)").toBe(true);
1941
+ });
1942
+
1943
+ // prove-evidence has valid verdict enum
1944
+ test("prove-evidence-verdict: verdict is PROVEN|UNPROVEN|INCONCLUSIVE", () => {
1945
+ const currentGate = process.env.GATE;
1946
+ if (currentGate !== "prove") return;
1947
+ const slug = sf("slug");
1948
+ if (!slug) return;
1949
+ const evidencePath = runArtifact("prove-evidence.json");
1950
+ if (!existsSync(evidencePath)) return;
1951
+ const evidence = JSON.parse(readFileSync(evidencePath, "utf-8"));
1952
+ const VALID_VERDICTS = ["PROVEN", "UNPROVEN", "INCONCLUSIVE"];
1953
+ expect(VALID_VERDICTS.includes(evidence.verdict), `prove-evidence verdict must be PROVEN|UNPROVEN|INCONCLUSIVE, got "${evidence.verdict}" (ADR-009 B3)`).toBe(true);
1954
+ });
1955
+ });
1956
+
1957
+ // ═══ TEST BASELINE SUBTRACTION (#599) ═══════════════════════════════
1958
+ describe("test-baseline-subtraction", () => {
1959
+ // AC-2: subtracts baseline from total test failures
1960
+ test("subtracts baseline from total test failures", () => {
1961
+ expect(subtractTestBaseline(5, 3)).toBe(2);
1962
+ expect(subtractTestBaseline(10, 0)).toBe(10);
1963
+ expect(subtractTestBaseline(0, 0)).toBe(0);
1964
+ });
1965
+
1966
+ test("baseline diffing clamps to zero for negative values", () => {
1967
+ // If baseline is higher than total (tests were fixed), new failures = 0
1968
+ expect(subtractTestBaseline(2, 5)).toBe(0);
1969
+ expect(subtractTestBaseline(0, 3)).toBe(0);
1970
+ });
1971
+
1972
+ // AC-3: zero new failures = PASS
1973
+ test("zero new failures when total equals baseline", () => {
1974
+ const newFailures = subtractTestBaseline(3, 3);
1975
+ expect(newFailures).toBe(0);
1976
+ // 0 new failures should mean PASS
1977
+ const resultVal: "PASS" | "FAIL" = newFailures > 0 ? "FAIL" : "PASS";
1978
+ expect(resultVal).toBe("PASS");
1979
+ });
1980
+
1981
+ test("pre-existing failures do not block when no new failures introduced", () => {
1982
+ // Scenario: 7 pre-existing failures, Marcus introduces 0 new ones
1983
+ const baseline = 7;
1984
+ const totalAfterMarcus = 7;
1985
+ const newFailures = subtractTestBaseline(totalAfterMarcus, baseline);
1986
+ expect(newFailures).toBe(0);
1987
+ const resultVal: "PASS" | "FAIL" = newFailures > 0 ? "FAIL" : "PASS";
1988
+ expect(resultVal).toBe("PASS");
1989
+ });
1990
+
1991
+ // extractTestFailureCount unit tests
1992
+ test("extractTestFailureCount: parses failure count from bun test output", () => {
1993
+ const output = `
1994
+ 42 pass
1995
+ 3 fail
1996
+ `;
1997
+ expect(extractTestFailureCount(output)).toBe(3);
1998
+ });
1999
+
2000
+ test("extractTestFailureCount: returns 0 when no failures in output", () => {
2001
+ const output = `
2002
+ 42 pass
2003
+ `;
2004
+ expect(extractTestFailureCount(output)).toBe(0);
2005
+ });
2006
+
2007
+ test("extractTestFailureCount: uses last fail line when multiple present", () => {
2008
+ // bun test can print intermediate summaries
2009
+ const output = `
2010
+ 10 pass
2011
+ 2 fail
2012
+
2013
+ 52 pass
2014
+ 5 fail
2015
+ `;
2016
+ expect(extractTestFailureCount(output)).toBe(5);
2017
+ });
2018
+ });