rungate 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/lib/affected-tests.ts +398 -0
  2. package/lib/agent-audit.ts +2 -3
  3. package/lib/agent-worktree.ts +287 -0
  4. package/lib/agents-md-sections.ts +81 -16
  5. package/lib/behavioral-cache.ts +150 -2
  6. package/lib/branch-protection.ts +790 -0
  7. package/lib/brief-ac-alignment.ts +104 -0
  8. package/lib/brief-references.ts +205 -0
  9. package/lib/bun-version.ts +93 -14
  10. package/lib/check-remedy.ts +140 -0
  11. package/lib/ci-checkout.ts +17 -0
  12. package/lib/ci-history-depth.ts +159 -0
  13. package/lib/ci-setup-step.ts +96 -0
  14. package/lib/claude-code-action.ts +85 -0
  15. package/lib/claude-tool-vocabulary.ts +188 -0
  16. package/lib/cli-commands.ts +15 -0
  17. package/lib/commit-sha.ts +42 -0
  18. package/lib/commit-subject.ts +90 -0
  19. package/lib/compliance-report.ts +377 -45
  20. package/lib/compliance.ts +191 -17
  21. package/lib/config-loader.ts +39 -1
  22. package/lib/config-trackable.ts +391 -0
  23. package/lib/conformity.ts +1087 -100
  24. package/lib/containment.ts +238 -0
  25. package/lib/content-hash.ts +24 -0
  26. package/lib/create-sc.ts +249 -16
  27. package/lib/dev-liveness.ts +180 -0
  28. package/lib/discovery-branch-points.ts +432 -0
  29. package/lib/discovery-stability.ts +481 -0
  30. package/lib/distribution-currency.ts +186 -0
  31. package/lib/doc-routing.ts +157 -0
  32. package/lib/evidence-prevalidator.ts +224 -2
  33. package/lib/evidence-satisfiability.ts +534 -12
  34. package/lib/failure-intake.ts +317 -0
  35. package/lib/fallow-availability.ts +157 -0
  36. package/lib/fs-write-capabilities.ts +232 -0
  37. package/lib/gate-run-split.ts +147 -0
  38. package/lib/generators/agents-md.ts +140 -32
  39. package/lib/git-isolation.ts +140 -0
  40. package/lib/github.ts +15 -0
  41. package/lib/google-auth-action.ts +93 -0
  42. package/lib/harness-project.ts +48 -0
  43. package/lib/harness-provenance.ts +255 -0
  44. package/lib/hook-evidence.ts +235 -0
  45. package/lib/hook-scopes.ts +386 -0
  46. package/lib/hook-target-tree.ts +398 -0
  47. package/lib/hook-trees.ts +447 -0
  48. package/lib/js-top-level-code.ts +78 -10
  49. package/lib/junit-checks.ts +52 -0
  50. package/lib/launch-base.ts +253 -0
  51. package/lib/line-citation.ts +372 -0
  52. package/lib/orphan-reaper.ts +468 -0
  53. package/lib/parity-binding.ts +205 -0
  54. package/lib/per-test-timeouts.ts +219 -0
  55. package/lib/plan-coverage.ts +151 -0
  56. package/lib/prior-branch.ts +201 -8
  57. package/lib/process-table.ts +146 -0
  58. package/lib/project-state-derivation.ts +219 -0
  59. package/lib/project-state-root.ts +258 -0
  60. package/lib/published-artifact.ts +525 -0
  61. package/lib/readdir-sorted.ts +45 -0
  62. package/lib/review-execution-reading.ts +206 -0
  63. package/lib/review-tool-agreement.ts +433 -0
  64. package/lib/rule-promotion.ts +234 -0
  65. package/lib/rule-retirement.ts +525 -0
  66. package/lib/run-branch.ts +612 -0
  67. package/lib/rungate-schema.ts +19 -0
  68. package/lib/runner-home.ts +274 -0
  69. package/lib/sc-id-cross-tree.ts +19 -7
  70. package/lib/sc-ids.ts +121 -13
  71. package/lib/sc-reservations.ts +289 -0
  72. package/lib/scaffold/ai-review-findings.ts +403 -0
  73. package/lib/scaffold/ai-review-step.ts +4661 -64
  74. package/lib/scaffold/managed-workflow.ts +587 -192
  75. package/lib/scaffold/owned-files.ts +152 -45
  76. package/lib/scaffold/pre-push-hook.ts +49 -5
  77. package/lib/scaffold/steps.ts +1413 -244
  78. package/lib/scaffold/workflow-census.ts +217 -0
  79. package/lib/scaffold/workflow-drift.ts +217 -0
  80. package/lib/scaffold/write-steps.ts +4 -0
  81. package/lib/scan-budget.ts +247 -0
  82. package/lib/scanner.ts +43 -51
  83. package/lib/scratch-roots.ts +50 -5
  84. package/lib/screenshot-discovery.ts +121 -0
  85. package/lib/screenshot-upload.ts +41 -1
  86. package/lib/secret-patterns.ts +511 -8
  87. package/lib/ship-mode.ts +219 -0
  88. package/lib/source-slice.ts +294 -0
  89. package/lib/stale-cleanup.ts +37 -3
  90. package/lib/suite-flake.ts +278 -0
  91. package/lib/suite-measurement.ts +23 -4
  92. package/lib/suite-run-reading.ts +241 -1
  93. package/lib/suite-timeout-margin.ts +193 -0
  94. package/lib/test-case-bodies.ts +883 -0
  95. package/lib/test-case-deletion.ts +653 -0
  96. package/lib/test-census.ts +97 -29
  97. package/lib/test-command-positions.ts +274 -0
  98. package/lib/test-suite-lock.ts +159 -16
  99. package/lib/tool-probe.ts +31 -9
  100. package/lib/top-level-cd.ts +393 -0
  101. package/lib/tree-entries.ts +89 -0
  102. package/lib/ts-ast.ts +200 -0
  103. package/lib/walk-test-files.ts +97 -0
  104. package/lib/workflow-actions.ts +263 -0
  105. package/lib/worktree-collect.ts +177 -5
  106. package/lib/worktree-repo.ts +36 -1
  107. package/package.json +2 -1
  108. package/scripts/ai-review-findings.ts +45 -0
  109. package/scripts/create-sc.ts +21 -1
  110. package/scripts/extract-constraints.ts +107 -7
  111. package/scripts/generate-code-map.ts +23 -30
  112. package/scripts/scaffold-project.ts +8 -1
  113. package/scripts/sync-sc-status.ts +77 -16
package/lib/conformity.ts CHANGED
@@ -5,8 +5,9 @@
5
5
  * import { runScaffoldConformity, runSpecDiscovery, runSpecDrift } from "rungate/lib/conformity";
6
6
  * runScaffoldConformity(import.meta.dir + "/..");
7
7
  */
8
- import { describe, test, expect } from "bun:test";
9
- import { existsSync, readFileSync, readdirSync, realpathSync, statSync, writeFileSync } from "fs";
8
+ import { beforeAll, describe, test, expect } from "bun:test";
9
+ import { existsSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from "fs";
10
+ import { tmpdir } from "os";
10
11
  import type { Dirent } from "fs";
11
12
  import { collectSpecFiles } from "./spec-files";
12
13
  import {
@@ -21,14 +22,56 @@ import { spawnSync } from "child_process";
21
22
  import { join, dirname, sep } from "path";
22
23
  import { fileURLToPath } from "url";
23
24
  import { deriveDirectoryName } from "../scripts/split-spec";
24
- import { readFreshCache } from "./behavioral-cache";
25
+ import {
26
+ type BehavioralCacheState,
27
+ type BehavioralCensus,
28
+ type BehavioralReadingKind,
29
+ behavioralCachePath,
30
+ behavioralCensus,
31
+ behavioralCensusLine,
32
+ behavioralReading,
33
+ readCache,
34
+ } from "./behavioral-cache";
25
35
  import { tryLoadRungateConfig } from "./config-loader";
36
+ import {
37
+ type ProtectionProbe,
38
+ githubProtectionProbe,
39
+ protectionSubject,
40
+ readMergeSerialisation,
41
+ readPushCi,
42
+ } from "./branch-protection";
26
43
  import { stripJsComments } from "./strip-js-comments";
44
+ import { remedyMessage } from "./check-remedy";
27
45
  import { parseScDeclarations } from "./sc-ids";
28
46
  import { isHarnessItself } from "./harness-identity";
29
- import { fallowAvailable } from "./fallow-availability";
47
+ import {
48
+ FALLOW_BUDGET_MS,
49
+ fallowAvailable,
50
+ runFallowJson,
51
+ warmFallow,
52
+ type FallowReading,
53
+ type FallowUnavailable,
54
+ } from "./fallow-availability";
55
+ import {
56
+ briefReferenceReading,
57
+ unmeasuredBriefReferenceNotice,
58
+ unresolvableBriefReferenceMessage,
59
+ } from "./brief-references";
30
60
  import { tsconfigVerdict, TS_SOURCE_DIRS } from "./typescript-project";
31
61
  import { readSuccessDoc, readSuccessTemplate, successViolations } from "./success-doc";
62
+ import {
63
+ AI_REVIEW_AUTH_PRECEDENCE,
64
+ AI_REVIEW_AUTH_STEP_ID,
65
+ AI_REVIEW_JOB,
66
+ AI_REVIEW_MODES,
67
+ AI_REVIEW_MODE_NONE,
68
+ AI_REVIEW_MODE_VERTEX,
69
+ AI_REVIEW_REQUIRE_VAR,
70
+ AI_REVIEW_VERTEX_AUTH_STEP_ID,
71
+ AI_REVIEW_WORKFLOW_FILE,
72
+ } from "./scaffold/ai-review-step";
73
+ import { workflowOverrides } from "./scaffold/managed-workflow";
74
+ import { workflowDrift } from "./scaffold/workflow-drift";
32
75
 
33
76
  // ── Shared utilities ────────────────────────────────────────
34
77
 
@@ -179,6 +222,11 @@ export function clearFindings(): void {
179
222
  _candidates.length = 0;
180
223
  _staleness.length = 0;
181
224
  _behavioralCount = 0;
225
+ for (const k of Object.keys(_behavioralReadings) as BehavioralReadingKind[]) {
226
+ _behavioralReadings[k] = 0;
227
+ }
228
+ _behavioralCacheStates = [];
229
+ _behavioralUnmeasured.length = 0;
182
230
  }
183
231
 
184
232
  export function addBehavioralCount(count: number): void {
@@ -189,6 +237,56 @@ export function getBehavioralCount(): number {
189
237
  return _behavioralCount;
190
238
  }
191
239
 
240
+ /**
241
+ * Every ticked `(behavioral)` criterion the run saw, and what the cache said
242
+ * about it (#758).
243
+ *
244
+ * `_behavioralCount` counts the ones that were ROUTED — the ones with no
245
+ * fresh reading — and a criterion that was measured and passed never reaches
246
+ * it. So `behavioralCount: 0` is produced by a run in which everything was
247
+ * measured AND by a run in which there are no behavioral criteria at all,
248
+ * and the number that matters — how many went unread — was never written
249
+ * down anywhere.
250
+ */
251
+ const _behavioralReadings: Record<BehavioralReadingKind, number> = {
252
+ "fresh-pass": 0, "fresh-fail": 0, stale: 0, absent: 0,
253
+ };
254
+ let _behavioralCacheStates: BehavioralCacheState[] = [];
255
+ const _behavioralUnmeasured: string[] = [];
256
+
257
+ export function addBehavioralCensus(c: BehavioralCensus): void {
258
+ for (const k of Object.keys(_behavioralReadings) as BehavioralReadingKind[]) {
259
+ _behavioralReadings[k] += c.counts[k];
260
+ }
261
+ if (!_behavioralCacheStates.includes(c.cacheState)) _behavioralCacheStates.push(c.cacheState);
262
+ _behavioralUnmeasured.push(...c.unmeasured);
263
+ }
264
+
265
+ /**
266
+ * The run's whole behavioral reading, as a figure.
267
+ *
268
+ * `reading` is derived here and nowhere else, from the same lists the counts
269
+ * are projections of — a second boolean beside it would always agree with it.
270
+ * An empty population is UNMEASURED, because zero unread criteria out of zero
271
+ * criteria is the number a scan that collected nothing also produces.
272
+ */
273
+ export function getBehavioralReading(): {
274
+ reading: "MEASURED" | "UNMEASURED";
275
+ counts: Record<BehavioralReadingKind, number>;
276
+ unmeasured: string[];
277
+ cacheStates: BehavioralCacheState[];
278
+ total: number;
279
+ } {
280
+ const total = Object.values(_behavioralReadings).reduce((a, b) => a + b, 0);
281
+ return {
282
+ reading: total > 0 && _behavioralUnmeasured.length === 0 ? "MEASURED" : "UNMEASURED",
283
+ counts: { ..._behavioralReadings },
284
+ unmeasured: [..._behavioralUnmeasured],
285
+ cacheStates: [..._behavioralCacheStates],
286
+ total,
287
+ };
288
+ }
289
+
192
290
  // ── Auto-flip: track passing SCs and flip checkboxes ────────
193
291
 
194
292
  interface PassingSC {
@@ -257,6 +355,7 @@ export function writeFindingsReport(root: string): string {
257
355
  warnings: _findings.filter(f => f.severity === "WARN").length,
258
356
  candidateCount: _candidates.length,
259
357
  behavioralCount: _behavioralCount,
358
+ behavioral: getBehavioralReading(),
260
359
  staleCount: _staleness.length,
261
360
  findings: _findings,
262
361
  constraintCandidates: _candidates,
@@ -321,8 +420,12 @@ export function getBehavioralRouting(sc: ParsedSC): string | null {
321
420
  * Stale entries (>= 7 days old) return null so they're treated as unmatchable.
322
421
  */
323
422
  export function behavioralPattern(sc: ParsedSC, cachePath: string): ((root: string) => void) | null {
324
- const cache = readFreshCache(cachePath);
325
- const entry = cache[sc.id];
423
+ const entry = readCache(cachePath)[sc.id];
424
+ // One decider for "is there a fresh reading", shared with the census that
425
+ // reports how many there are (#758). Asking `readFreshCache` here and
426
+ // `behavioralReading` there would be two guards over one question, always
427
+ // agreeing, and the count would be unfalsifiable against the routing.
428
+ if (!behavioralReading(entry).startsWith("fresh")) return null;
326
429
  if (!entry) return null;
327
430
 
328
431
  return (_root: string) => {
@@ -1232,7 +1335,28 @@ export type SCPlanEntry =
1232
1335
  */
1233
1336
  export function planSCTests(scs: ParsedSC[], projectRoot?: string): SCPlanEntry[] {
1234
1337
  return scs.map((sc): SCPlanEntry => {
1235
- if (isBehavioralSC(sc)) return { kind: "behavioral", sc, routing: getBehavioralRouting(sc) };
1338
+ if (isBehavioralSC(sc)) {
1339
+ // Ask the cache BEFORE classifying. This line used to classify first,
1340
+ // which made `behavioralPattern` — the function that turns a recorded
1341
+ // runtime result into an assertion that throws — unreachable from any
1342
+ // real run: defined, exported, unit-tested, and called by nothing but
1343
+ // its own tests. Meanwhile `scripts/sync-sc-status.ts` DID read the
1344
+ // cache and would move a `passed: false` criterion into `failing`, so
1345
+ // the decider that TICKS a checkbox looked at the evidence and the
1346
+ // suite that decides the build's colour did not (#328).
1347
+ //
1348
+ // `behavioralPattern` returns null for an absent entry and for a stale
1349
+ // one — `behavioralReading` calls both `absent` and `stale` — so a
1350
+ // criterion with no current reading falls through to `behavioral` as
1351
+ // before: reported, ratcheted, UNMEASURED. A stale PASS must not
1352
+ // certify, because a measurement taken a fortnight ago read as one
1353
+ // taken now is the silent default this whole subsystem exists to avoid.
1354
+ const assertion = projectRoot
1355
+ ? behavioralPattern(sc, behavioralCachePath(projectRoot))
1356
+ : null;
1357
+ if (assertion) return { kind: "assert", sc, assertion };
1358
+ return { kind: "behavioral", sc, routing: getBehavioralRouting(sc) };
1359
+ }
1236
1360
  if (!sc.checked) return { kind: "todo", sc };
1237
1361
  const outcome = resolveMatcher(sc, projectRoot);
1238
1362
  if (outcome.kind === "assert") return { kind: "assert", sc, assertion: outcome.assertion };
@@ -1292,6 +1416,25 @@ export function runScaffoldConformity(root: string, opts?: { extraSpecDirs?: str
1292
1416
  // and it was not hypothetical: the planted-project test in
1293
1417
  // test/matcher-registry-parity.test.ts reported its criterion as
1294
1418
  // `unmatched` until this argument was supplied.
1419
+ // The census over EVERY ticked behavioral criterion in this spec,
1420
+ // measured or not (#758).
1421
+ //
1422
+ // It cannot be read off `behavioral` below: that list is appended to
1423
+ // only when the plan routed a criterion, which happens only when the
1424
+ // cache had nothing fresh — so it is already the unmeasured half and
1425
+ // carries no denominator. A criterion that was measured and passed is
1426
+ // an ordinary `assert` entry and is invisible to it.
1427
+ //
1428
+ // No second decider: `behavioralCensus` classifies through
1429
+ // `behavioralReading`, which is the function `planSCTests` routed on
1430
+ // one line above.
1431
+ const tickedBehavioral = metadata.scs.filter(sc => sc.checked && isBehavioralSC(sc));
1432
+ if (tickedBehavioral.length > 0) {
1433
+ const census = behavioralCensus(tickedBehavioral.map(sc => sc.id), behavioralCachePath(root));
1434
+ addBehavioralCensus(census);
1435
+ console.log(`${specFile} — ${behavioralCensusLine(census)}`);
1436
+ }
1437
+
1295
1438
  for (const entry of planSCTests(metadata.scs, root)) {
1296
1439
  const { sc } = entry;
1297
1440
  if (entry.kind === "behavioral") {
@@ -1360,14 +1503,30 @@ export function runScaffoldConformity(root: string, opts?: { extraSpecDirs?: str
1360
1503
  // to — 54 of them on the day they first became visible (#209).
1361
1504
  //
1362
1505
  // Failing 54 specs at once would get the whole file skipped, so the
1363
- // teeth live in a single repo-wide ratchet instead:
1364
- // test/conformity-sc-binding.test.ts asserts the total may fall but
1365
- // never rise. Here we name them, per spec, so they are findable.
1506
+ // teeth live in a repo-wide ratchet instead. That ratchet has MOVED:
1507
+ // `UNMATCHED_CHECKED_RATCHET` in test/conformity-sc-binding.test.ts
1508
+ // is gone (#441, and that file's line 44 says so), replaced by the
1509
+ // two-sided bounds named below. Here we name the SCs, per spec, so
1510
+ // they are findable.
1511
+ //
1512
+ // A report, not a test (#211). This slot held
1513
+ // `expect(unmatched.length).toBeGreaterThan(0)` INSIDE
1514
+ // `if (unmatched.length > 0)` — true by construction, with no input
1515
+ // on which it could go red. That is #328's defect verbatim, twenty
1516
+ // lines below the branch #328 repaired, and it was minting one
1517
+ // passing test per strict spec out of a number nothing asserts.
1518
+ //
1519
+ // The repair is #328's: the block says what it found and stops
1520
+ // claiming to have checked it. The teeth stay where #441 put them —
1521
+ // test/sc-binding-split.test.ts bounds UNCLAIMED_RATCHET above and
1522
+ // CLAIMED_UNBINDABLE in both directions — deliberately NOT a
1523
+ // per-spec ratchet to zero, because that number measures the
1524
+ // matcher's vocabulary and the only way to lower it is to make the
1525
+ // specs say less.
1366
1526
  const level = metadata.compliance === "permissive" ? "WARN" : "WARN (strict)";
1367
- test(`${level}: ${unmatched.length} checked SCs have no pattern matcher`, () => {
1368
- console.warn(`Unmatched checked SCs in ${specFile}:\n ${unmatched.join("\n ")}`);
1369
- expect(unmatched.length).toBeGreaterThan(0);
1370
- });
1527
+ console.warn(
1528
+ `${level}: ${unmatched.length} checked SCs have no pattern matcher in ${specFile}:\n ${unmatched.join("\n ")}`,
1529
+ );
1371
1530
  }
1372
1531
  // Deliberately BELOW the unmatched branch, not between it and the
1373
1532
  // behavioral report. #328's guard slices the source from the
@@ -1450,25 +1609,95 @@ export function runSpecDiscovery(root: string) {
1450
1609
  expect(missing).toEqual([]);
1451
1610
  });
1452
1611
 
1453
- test("at least one testable spec exists", () => {
1454
- if (!existsSync(specsDir)) return;
1455
- const testable = collectSpecFiles(specsDir).filter(f => {
1456
- const fm = parseFrontmatter(readFileSync(join(specsDir, f), "utf-8"));
1457
- return fm?.testable === "true";
1612
+ const countTestable = () => collectSpecFiles(specsDir).filter(f => {
1613
+ const fm = parseFrontmatter(readFileSync(join(specsDir, f), "utf-8"));
1614
+ return fm?.testable === "true";
1615
+ }).length;
1616
+
1617
+ // #211. This test had three paths and no failing input:
1618
+ //
1619
+ // specs/ absent -> `if (!existsSync(specsDir)) return;` -> PASS
1620
+ // specs/ with 0 testable -> addFinding(WARN) + console.warn -> PASS
1621
+ // specs/ with N testable -> expect(N).toBeGreaterThan(0) -> PASS
1622
+ //
1623
+ // The third is #328's tautology — the assertion sat in the `else` of
1624
+ // `testable.length === 0`, so the only branch that reached it already knew
1625
+ // the answer. The first two are worse than vacuous: a test NAMED "at least
1626
+ // one testable spec exists" reported green on the two trees where no
1627
+ // testable spec exists. A tree that could not be measured had not passed,
1628
+ // and a gap that was tolerated had not been checked; both were being read
1629
+ // as evidence.
1630
+ //
1631
+ // The zero-testable grace is kept, because it is load-bearing: a scaffolded
1632
+ // consumer gets `testable: no` (lib/scaffold/write-steps.ts), so failing
1633
+ // here would turn every fresh consumer red on day one. It is kept as
1634
+ // `test.todo` — the honest report of a known-false property — rather than
1635
+ // as a pass.
1636
+ //
1637
+ // The probe is TOTAL, and #1027 left it partial. `countTestable` does a
1638
+ // `readFileSync` per spec file and ran bare in the `describe` body, so a
1639
+ // file that vanished between `collectSpecFiles` and the read — or a
1640
+ // dangling symlink, or an unreadable mode — threw at REGISTRATION time.
1641
+ // A throw there does not fail this check: it aborts registration of
1642
+ // everything after it in the file, and a check that never registered
1643
+ // reports as not existing rather than as failing.
1644
+ //
1645
+ // Second order, and the worse half: `writeFindingsReport` runs from an
1646
+ // `afterAll` that an aborted registration never reaches, so the previous
1647
+ // run's `.rungate/conformity-findings.json` stays on disk and
1648
+ // `readConformityFindings` (gates/gate-executor.ts) consumes it as
1649
+ // current — byte-indistinguishable from a fresh clean one.
1650
+ //
1651
+ // That is the same sentence as the comment above, reached by exception
1652
+ // instead of by reading: a tree that could not be measured has not
1653
+ // passed. So the read is caught here and reported as UNMEASURED.
1654
+ let measured: number | null = null;
1655
+ let probeError: unknown = null;
1656
+ try {
1657
+ measured = existsSync(specsDir) ? countTestable() : null;
1658
+ } catch (e) {
1659
+ probeError = e;
1660
+ }
1661
+
1662
+ if (probeError !== null) {
1663
+ const why = probeError instanceof Error ? probeError.message : String(probeError);
1664
+ addFinding({
1665
+ ruleId: "SPEC-DISCOVERY-UNMEASURED",
1666
+ severity: "FAIL",
1667
+ file: "specs/",
1668
+ message: `UNMEASURED: a spec under specs/ could not be read, so testability was never measured — ${why}`,
1669
+ fixCommand: "Repair or remove the unreadable file named in the message, then re-run",
1458
1670
  });
1459
- if (testable.length === 0) {
1460
- addFinding({
1461
- ruleId: "SPEC-DISCOVERY-TESTABLE",
1462
- severity: "WARN",
1463
- file: "specs/",
1464
- message: "No spec has testable: true. Add testable: true to a spec when ready for conformity testing.",
1465
- fixCommand: `Add "testable: true" to frontmatter of a spec in specs/`,
1466
- });
1467
- console.warn("SPEC-DISCOVERY: No testable specs found — skipping (add testable: true when ready)");
1468
- } else {
1469
- expect(testable.length).toBeGreaterThan(0);
1470
- }
1471
- });
1671
+ // A FAILING test, deliberately not the `test.todo` below. That todo is
1672
+ // the grace for a tree with nothing marked testable — a fact this run
1673
+ // READ. A tree it could not read is a different fact and must not
1674
+ // borrow that grace, which is exactly what a per-file
1675
+ // `catch { return false }` inside the filter would have let it do.
1676
+ test("at least one testable spec exists (specs/ could not be read)", () => {
1677
+ expect(
1678
+ probeError,
1679
+ `specs/ could not be read, so testability was never measured: ${why}`,
1680
+ ).toBeNull();
1681
+ });
1682
+ } else if (measured === 0) {
1683
+ addFinding({
1684
+ ruleId: "SPEC-DISCOVERY-TESTABLE",
1685
+ severity: "WARN",
1686
+ file: "specs/",
1687
+ message: "No spec has testable: true. Add testable: true to a spec when ready for conformity testing.",
1688
+ fixCommand: `Add "testable: true" to frontmatter of a spec in specs/`,
1689
+ });
1690
+ test.todo("at least one testable spec exists (none marked testable: true yet)");
1691
+ } else {
1692
+ test("at least one testable spec exists", () => {
1693
+ // Deliberately duplicates the "specs/ directory exists" check above.
1694
+ // That one answers "is the directory there"; this one refuses to
1695
+ // answer its OWN question over a tree it could not read, which is the
1696
+ // early `return` this replaces. Unmeasured is a failure, not a pass.
1697
+ expect(existsSync(specsDir), "specs/ is absent, so testability was never measured").toBe(true);
1698
+ expect(countTestable()).toBeGreaterThan(0);
1699
+ });
1700
+ }
1472
1701
  });
1473
1702
  }
1474
1703
 
@@ -1575,12 +1804,35 @@ export function runSpecDrift(
1575
1804
  }
1576
1805
 
1577
1806
  const newlyOrphaned = orphaned.filter(s => !untestedAllowlist.includes(s));
1578
- expect(newlyOrphaned).toEqual([]);
1807
+ expect(
1808
+ newlyOrphaned,
1809
+ remedyMessage({
1810
+ root,
1811
+ check: "DRIFT-2",
1812
+ violations: newlyOrphaned,
1813
+ fix:
1814
+ "name the spec in a test — a `test(\"SC-N: ...\")` title counts as a declaration. " +
1815
+ "Do NOT add it to .claude/conformity-allowlists.json: DRIFT-3 diffs that file " +
1816
+ "against origin/main and fails any array that grew, so the obvious escape hatch " +
1817
+ "turns one red check into two.",
1818
+ }),
1819
+ ).toEqual([]);
1579
1820
 
1580
1821
  const staleAllowlist = untestedAllowlist.filter(
1581
1822
  s => testable.includes(s) && !orphaned.includes(s),
1582
1823
  );
1583
- expect(staleAllowlist).toEqual([]);
1824
+ expect(
1825
+ staleAllowlist,
1826
+ remedyMessage({
1827
+ root,
1828
+ check: "DRIFT-2 (stale exemption)",
1829
+ violations: staleAllowlist,
1830
+ fix:
1831
+ "these specs now HAVE a referencing test, so delete their entries from " +
1832
+ "untestedSpecs in .claude/conformity-allowlists.json. Shrinking the list is " +
1833
+ "always allowed; only growth is refused.",
1834
+ }),
1835
+ ).toEqual([]);
1584
1836
  });
1585
1837
 
1586
1838
  test("DRIFT-3: the exemption lists did not GROW", () => {
@@ -1795,10 +2047,34 @@ export function runDocHygiene(
1795
2047
  }
1796
2048
 
1797
2049
  const newOrphans = orphans.filter(o => !orphanAllowlist.includes(o));
1798
- expect(newOrphans).toEqual([]);
2050
+ expect(
2051
+ newOrphans,
2052
+ remedyMessage({
2053
+ root,
2054
+ check: "HYGIENE-3",
2055
+ violations: newOrphans,
2056
+ fix:
2057
+ "make something genuinely reference the file — a routing row, a spec, an import. " +
2058
+ "Do NOT add it to .claude/conformity-allowlists.json: DRIFT-3 diffs that file " +
2059
+ "against origin/main and fails any array that grew, so the obvious escape hatch " +
2060
+ "turns one red check into two. If the file is history rather than guidance, " +
2061
+ "move it under reference/ instead.",
2062
+ }),
2063
+ ).toEqual([]);
1799
2064
 
1800
2065
  const staleAllowlist = orphanAllowlist.filter(o => !orphans.includes(o));
1801
- expect(staleAllowlist).toEqual([]);
2066
+ expect(
2067
+ staleAllowlist,
2068
+ remedyMessage({
2069
+ root,
2070
+ check: "HYGIENE-3 (stale exemption)",
2071
+ violations: staleAllowlist,
2072
+ fix:
2073
+ "these files are now referenced, so delete their entries from orphanFiles in " +
2074
+ ".claude/conformity-allowlists.json. Shrinking the list is always allowed; " +
2075
+ "only growth is refused.",
2076
+ }),
2077
+ ).toEqual([]);
1802
2078
  });
1803
2079
 
1804
2080
  test("HYGIENE-4: Every spec appears in the specs routing table", () => {
@@ -1816,7 +2092,18 @@ export function runDocHygiene(
1816
2092
  // SPEC-TEMPLATE.md documents how to write a spec; it governs nothing.
1817
2093
  .filter(f => f !== "SPEC-TEMPLATE.md");
1818
2094
  const unlisted = specFiles.filter(f => !routing.includes(f));
1819
- expect(unlisted).toEqual([]);
2095
+ expect(
2096
+ unlisted,
2097
+ remedyMessage({
2098
+ root,
2099
+ check: "HYGIENE-4",
2100
+ violations: unlisted,
2101
+ fix:
2102
+ "add a row naming each file to .claude/rules/specs-routing.md — the table this " +
2103
+ "check reads. A spec with no `- [ ]` criteria needs its row to say so explicitly. " +
2104
+ "There is no exemption list for this one.",
2105
+ }),
2106
+ ).toEqual([]);
1820
2107
  });
1821
2108
 
1822
2109
  test("HYGIENE-5: reference/ contents not in active routing", () => {
@@ -1991,6 +2278,25 @@ export function runDocHygiene(
1991
2278
  `HYGIENE-6 coverage: read ${result.coverage.read} of ${result.coverage.planned} planned documents ` +
1992
2279
  `from ${result.scan.roots.length} directories (${result.scan.source})`,
1993
2280
  );
2281
+
2282
+ // #545/#544. The failure intake and the promotion decision are reported
2283
+ // here and asserted nowhere: HYGIENE-6 is advisory by declaration, and a
2284
+ // new refusal in it would turn a backlog report into a gate.
2285
+ //
2286
+ // What it is NOT allowed to be is silent. The intake reads `git log`,
2287
+ // and `actions/checkout@v4` with no `fetch-depth` clones to depth 1, so
2288
+ // on a runner this line says UNMEASURED/SHALLOW_HISTORY — which is the
2289
+ // state, printed, rather than a zero that reads as "no failures found".
2290
+ const { promotionSummary } = await import("./rule-promotion");
2291
+ console.log(
2292
+ `HYGIENE-6 intake: ${result.evidence.reading}, ` +
2293
+ `${result.evidence.candidates.length} evidence-backed candidates ` +
2294
+ `over ${result.evidence.commitsScanned} commits` +
2295
+ (result.evidence.unmeasuredReasons.length > 0
2296
+ ? ` — ${result.evidence.unmeasuredReasons.join("; ")}`
2297
+ : ""),
2298
+ );
2299
+ console.log(`HYGIENE-6 ${promotionSummary(result.promotion)}`);
1994
2300
  expect(
1995
2301
  result.coverage.planned,
1996
2302
  "the documentation scan planned no files, so this check measured nothing",
@@ -2364,24 +2670,35 @@ export function runAgentFileValidation(root: string) {
2364
2670
 
2365
2671
  test("AGENT-6: All file paths referenced in briefs exist", () => {
2366
2672
  if (!existsSync(agentsDir)) return;
2367
- const broken: string[] = [];
2368
- for (const f of readdirSync(agentsDir).filter(f => f.endsWith(".md"))) {
2369
- const content = readFileSync(join(agentsDir, f), "utf-8");
2370
- const refs = content.match(/node_modules\/[^\s`*)"']+/g) || [];
2371
- for (const ref of refs) {
2372
- const clean = ref.replace(/[`*)"']+$/, "");
2373
- if (!existsSync(join(root, clean))) {
2374
- broken.push(`${f} → ${clean}`);
2375
- addFinding({
2376
- ruleId: "AGENT-BROKEN-REF",
2377
- severity: "FAIL",
2378
- file: `.claude/agents/${f}`,
2379
- message: `Brief references "${clean}" but file does not exist`,
2380
- fixCommand: `Check prompts/ directory for correct filename — may be a typo or renamed file`,
2381
- });
2382
- }
2383
- }
2673
+ // #884: a reference into a package that is not installed here is
2674
+ // UNMEASURED, not broken. A ship worktree has no install, and reporting
2675
+ // its missing corpus as the project's test health failed `tests-pass` on
2676
+ // every run. The three outcomes live in `lib/brief-references.ts`.
2677
+ const reading = briefReferenceReading(root, agentsDir);
2678
+
2679
+ for (const u of reading.unresolvable) {
2680
+ addFinding({
2681
+ ruleId: "AGENT-REF-UNMEASURED",
2682
+ severity: "WARN",
2683
+ file: `.claude/agents/${u.brief}`,
2684
+ message: unresolvableBriefReferenceMessage(u),
2685
+ fixCommand: `Install dependencies in this checkout and re-run — "${u.packageRoot}" is absent, so AGENT-6 could not check this reference`,
2686
+ });
2384
2687
  }
2688
+ for (const b of reading.broken) {
2689
+ addFinding({
2690
+ ruleId: "AGENT-BROKEN-REF",
2691
+ severity: "FAIL",
2692
+ file: `.claude/agents/${b.brief}`,
2693
+ message: `Brief references "${b.ref}" but file does not exist`,
2694
+ fixCommand: `Check prompts/ directory for correct filename — may be a typo or renamed file`,
2695
+ });
2696
+ }
2697
+
2698
+ const notice = unmeasuredBriefReferenceNotice(reading);
2699
+ if (notice !== null) console.error(notice);
2700
+
2701
+ const broken = reading.broken.map(b => `${b.brief} → ${b.ref}`);
2385
2702
  if (broken.length > 0) {
2386
2703
  console.error(`Broken references in agent briefs:\n ${broken.join("\n ")}`);
2387
2704
  }
@@ -2467,76 +2784,86 @@ interface FallowResult {
2467
2784
  // months, which is the defect #581 is about.
2468
2785
  export { fallowAvailable };
2469
2786
 
2470
- /** Shared skip notice — a skipped check must say so, not read as a pass. */
2471
- function fallowSkipped(check: string): void {
2472
- console.warn(
2473
- `SKIP ${check}: fallow is not installed (not a declared dependency) — ` +
2474
- `this check did NOT run. Install it to enable static analysis.`,
2475
- );
2787
+ /**
2788
+ * Shared skip notice — a skipped check must say so, not read as a pass, and
2789
+ * must say WHICH silence it is (#935).
2790
+ *
2791
+ * It used to take only the check name and print one sentence: *"fallow is not
2792
+ * installed (not a declared dependency)"*. That sentence was printed for a
2793
+ * run the clock killed, on trees where fallow is an exactly-pinned declared
2794
+ * dependency, because `runFallowCommand` returned `null` for both. The
2795
+ * reading is passed in now, so the notice is the reading's own sentence
2796
+ * rather than a guess about which of three things happened.
2797
+ */
2798
+ function fallowSkipped(check: string, why: FallowUnavailable): void {
2799
+ console.warn(`SKIP ${check}: ${why.reason}. This check did NOT run.`);
2476
2800
  }
2477
2801
 
2478
- function runFallowCommand(root: string, args: string[]): FallowResult | null {
2479
- if (!fallowAvailable(root)) return null;
2480
- const result = Bun.spawnSync(["npx", "fallow", ...args, "--format", "json", "--quiet"], {
2481
- cwd: root,
2482
- env: { ...process.env, NODE_NO_WARNINGS: "1" },
2483
- timeout: 30_000,
2484
- });
2485
- const stdout = result.stdout.toString().trim();
2486
- if (!stdout || stdout.startsWith("{\"error\"")) return null;
2487
- try {
2488
- return JSON.parse(stdout);
2489
- } catch {
2490
- return null;
2491
- }
2802
+ /**
2803
+ * The reading, or the refusal that says why there isn't one.
2804
+ *
2805
+ * Delegates to `lib/fallow-availability.ts` — one spawn in the tree, one
2806
+ * budget, and the timeout classified before the empty-stdout branch. This
2807
+ * function used to hold its own copy of all three (#935).
2808
+ */
2809
+ function runFallowCommand(root: string, args: string[]): FallowReading<FallowResult> {
2810
+ return runFallowJson<FallowResult>(root, args);
2492
2811
  }
2493
2812
 
2494
2813
  export function runFallowCheck(root: string, opts?: { skipUnusedExports?: boolean; warnOnly?: boolean }) {
2495
2814
  const fail = !opts?.warnOnly;
2496
2815
  describe("Fallow: static analysis", () => {
2816
+ // Before the four checks, not inside the first of them. fallow builds its
2817
+ // index on first use — ~1.5s cold against ~455ms warm — and whichever
2818
+ // check ran first paid for it alone. That made FALLOW-1 look slow and the
2819
+ // other three look fast, which sent two investigations after FALLOW-1's
2820
+ // own query instead of after the cold start (#935).
2821
+ beforeAll(() => warmFallow(root), FALLOW_BUDGET_MS);
2822
+
2497
2823
  test("FALLOW-1: No unused files", () => {
2498
- const result = runFallowCommand(root, ["dead-code", "--unused-files"]);
2499
- if (!result) { fallowSkipped("FALLOW-1 (unused files)"); return; }
2824
+ const reading = runFallowCommand(root, ["dead-code", "--unused-files"]);
2825
+ if ("unavailable" in reading) { fallowSkipped("FALLOW-1 (unused files)", reading.unavailable); return; }
2826
+ const result = reading.data;
2500
2827
  const unused = result.unused_files.map(f => f.path).filter(p => !p.includes("test/fixtures/") && !p.includes("reference/"));
2501
2828
  if (unused.length > 0) {
2502
2829
  console.warn(`Unused files (${unused.length}):\n ${unused.slice(0, 10).join("\n ")}${unused.length > 10 ? `\n ... and ${unused.length - 10} more` : ""}`);
2503
2830
  }
2504
2831
  if (fail) expect(unused).toEqual([]);
2505
- });
2832
+ }, FALLOW_BUDGET_MS);
2506
2833
 
2507
2834
  if (!opts?.skipUnusedExports) {
2508
2835
  test("FALLOW-2: No unused exports", () => {
2509
- const result = runFallowCommand(root, ["dead-code", "--unused-exports"]);
2510
- if (!result) { fallowSkipped("FALLOW-2 (unused exports)"); return; }
2511
- const unused = (result.unused_exports || [])
2836
+ const reading = runFallowCommand(root, ["dead-code", "--unused-exports"]);
2837
+ if ("unavailable" in reading) { fallowSkipped("FALLOW-2 (unused exports)", reading.unavailable); return; }
2838
+ const unused = (reading.data.unused_exports || [])
2512
2839
  .filter(e => !e.path.includes("test/fixtures/") && !e.path.includes("reference/"))
2513
2840
  .map(e => `${e.path}:${e.line ?? "?"} ${e.export_name}`);
2514
2841
  if (unused.length > 0) {
2515
2842
  console.warn(`Unused exports (${unused.length}):\n ${unused.slice(0, 10).join("\n ")}${unused.length > 10 ? `\n ... and ${unused.length - 10} more` : ""}`);
2516
2843
  }
2517
2844
  if (fail) expect(unused).toEqual([]);
2518
- });
2845
+ }, FALLOW_BUDGET_MS);
2519
2846
  }
2520
2847
 
2521
2848
  test("FALLOW-3: No unused dependencies", () => {
2522
- const result = runFallowCommand(root, ["dead-code", "--unused-deps"]);
2523
- if (!result) { fallowSkipped("FALLOW-3 (unused dependencies)"); return; }
2524
- const unused = result.unused_dependencies?.map(d => d.name) || [];
2849
+ const reading = runFallowCommand(root, ["dead-code", "--unused-deps"]);
2850
+ if ("unavailable" in reading) { fallowSkipped("FALLOW-3 (unused dependencies)", reading.unavailable); return; }
2851
+ const unused = reading.data.unused_dependencies?.map(d => d.name) || [];
2525
2852
  if (unused.length > 0) {
2526
2853
  console.warn(`Unused dependencies (${unused.length}):\n ${unused.join("\n ")}`);
2527
2854
  }
2528
2855
  if (fail) expect(unused).toEqual([]);
2529
- });
2856
+ }, FALLOW_BUDGET_MS);
2530
2857
 
2531
2858
  test("FALLOW-4: No circular dependencies", () => {
2532
- const result = runFallowCommand(root, ["dead-code"]);
2533
- if (!result) { fallowSkipped("FALLOW-4 (circular dependencies)"); return; }
2534
- const circles = result.circular_dependencies || [];
2859
+ const reading = runFallowCommand(root, ["dead-code"]);
2860
+ if ("unavailable" in reading) { fallowSkipped("FALLOW-4 (circular dependencies)", reading.unavailable); return; }
2861
+ const circles = reading.data.circular_dependencies || [];
2535
2862
  if (circles.length > 0) {
2536
2863
  console.warn(`Circular dependencies (${circles.length}):\n ${circles.slice(0, 5).map(c => c.path || JSON.stringify(c.chain ?? c)).join("\n ")}${circles.length > 5 ? `\n ... and ${circles.length - 5} more` : ""}`);
2537
2864
  }
2538
2865
  if (fail) expect(circles).toEqual([]);
2539
- });
2866
+ }, FALLOW_BUDGET_MS);
2540
2867
  });
2541
2868
  }
2542
2869
 
@@ -2988,6 +3315,16 @@ export interface ConfigAccessScan {
2988
3315
  filesScanned: number;
2989
3316
  /** Bytes read. Zero with a non-zero file count means empty sources. */
2990
3317
  bytesScanned: number;
3318
+ /**
3319
+ * Files read by the out-of-reach fallback rather than by walking
3320
+ * `CONFIG_SCAN_DIRS` (#506). Empty on every ordinary scan.
3321
+ *
3322
+ * Recorded rather than merged invisibly because `coverageShortfall` is
3323
+ * about to be asked how many source files went unread, and these are
3324
+ * precisely the ones that would otherwise be counted as unread a line
3325
+ * after being read.
3326
+ */
3327
+ fallbackFiles: string[];
2991
3328
  }
2992
3329
 
2993
3330
  /**
@@ -3009,7 +3346,93 @@ export function scanConfigAccess(root: string): ConfigAccessScan {
3009
3346
  const n = classifyConfigSites(f.text).filter(site => site.kind === "use").length;
3010
3347
  if (n > 0) counts[f.path] = n;
3011
3348
  }
3012
- return { counts, filesScanned: files.length, bytesScanned };
3349
+ return { counts, filesScanned: files.length, bytesScanned, fallbackFiles: [] };
3350
+ }
3351
+
3352
+ /**
3353
+ * CONFIG-1's reading, with the files it could not reach added when — and
3354
+ * only when — it reached nothing at all (#506).
3355
+ *
3356
+ * ## The refusal this replaces could not be cleared
3357
+ *
3358
+ * A documentation site whose only code-extension file is `docs/assets/
3359
+ * search.js` got a CONFIG-1 FAIL with no action available but deleting the
3360
+ * asset. `docs/` is not in `CONFIG_SCAN_DIRS`, and that list is fixed in the
3361
+ * harness, so no change inside the project could make the check read it.
3362
+ *
3363
+ * ## Why not a `docs/` exclusion, an allowlist, or a lower severity
3364
+ *
3365
+ * A bundled `search.js` and a runnable `docs/examples/client.ts` are the
3366
+ * same bytes to any heuristic available here, so excluding the directory is
3367
+ * the narrowing #506's body warns is most likely to buy green with
3368
+ * blindness. An allowlist in the consumer's config makes the file's name
3369
+ * evidence that somebody considered it. And dropping the severity to
3370
+ * UNMEASURED was the first thing I built and the wrong thing: it leaves a
3371
+ * live `join(process.cwd(), ".claude", "rungate.json")` in `bin/cli.ts`
3372
+ * unreported, which is one of the five layouts #476's security review
3373
+ * specifically fixed.
3374
+ *
3375
+ * Reading the files has none of those costs. The walk has ALREADY located
3376
+ * them and already applied every exclusion it applies — tests, type
3377
+ * declarations, root tool configs, dot-directories, build output, symlinks
3378
+ * leaving the project — so this re-reads that decision rather than making a
3379
+ * new one. A clean `search.js` becomes a PASS that MEASURED something; a
3380
+ * `docs/examples/client.ts` with a real direct read becomes a FAIL that
3381
+ * names the file.
3382
+ *
3383
+ * ## Why it is not "widen `CONFIG_SCAN_DIRS`" (#492/#501's option 3)
3384
+ *
3385
+ * It engages only when `filesScanned === 0` — the state where the check's
3386
+ * output today is an unsatisfiable refusal and therefore worth nothing. No
3387
+ * project that passes today can start failing, no ratchet moves, and
3388
+ * `coverageShortfall`'s domain (`filesScanned > 0`) is untouched, so the
3389
+ * guard #492 measured yesterday keeps the meaning it was measured with.
3390
+ * This closes neither #492 nor #501.
3391
+ *
3392
+ * ## Fail-closed on an unreadable file
3393
+ *
3394
+ * If ANY of them cannot be read, the fallback does not engage at all and
3395
+ * the original refusal stands, naming the source. Reading three of four and
3396
+ * reporting a PASS would be a scan that shrank silently — the second shape
3397
+ * in `.claude/rules/checks-must-be-able-to-fail.md` — and the window is
3398
+ * real: these paths come from a walk that finished before this call.
3399
+ */
3400
+ export function scanConfigAccessWithReach(
3401
+ root: string,
3402
+ evidence: SourceEvidence,
3403
+ ): ConfigAccessScan {
3404
+ const scan = scanConfigAccess(root);
3405
+ // Asked FIRST and separately. A fallback that merged into a non-empty
3406
+ // scan would change what every project is measured on, which is the
3407
+ // behavioural change this one is scoped not to be.
3408
+ if (scan.filesScanned > 0) return scan;
3409
+ if (evidence.unscannedFiles.length === 0) return scan;
3410
+
3411
+ const read: Array<{ path: string; text: string }> = [];
3412
+ for (const rel of evidence.unscannedFiles) {
3413
+ try {
3414
+ read.push({ path: rel, text: readFileSync(join(root, rel), "utf-8") });
3415
+ } catch {
3416
+ // Fail closed — see the docstring. The caller's refusal is the
3417
+ // correct outcome for a project this check could not finish reading.
3418
+ return scan;
3419
+ }
3420
+ }
3421
+
3422
+ const counts = { ...scan.counts };
3423
+ let bytesScanned = scan.bytesScanned;
3424
+ for (const f of read) {
3425
+ bytesScanned += f.text.length;
3426
+ if (f.path === CONFIG_LOADER) continue;
3427
+ const n = classifyConfigSites(f.text).filter(site => site.kind === "use").length;
3428
+ if (n > 0) counts[f.path] = n;
3429
+ }
3430
+ return {
3431
+ counts,
3432
+ filesScanned: scan.filesScanned + read.length,
3433
+ bytesScanned,
3434
+ fallbackFiles: read.map(f => f.path),
3435
+ };
3013
3436
  }
3014
3437
 
3015
3438
  /**
@@ -3085,7 +3508,7 @@ export function scanPromptConfigAccess(root: string): ConfigAccessScan {
3085
3508
  const n = (stripJsComments(text).match(configAccessPattern()) || []).length;
3086
3509
  if (n > 0) counts[`workflows/${f}`] = n;
3087
3510
  }
3088
- return { counts, filesScanned, bytesScanned };
3511
+ return { counts, filesScanned, bytesScanned, fallbackFiles: [] };
3089
3512
  }
3090
3513
 
3091
3514
  /**
@@ -3453,6 +3876,22 @@ export interface SourceEvidence {
3453
3876
  * One number, one walk, no arithmetic between them.
3454
3877
  */
3455
3878
  unscannedSourceCount: number;
3879
+ /**
3880
+ * The PATHS behind that count — every source file the walk counted that
3881
+ * lives outside `CONFIG_SCAN_DIRS` (#506).
3882
+ *
3883
+ * `unscannedSourceCount` is now `unscannedFiles.length` rather than a
3884
+ * second tally incremented beside it. Two expressions answering one
3885
+ * question are one decider wearing two hats, and a mutation of either
3886
+ * leaves the other holding every case up — the fourth shape in
3887
+ * `.claude/rules/checks-must-be-able-to-fail.md`.
3888
+ *
3889
+ * It exists because naming these files was never the end of the job.
3890
+ * CONFIG-1 refused over `docs/assets/search.js` while the file sat right
3891
+ * there, already located by this walk and already filtered by every rule
3892
+ * the walk applies. Reading it is strictly more than refusing to.
3893
+ */
3894
+ unscannedFiles: string[];
3456
3895
  /**
3457
3896
  * Top-level directories holding source that `CONFIG_SCAN_DIRS` does not
3458
3897
  * walk — `"."` for a source file at the root (#492).
@@ -3558,8 +3997,10 @@ const SCRIPT_LIKE = /\.(py|rb|sh|bash|zsh|fish|pl|php|lua|r|go|rs|java|kt|swift|
3558
3997
 
3559
3998
  export function projectSourceEvidence(root: string): SourceEvidence {
3560
3999
  const examples: string[] = [];
3561
- // #492: the COUNT and the WHERE, alongside the capped sample.
3562
- let unscannedSourceCount = 0;
4000
+ // #492: the COUNT and the WHERE, alongside the capped sample. #506 made
4001
+ // the count a projection of the paths — one list, so a file cannot be
4002
+ // counted as unread and then left out of the list of what to read.
4003
+ const unscannedFiles: string[] = [];
3563
4004
  const unscannedDirs = new Set<string>();
3564
4005
  const unreadable: string[] = [];
3565
4006
  const skipped: string[] = [];
@@ -3578,7 +4019,7 @@ export function projectSourceEvidence(root: string): SourceEvidence {
3578
4019
  } catch {
3579
4020
  return {
3580
4021
  found: false, examples: [], unreadable: ["."], skipped: [],
3581
- unscannedSourceCount: 0, unscannedDirs: [],
4022
+ unscannedSourceCount: 0, unscannedFiles: [], unscannedDirs: [],
3582
4023
  };
3583
4024
  }
3584
4025
  const insideProject = (p: string) => p === rootReal || p.startsWith(rootReal + sep);
@@ -3791,7 +4232,7 @@ export function projectSourceEvidence(root: string): SourceEvidence {
3791
4232
  }
3792
4233
  if (!CONFIG_SCAN_DIRS.includes(top) && !measuredElsewhere(top)) {
3793
4234
  unscannedDirs.add(top);
3794
- unscannedSourceCount++;
4235
+ unscannedFiles.push(childRel);
3795
4236
  }
3796
4237
  if (examples.length < 3) examples.push(childRel);
3797
4238
  }
@@ -3831,7 +4272,9 @@ export function projectSourceEvidence(root: string): SourceEvidence {
3831
4272
  examples,
3832
4273
  unreadable,
3833
4274
  skipped,
3834
- unscannedSourceCount,
4275
+ // A projection of the list, not a tally kept beside it (#506).
4276
+ unscannedSourceCount: unscannedFiles.length,
4277
+ unscannedFiles,
3835
4278
  unscannedDirs: [...unscannedDirs].sort(),
3836
4279
  };
3837
4280
  }
@@ -3947,8 +4390,16 @@ export function coverageShortfall(scan: ConfigAccessScan, evidence: SourceEviden
3947
4390
  // where that entry already closed it. Both measured at 0 red before the
3948
4391
  // boundary cases were added (sec-492).
3949
4392
  if (scan.filesScanned === 0) return null; // the empty-scan refusal owns this case
3950
- const unread = evidence.unscannedSourceCount;
3951
- if (unread === 0) return null;
4393
+ // Subtracted, not zeroed, and the difference is a statement: these are
4394
+ // the SAME files. `fallbackFiles` is sourced from `unscannedFiles`, which
4395
+ // is also what this count projects, so the subtraction is inside one walk
4396
+ // and cannot go negative (#506). Naming a file as unread one line after
4397
+ // reading it is the kind of false sentence #500 was filed about.
4398
+ //
4399
+ // It cannot silence the ordinary path: `fallbackFiles` is non-empty only
4400
+ // when the first scan read NOTHING, which the guard above already owns.
4401
+ const unread = evidence.unscannedSourceCount - scan.fallbackFiles.length;
4402
+ if (unread <= 0) return null;
3952
4403
  const where = evidence.unscannedDirs.map(d => (d === "." ? "the project root" : `${d}/`)).join(", ");
3953
4404
  return (
3954
4405
  `UNMEASURED: CONFIG-1 read ${scan.filesScanned} file(s), and ${unread} more source ` +
@@ -4045,8 +4496,10 @@ export function runSuccessDocValidation(root: string) {
4045
4496
  export function runConfigAccessValidation(root: string) {
4046
4497
  describe("Config access (ADR-002 D1)", () => {
4047
4498
  test("CONFIG-1: only the loader reads the rungate config", () => {
4048
- const scan = scanConfigAccess(root);
4049
4499
  const evidence = projectSourceEvidence(root);
4500
+ // #506: when the scan dirs hold nothing, read the source the walk
4501
+ // found anyway rather than refusing over a file nobody can move.
4502
+ const scan = scanConfigAccessWithReach(root, evidence);
4050
4503
  const unmeasured = unmeasuredScanReason(
4051
4504
  scan,
4052
4505
  evidence,
@@ -4101,6 +4554,540 @@ export function runConfigAccessValidation(root: string) {
4101
4554
  });
4102
4555
  }
4103
4556
 
4557
+ // ── AIREVIEW-1: the generated ai-review job (#630) ──────────────────────
4558
+ //
4559
+ // `test/ai-review-scaffold.test.ts` proves the BUILDER produces a script that
4560
+ // annotates UNMEASURED. This proves the FILE ON DISK does — which is a
4561
+ // different claim, and the one a consumer's pipeline depends on. A generator
4562
+ // that is correct and a workflow file that drifted from it look identical
4563
+ // from the generator's side.
4564
+
4565
+ export { AI_REVIEW_JOB, AI_REVIEW_WORKFLOW_FILE } from "./scaffold/ai-review-step";
4566
+
4567
+ /** What `aiReviewAuthScript` could establish about a project's ai-review job. */
4568
+ export interface AiReviewReading {
4569
+ /** The bash the generated auth-detect step runs, or null when none was read. */
4570
+ script: string | null;
4571
+ /**
4572
+ * Non-null when this project is not one the check can reach a verdict on —
4573
+ * it has no scaffolded CI at all. UNMEASURED, said out loud, because `0`,
4574
+ * `{}`, `null` and `SKIP` are indistinguishable from success.
4575
+ */
4576
+ unmeasured: string | null;
4577
+ /**
4578
+ * Non-null when the project IS scaffolded and the job is absent, renamed,
4579
+ * conditional, or unreadable.
4580
+ *
4581
+ * Absence is a FAILURE and not UNMEASURED on purpose. The cheapest repair
4582
+ * for a red AIREVIEW-1 is to delete `.github/workflows/ai-review.yml`, and a
4583
+ * check that goes green under its own cheapest wrong fix certifies the
4584
+ * regression it exists to block (.claude/rules/checks-must-be-able-to-fail.md,
4585
+ * the fifth shape).
4586
+ */
4587
+ failure: string | null;
4588
+ /**
4589
+ * What the same parse found about Workload Identity Federation (#631).
4590
+ *
4591
+ * Read HERE rather than by a second parse, because the question
4592
+ * `aiReviewUnmeasuredViolations` has to answer about a federated consumer
4593
+ * is "does the job that resolved `mode=vertex` also carry the step that
4594
+ * mints the credential" — two readings of one file, and two parses would be
4595
+ * two deciders with nothing comparing them.
4596
+ */
4597
+ federation: AiReviewFederation;
4598
+ }
4599
+
4600
+ /** The federation evidence in a generated ai-review job. */
4601
+ export interface AiReviewFederation {
4602
+ /** Whether the job may mint an OIDC token at all. */
4603
+ idTokenWrite: boolean;
4604
+ /** The `google-github-actions/auth` step, or null when the job has none. */
4605
+ authStep: Record<string, any> | null;
4606
+ }
4607
+
4608
+ /** No evidence read. The direction this has to be wrong in: it refuses. */
4609
+ const NO_FEDERATION: AiReviewFederation = { idTokenWrite: false, authStep: null };
4610
+
4611
+ /**
4612
+ * Read the auth-detect script out of a project's generated ai-review workflow.
4613
+ *
4614
+ * Parses rather than greps: the property is "GitHub will run this script in a
4615
+ * job that cannot report skipped", and only a parse can say that. A `# if:` in
4616
+ * a comment is not a condition and a condition reformatted across lines still
4617
+ * is one, so a text match is wrong in both directions.
4618
+ */
4619
+ export function aiReviewAuthScript(root: string): AiReviewReading {
4620
+ const dir = join(root, ".github", "workflows");
4621
+ const path = join(dir, AI_REVIEW_WORKFLOW_FILE);
4622
+ const none = { script: null, unmeasured: null, failure: null, federation: NO_FEDERATION };
4623
+
4624
+ if (!existsSync(path)) {
4625
+ if (!existsSync(join(dir, "ci.yml"))) {
4626
+ return {
4627
+ ...none,
4628
+ unmeasured: `UNMEASURED: this project has no .github/workflows/ci.yml, so it has ` +
4629
+ `no scaffolded CI and no ${AI_REVIEW_WORKFLOW_FILE} to execute. Run the scaffold ` +
4630
+ `to get one; until then nothing here says an AI review passed.`,
4631
+ };
4632
+ }
4633
+ return {
4634
+ ...none,
4635
+ failure: `.github/workflows/${AI_REVIEW_WORKFLOW_FILE} is absent while ci.yml is present, ` +
4636
+ `so this project took a scaffold that writes it and no longer has it. ` +
4637
+ `A deleted review job is not a passing review — re-run the scaffold.`,
4638
+ };
4639
+ }
4640
+
4641
+ let doc: unknown;
4642
+ try {
4643
+ doc = Bun.YAML.parse(readFileSync(path, "utf-8"));
4644
+ } catch (e) {
4645
+ return { ...none, failure: `${AI_REVIEW_WORKFLOW_FILE} does not parse as YAML: ${(e as Error).message}` };
4646
+ }
4647
+ const jobs = (doc as any)?.jobs;
4648
+ if (!jobs || typeof jobs !== "object") {
4649
+ return { ...none, failure: `${AI_REVIEW_WORKFLOW_FILE} declares no jobs: mapping` };
4650
+ }
4651
+ const job = jobs[AI_REVIEW_JOB];
4652
+ if (!job || typeof job !== "object") {
4653
+ return {
4654
+ ...none,
4655
+ failure: `${AI_REVIEW_WORKFLOW_FILE} has no job named ${AI_REVIEW_JOB} ` +
4656
+ `(found: ${Object.keys(jobs).join(", ") || "none"})`,
4657
+ };
4658
+ }
4659
+ if ("if" in job) {
4660
+ return {
4661
+ ...none,
4662
+ failure: `the ${AI_REVIEW_JOB} job carries a job-level \`if:\` — a job skipped by a ` +
4663
+ `condition reports \`skipped\`, and a skipped required check can be treated as ` +
4664
+ `satisfied. Resolve the condition inside \`run:\` instead.`,
4665
+ };
4666
+ }
4667
+ const steps = Array.isArray(job.steps) ? job.steps : [];
4668
+ const conditional = steps.filter((s: any) => s && typeof s === "object" && "if" in s);
4669
+ if (conditional.length > 0) {
4670
+ return {
4671
+ ...none,
4672
+ failure: `${conditional.length} step(s) in the ${AI_REVIEW_JOB} job carry an \`if:\`, ` +
4673
+ `which reports \`skipped\` rather than a verdict`,
4674
+ };
4675
+ }
4676
+ const auth = steps.find((s: any) => s && s.id === AI_REVIEW_AUTH_STEP_ID);
4677
+ if (!auth || typeof auth.run !== "string") {
4678
+ return {
4679
+ ...none,
4680
+ failure: `${AI_REVIEW_WORKFLOW_FILE}'s ${AI_REVIEW_JOB} job has no \`run:\` step with ` +
4681
+ `id ${AI_REVIEW_AUTH_STEP_ID}, so nothing resolves a credential mode`,
4682
+ };
4683
+ }
4684
+ // The federation evidence, off the SAME parse (#631). `permissions:` is
4685
+ // legal at workflow level and at job level and the job's wins, so both are
4686
+ // read — a check that looked at only one would report a correctly granted
4687
+ // job as ungranted, or the reverse.
4688
+ const permissions = (job as any).permissions ?? (doc as any)?.permissions;
4689
+ const idTokenWrite =
4690
+ permissions === "write-all" ||
4691
+ (!!permissions && typeof permissions === "object" && (permissions as any)["id-token"] === "write");
4692
+ const authStep =
4693
+ steps.find((s: any) => s && typeof s === "object" && s.id === AI_REVIEW_VERTEX_AUTH_STEP_ID) ?? null;
4694
+
4695
+ return {
4696
+ script: auth.run,
4697
+ unmeasured: null,
4698
+ failure: null,
4699
+ federation: { idTokenWrite, authStep },
4700
+ };
4701
+ }
4702
+
4703
+ /** One execution of a generated script, under an environment built from scratch. */
4704
+ function runAiReviewScript(script: string, env: Record<string, string>): { code: number; stdout: string } {
4705
+ const dir = mkdtempSync(join(tmpdir(), "rungate-ai-review-"));
4706
+ try {
4707
+ const sh = join(dir, "auth-detect.sh");
4708
+ writeFileSync(sh, script);
4709
+ const out = join(dir, "github_output");
4710
+ writeFileSync(out, "");
4711
+ // Built from scratch, NOT inherited. A developer or a runner with
4712
+ // ANTHROPIC_API_KEY exported would make the credential-free case resolve a
4713
+ // mode, and the whole check would go green for a reason unrelated to the
4714
+ // code — test/ai-review-scaffold.test.ts makes the same argument.
4715
+ const r = spawnSync("bash", [sh], {
4716
+ encoding: "utf-8",
4717
+ env: { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: dir, GITHUB_OUTPUT: out, ...env },
4718
+ });
4719
+ return { code: r.status ?? -1, stdout: `${r.stdout ?? ""}${r.stderr ?? ""}` };
4720
+ } finally {
4721
+ rmSync(dir, { recursive: true, force: true });
4722
+ }
4723
+ }
4724
+
4725
+ /** Warning annotations naming UNMEASURED, which is the state AC-4 counts. */
4726
+ function unmeasuredWarnings(stdout: string): string[] {
4727
+ return stdout.split("\n").filter(l => l.startsWith("::warning") && l.includes("UNMEASURED"));
4728
+ }
4729
+
4730
+ /**
4731
+ * The claims that apply when the generated script resolves a credential with
4732
+ * no environment at all (#631).
4733
+ *
4734
+ * The credential-free claim is unreachable for this consumer, so this is the
4735
+ * claim that replaces it rather than the absence of one. `mode=vertex` is a
4736
+ * statement that a review WILL run, and a job that makes it while carrying no
4737
+ * step that could mint the credential is a green check for a review that
4738
+ * cannot authenticate — AIREVIEW-1's own subject, reached through the new
4739
+ * mode. **It is also the cheapest wrong fix: accept any resolved mode and
4740
+ * return early.** These are the assertions that repair would skip.
4741
+ */
4742
+ function aiReviewGeneratedCredentialViolations(
4743
+ mode: string,
4744
+ bare: { code: number; stdout: string },
4745
+ federation: AiReviewFederation,
4746
+ ): string[] {
4747
+ const out: string[] = [];
4748
+
4749
+ if (!(AI_REVIEW_MODES as readonly string[]).includes(mode)) {
4750
+ out.push(
4751
+ `the ai-review step resolved \`${mode}\`, which is not one of the ${AI_REVIEW_MODES.length} ` +
4752
+ `modes it declares (${AI_REVIEW_MODES.join(", ")})`,
4753
+ );
4754
+ }
4755
+
4756
+ const stillWarned = unmeasuredWarnings(bare.stdout);
4757
+ if (stillWarned.length !== 0) {
4758
+ out.push(
4759
+ `the ai-review step resolved \`${mode}\` from the generated file and still emitted ` +
4760
+ `${stillWarned.length} UNMEASURED annotation(s) — a configured review that reports ` +
4761
+ `itself unmeasured makes the word mean nothing\n${bare.stdout.trim()}`,
4762
+ );
4763
+ }
4764
+ if (bare.code !== 0) {
4765
+ out.push(`the ai-review step exited ${bare.code} with \`${mode}\` resolved from the generated file`);
4766
+ }
4767
+
4768
+ if (mode !== AI_REVIEW_MODE_VERTEX) {
4769
+ // `oauth` and `api-key` come from secrets, and this run had none. A
4770
+ // script that resolved one anyway is reading a credential from somewhere
4771
+ // nobody declared.
4772
+ out.push(
4773
+ `the ai-review step resolved \`${mode}\` with an EMPTY environment. That mode is ` +
4774
+ `supposed to come from a repository secret, so a run with no secrets set must not ` +
4775
+ `reach it — something in the generated file is standing in for a credential`,
4776
+ );
4777
+ return out;
4778
+ }
4779
+
4780
+ const step = federation.authStep;
4781
+ if (step === null) {
4782
+ out.push(
4783
+ `the ai-review step resolved \`${AI_REVIEW_MODE_VERTEX}\` and the ${AI_REVIEW_JOB} job has no ` +
4784
+ `\`${AI_REVIEW_VERTEX_AUTH_STEP_ID}\` step, so nothing in it mints the credential the mode ` +
4785
+ `claims. A review that cannot authenticate is UNMEASURED, and this one reports a mode`,
4786
+ );
4787
+ return out;
4788
+ }
4789
+ if (!federation.idTokenWrite) {
4790
+ out.push(
4791
+ `the ${AI_REVIEW_JOB} job federates to Vertex without \`id-token: write\`, so the runner ` +
4792
+ `cannot mint the OIDC token the exchange consumes and the job resolves a mode it will ` +
4793
+ `never hold`,
4794
+ );
4795
+ }
4796
+ const inputs = (step.with ?? {}) as Record<string, unknown>;
4797
+ if ("credentials_json" in inputs) {
4798
+ out.push(
4799
+ `the ${AI_REVIEW_VERTEX_AUTH_STEP_ID} step passes \`credentials_json\`, which writes a ` +
4800
+ `permanent service-account key into the workspace the review agent then reads (#677 ` +
4801
+ `AC-15). Federation exists so that key does not have to be anywhere`,
4802
+ );
4803
+ }
4804
+ for (const required of ["workload_identity_provider", "project_id"]) {
4805
+ if (!(required in inputs)) {
4806
+ out.push(
4807
+ `the ${AI_REVIEW_VERTEX_AUTH_STEP_ID} step declares no \`${required}\`, so the exchange ` +
4808
+ `has nothing to present and the resolved mode is a claim about a credential that ` +
4809
+ `cannot be obtained`,
4810
+ );
4811
+ }
4812
+ }
4813
+
4814
+ return out;
4815
+ }
4816
+
4817
+ /**
4818
+ * Execute a generated auth-detect script and report what it got wrong.
4819
+ *
4820
+ * Two runs, and the second one is what makes the first mean anything: a script
4821
+ * that annotated UNMEASURED unconditionally would satisfy the credential-free
4822
+ * case while telling a consumer nothing, so the word has to be absent when a
4823
+ * credential IS configured.
4824
+ */
4825
+ export function aiReviewUnmeasuredViolations(
4826
+ script: string,
4827
+ federation: AiReviewFederation = NO_FEDERATION,
4828
+ ): string[] {
4829
+ const out: string[] = [];
4830
+
4831
+ const bare = runAiReviewScript(script, {});
4832
+
4833
+ // Which claim applies is a READING, not an assumption (#631). Every fixture
4834
+ // this check ever had took its credential from a secret, so emptying the
4835
+ // environment really did produce the credential-free state. A consumer who
4836
+ // declares `ci.vertex` gets `mode=vertex` written into the file, no
4837
+ // environment can move it, and demanding an UNMEASURED annotation of that
4838
+ // tree is demanding a state it cannot enter.
4839
+ const resolved = /^ai-review auth mode: (.*)$/m.exec(bare.stdout);
4840
+ if (!resolved) {
4841
+ out.push(
4842
+ `the ai-review step printed no \`ai-review auth mode:\` line, so which credential it ` +
4843
+ `resolved cannot be read at all. A mode nobody can read is UNMEASURED, and the ` +
4844
+ `credential-free claims below are being applied to it unverified`,
4845
+ );
4846
+ }
4847
+ const mode = resolved ? resolved[1].trim() : AI_REVIEW_MODE_NONE;
4848
+
4849
+ if (mode !== AI_REVIEW_MODE_NONE) {
4850
+ return [...out, ...aiReviewGeneratedCredentialViolations(mode, bare, federation)];
4851
+ }
4852
+
4853
+ const warnings = unmeasuredWarnings(bare.stdout);
4854
+ if (warnings.length !== 1) {
4855
+ out.push(
4856
+ `with no credential configured the ai-review step emitted ${warnings.length} UNMEASURED ` +
4857
+ `warning annotation(s), expected exactly 1 — a tree with no credential has not PASSED ` +
4858
+ `an AI review, it has not HAD one\n${bare.stdout.trim()}`,
4859
+ );
4860
+ }
4861
+ for (const { secret } of AI_REVIEW_AUTH_PRECEDENCE) {
4862
+ if (warnings.length === 1 && !warnings[0].includes(secret)) {
4863
+ out.push(`the UNMEASURED annotation does not name ${secret}, so nobody can act on it`);
4864
+ }
4865
+ }
4866
+ if (bare.code !== 0) {
4867
+ out.push(
4868
+ `the ai-review step exited ${bare.code} with no credential and no ` +
4869
+ `${AI_REVIEW_REQUIRE_VAR} set — the honest default for a tree that has never ` +
4870
+ `configured one is an annotation, not a red pipeline on the first scaffold`,
4871
+ );
4872
+ }
4873
+
4874
+ const credentialed = runAiReviewScript(script, { [AI_REVIEW_AUTH_PRECEDENCE[0].secret]: "x" });
4875
+ const stillWarned = credentialed.stdout.split("\n").filter(l => l.includes("UNMEASURED"));
4876
+ if (stillWarned.length !== 0) {
4877
+ out.push(
4878
+ `with ${AI_REVIEW_AUTH_PRECEDENCE[0].secret} configured the step still emitted ` +
4879
+ `${stillWarned.length} UNMEASURED line(s) — an unconditional annotation makes the ` +
4880
+ `word mean nothing\n${credentialed.stdout.trim()}`,
4881
+ );
4882
+ }
4883
+ if (credentialed.code !== 0) {
4884
+ out.push(`the ai-review step exited ${credentialed.code} with a credential configured`);
4885
+ }
4886
+
4887
+ return out;
4888
+ }
4889
+
4890
+ /**
4891
+ * AIREVIEW-1 (#630) — a scaffolded project reports UNMEASURED rather than a
4892
+ * passing review when no credential is configured.
4893
+ *
4894
+ * FAILS; does not warn, for every state except "this project has no CI at
4895
+ * all". See `.claude/rules/config-access.md` — an absent credential is
4896
+ * UNMEASURED, and UNMEASURED is not a pass.
4897
+ */
4898
+ export function runAiReviewValidation(root: string) {
4899
+ describe("AI review workflow (#630)", () => {
4900
+ test("AIREVIEW-1: no credential is UNMEASURED, and a credential is not", () => {
4901
+ const reading = aiReviewAuthScript(root);
4902
+ if (reading.unmeasured) {
4903
+ addFinding({
4904
+ ruleId: "AIREVIEW-1",
4905
+ severity: "WARN",
4906
+ file: `.github/workflows/${AI_REVIEW_WORKFLOW_FILE}`,
4907
+ message: reading.unmeasured,
4908
+ });
4909
+ return;
4910
+ }
4911
+ const violations = reading.failure
4912
+ ? [reading.failure]
4913
+ : aiReviewUnmeasuredViolations(reading.script!, reading.federation);
4914
+ for (const v of violations) {
4915
+ addFinding({
4916
+ ruleId: "AIREVIEW-1",
4917
+ severity: "FAIL",
4918
+ file: `.github/workflows/${AI_REVIEW_WORKFLOW_FILE}`,
4919
+ message: v,
4920
+ fixCommand: "bunx rungate scaffold .",
4921
+ });
4922
+ }
4923
+ expect(violations, violations.join("\n")).toEqual([]);
4924
+ });
4925
+ });
4926
+ }
4927
+
4928
+ /**
4929
+ * PROTECT-1 (#909) — the repo's protection does not serialise every PR.
4930
+ *
4931
+ * `required_status_checks.strict = true` makes merging one PR invalidate all
4932
+ * the others. On a swarm that is the whole throughput limit, and it is
4933
+ * invisible: every PR is individually green and `BEHIND`. See
4934
+ * `lib/branch-protection.ts` for the measurement and for why the remedy
4935
+ * differs by `owner.type`.
4936
+ *
4937
+ * FAILS; does not warn, when the serialisation is established — the reason
4938
+ * this is in RunGate rather than a doc is that three consumers hit it
4939
+ * independently and nobody wrote it down, and a rule that is documented but
4940
+ * not enforced is the failure mode
4941
+ * `.claude/rules/checks-must-be-able-to-fail.md` is about.
4942
+ *
4943
+ * WARNS, and says UNMEASURED out loud, for everything it could not read. The
4944
+ * protection endpoint needs admin and the default Actions token has none, so
4945
+ * UNMEASURED is the ordinary outcome in a consumer's CI; it must still never
4946
+ * be mistaken for clean.
4947
+ *
4948
+ * `probe` and `subject` are seams and not conveniences. The probe is the same
4949
+ * binding production calls through, so a test replacing it exercises this
4950
+ * function rather than a parallel path, and nothing here reaches the network
4951
+ * when it is replaced.
4952
+ */
4953
+ export function runBranchProtectionValidation(
4954
+ root: string,
4955
+ opts: { probe?: ProtectionProbe; subject?: (root: string) => string | null } = {},
4956
+ ) {
4957
+ const probe = opts.probe ?? githubProtectionProbe;
4958
+ const subject = opts.subject ?? protectionSubject;
4959
+
4960
+ describe("Branch protection (#909)", () => {
4961
+ test("PROTECT-1: merging one PR does not invalidate every other open PR", async () => {
4962
+ const slug = subject(root);
4963
+ if (!slug) {
4964
+ addFinding({
4965
+ ruleId: "PROTECT-1",
4966
+ severity: "WARN",
4967
+ file: "",
4968
+ message:
4969
+ "UNMEASURED: this tree's origin remote names no owner/name, so there is no repo to " +
4970
+ "read protection for. Nothing was checked, which is not the same as nothing being wrong.",
4971
+ });
4972
+ return;
4973
+ }
4974
+
4975
+ const facts = await probe(slug);
4976
+ // Read against the branch the FACTS are about. Defaulting to "main"
4977
+ // here would answer a question about a branch the repo may not protect.
4978
+ const pushCi = facts.branch
4979
+ ? readPushCi(root, facts.contexts, facts.branch)
4980
+ : { onPush: [], missing: [], unmeasured: "UNMEASURED push-CI: the default branch is unknown, so no trigger could be matched against it." };
4981
+ const reading = readMergeSerialisation(facts, pushCi);
4982
+
4983
+ if (reading.verdict === "UNMEASURED") {
4984
+ addFinding({ ruleId: "PROTECT-1", severity: "WARN", file: "", message: reading.reason });
4985
+ return;
4986
+ }
4987
+ if (reading.verdict === "PASS") return;
4988
+
4989
+ const message = `${reading.reason}\n\nREMEDY: ${reading.remedy}`;
4990
+ addFinding({
4991
+ ruleId: "PROTECT-1",
4992
+ severity: "FAIL",
4993
+ file: "",
4994
+ message,
4995
+ // Not a command that fixes it. This check reads and refuses; a
4996
+ // fixCommand that mutated protection would be the scaffold silently
4997
+ // rewriting a repo's security settings, which #909 puts out of scope.
4998
+ fixCommand: `gh api repos/${facts.slug}/branches/${facts.branch ?? "main"}/protection -q .required_status_checks`,
4999
+ });
5000
+ // Widened deliberately: control flow has already narrowed `verdict` to
5001
+ // "FAIL" here, so the comparison typechecks as a contradiction and the
5002
+ // assertion reads as dead. It is not dead — it is what turns the
5003
+ // recorded finding into a red suite — and the cast keeps the assertion
5004
+ // written against the value rather than against the narrowing.
5005
+ expect(reading.verdict as string, message).toBe("PASS");
5006
+ });
5007
+ });
5008
+ }
5009
+
5010
+ /**
5011
+ * WORKFLOW-DRIFT-1 (#915) — a generated workflow still holds the bytes the
5012
+ * generator wrote.
5013
+ *
5014
+ * The consumer-facing end of #722's census, which has been able to tell a
5015
+ * hand-edited generated workflow from an untouched one since the day it
5016
+ * shipped and which nothing a consumer runs has ever asked. `lib/conformity.ts`
5017
+ * exported fifteen `run*` functions and none of them was a workflow drift
5018
+ * check, so a hand-edit to a stamped `ci.yml`, `gates.yml` or `ai-review.yml`
5019
+ * was detectable and undetected.
5020
+ *
5021
+ * FAILS; does not warn, for a file whose every byte is the generator's —
5022
+ * `bunx rungate scaffold .` reverts that edit and nothing says so first.
5023
+ * WARNs, naming the reason, for the two states that are not a pass and not a
5024
+ * verdict either: a scan with nothing stamped to compare, and a file in which
5025
+ * `ci.workflowOverrides` has made part of the content the consumer's. See
5026
+ * `lib/scaffold/workflow-drift.ts` for why the second of those cannot be
5027
+ * resolved from a whole-body digest.
5028
+ *
5029
+ * NOT-OWNED workflows pass and are named. A consumer's own CI is theirs; a
5030
+ * check that refused it would be deleted within a day.
5031
+ */
5032
+ export function runWorkflowDriftValidation(root: string) {
5033
+ describe("Generated workflow drift (#915)", () => {
5034
+ test("WORKFLOW-DRIFT-1: a generated workflow holds the bytes the generator wrote", () => {
5035
+ const overrides = workflowOverrides(tryLoadRungateConfig(root)?.ci);
5036
+ const reading = workflowDrift(root, overrides);
5037
+
5038
+ // Every cause, named. "UNMEASURED" with no reason beside it is the
5039
+ // silence this check exists to replace.
5040
+ //
5041
+ // Reported, and then execution CONTINUES (#958). UNMEASURED and DRIFTED
5042
+ // are not alternatives: `workflowDrift` pushes `UNREADABLE_WORKFLOW` and
5043
+ // moves on per file, so one unreadable workflow raises the whole reading
5044
+ // while a DIFFERENT file in the same directory lands in `drifted`. This
5045
+ // used to `return` here, which threw that drift away and rendered "I
5046
+ // could not fully look" as "I looked and it was fine" — #921's shape,
5047
+ // and a green suite over drift the check had already found.
5048
+ //
5049
+ // Drift found is a failure whatever happened to the rest of the scan.
5050
+ // The partiality is reported ALONGSIDE it, naming what could not be
5051
+ // read, rather than instead of it. The two readings are independent and
5052
+ // are no longer sequenced.
5053
+ for (const reason of reading.unmeasuredReasons) {
5054
+ addFinding({ ruleId: "WORKFLOW-DRIFT-1", severity: "WARN", file: "", message: reason });
5055
+ }
5056
+
5057
+ for (const f of reading.consumerOwned) {
5058
+ addFinding({
5059
+ ruleId: "WORKFLOW-DRIFT-1",
5060
+ severity: "WARN",
5061
+ file: `.github/workflows/${f.name}`,
5062
+ message: f.detail,
5063
+ });
5064
+ }
5065
+ for (const name of reading.notOwned) {
5066
+ addFinding({
5067
+ ruleId: "WORKFLOW-DRIFT-1",
5068
+ severity: "WARN",
5069
+ file: `.github/workflows/${name}`,
5070
+ message:
5071
+ "NOT-OWNED — rungate did not write this workflow and does not manage it. " +
5072
+ "Reported, not touched: what happens to it is yours",
5073
+ });
5074
+ }
5075
+
5076
+ const violations = reading.drifted.map(f => `${f.name}: ${f.detail}`);
5077
+ for (const f of reading.drifted) {
5078
+ addFinding({
5079
+ ruleId: "WORKFLOW-DRIFT-1",
5080
+ severity: "FAIL",
5081
+ file: `.github/workflows/${f.name}`,
5082
+ message: f.detail,
5083
+ fixCommand: "bunx rungate scaffold .",
5084
+ });
5085
+ }
5086
+ expect(violations, violations.join("\n")).toEqual([]);
5087
+ });
5088
+ });
5089
+ }
5090
+
4104
5091
  /**
4105
5092
  * CHECK-2 — config access inside workflow prompt text (ADR-002 D4).
4106
5093
  *
@@ -4118,7 +5105,7 @@ export function runPromptConfigAccessValidation(root: string) {
4118
5105
  // at the point it was read (#476).
4119
5106
  const unmeasured = unmeasuredScanReason(
4120
5107
  scan,
4121
- { found: existsSync(join(root, "workflows")), examples: [], unreadable: [], skipped: [], unscannedSourceCount: 0, unscannedDirs: [] },
5108
+ { found: existsSync(join(root, "workflows")), examples: [], unreadable: [], skipped: [], unscannedSourceCount: 0, unscannedFiles: [], unscannedDirs: [] },
4122
5109
  "CONFIG-2",
4123
5110
  "workflows/ directory",
4124
5111
  );