rungate 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/affected-tests.ts +398 -0
- package/lib/agent-audit.ts +2 -3
- package/lib/agent-worktree.ts +287 -0
- package/lib/agents-md-sections.ts +209 -0
- package/lib/behavioral-cache.ts +150 -2
- package/lib/branch-diff.ts +139 -0
- package/lib/branch-protection.ts +790 -0
- package/lib/brief-ac-alignment.ts +104 -0
- package/lib/brief-references.ts +205 -0
- package/lib/bun-version.ts +347 -0
- package/lib/check-remedy.ts +140 -0
- package/lib/ci-checkout.ts +18 -1
- package/lib/ci-history-depth.ts +159 -0
- package/lib/ci-setup-step.ts +96 -0
- package/lib/claude-code-action.ts +85 -0
- package/lib/claude-tool-vocabulary.ts +188 -0
- package/lib/cli-commands.ts +15 -0
- package/lib/commit-sha.ts +42 -0
- package/lib/commit-subject.ts +90 -0
- package/lib/compliance-report.ts +377 -45
- package/lib/compliance.ts +196 -20
- package/lib/config-loader.ts +51 -1
- package/lib/config-trackable.ts +391 -0
- package/lib/conformity.ts +1272 -189
- package/lib/containment.ts +238 -0
- package/lib/content-hash.ts +24 -0
- package/lib/context-docs.ts +143 -28
- package/lib/create-brief.ts +2 -1
- package/lib/create-sc.ts +323 -4
- package/lib/dev-liveness.ts +180 -0
- package/lib/discovery-branch-points.ts +432 -0
- package/lib/discovery-stability.ts +481 -0
- package/lib/distribution-currency.ts +186 -0
- package/lib/doc-routing.ts +157 -0
- package/lib/doc-scan-targets.ts +9 -0
- package/lib/evidence-command-transport.ts +160 -0
- package/lib/evidence-prevalidator.ts +246 -2
- package/lib/evidence-satisfiability.ts +534 -12
- package/lib/failure-intake.ts +317 -0
- package/lib/fallow-availability.ts +286 -0
- package/lib/fs-write-capabilities.ts +232 -0
- package/lib/gate-refusal-currency.ts +112 -0
- package/lib/gate-run-split.ts +147 -0
- package/lib/gate-timeouts.ts +82 -0
- package/lib/generators/agent-briefs.ts +4 -1
- package/lib/generators/agents-md.ts +204 -48
- package/lib/generators/code-map.ts +2 -1
- package/lib/generators/types.ts +8 -0
- package/lib/git-dates.ts +55 -0
- package/lib/git-isolation.ts +140 -0
- package/lib/github.ts +15 -0
- package/lib/google-auth-action.ts +93 -0
- package/lib/harness-project.ts +48 -0
- package/lib/harness-provenance.ts +255 -0
- package/lib/hook-evidence.ts +235 -0
- package/lib/hook-scopes.ts +386 -0
- package/lib/hook-target-tree.ts +398 -0
- package/lib/hook-trees.ts +447 -0
- package/lib/js-top-level-code.ts +78 -10
- package/lib/junit-checks.ts +536 -0
- package/lib/launch-base.ts +253 -0
- package/lib/line-citation.ts +372 -0
- package/lib/local-calendar-day.ts +78 -0
- package/lib/orphan-reaper.ts +468 -0
- package/lib/parity-binding.ts +205 -0
- package/lib/per-test-timeouts.ts +219 -0
- package/lib/plan-coverage.ts +151 -0
- package/lib/prior-branch.ts +201 -8
- package/lib/process-table.ts +146 -0
- package/lib/project-state-derivation.ts +219 -0
- package/lib/project-state-root.ts +258 -0
- package/lib/project-state-shape.ts +59 -0
- package/lib/promote-outputs.ts +6 -2
- package/lib/published-artifact.ts +525 -0
- package/lib/readdir-sorted.ts +45 -0
- package/lib/report-provenance.ts +156 -0
- package/lib/review-execution-reading.ts +206 -0
- package/lib/review-tool-agreement.ts +433 -0
- package/lib/rule-promotion.ts +234 -0
- package/lib/rule-retirement.ts +525 -0
- package/lib/run-branch.ts +612 -0
- package/lib/rungate-schema.ts +19 -0
- package/lib/runner-home.ts +274 -0
- package/lib/sc-claim-forms.ts +75 -0
- package/lib/sc-id-cross-tree.ts +323 -0
- package/lib/sc-ids.ts +121 -13
- package/lib/sc-reservations.ts +289 -0
- package/lib/scaffold/ai-review-findings.ts +403 -0
- package/lib/scaffold/ai-review-step.ts +4777 -0
- package/lib/scaffold/managed-workflow.ts +587 -192
- package/lib/scaffold/owned-files.ts +186 -39
- package/lib/scaffold/pre-push-hook.ts +49 -5
- package/lib/scaffold/rehearsal.ts +205 -71
- package/lib/scaffold/steps.ts +1517 -293
- package/lib/scaffold/workflow-census.ts +217 -0
- package/lib/scaffold/workflow-drift.ts +217 -0
- package/lib/scaffold/write-steps.ts +10 -4
- package/lib/scan-budget.ts +247 -0
- package/lib/scanner.ts +80 -62
- package/lib/scratch-roots.ts +50 -5
- package/lib/screenshot-discovery.ts +121 -0
- package/lib/screenshot-upload.ts +41 -1
- package/lib/secret-patterns.ts +511 -8
- package/lib/ship-mode.ts +219 -0
- package/lib/source-slice.ts +294 -0
- package/lib/spec-listing.ts +148 -0
- package/lib/spec-requirements.ts +209 -0
- package/lib/stale-cleanup.ts +37 -3
- package/lib/suite-flake.ts +278 -0
- package/lib/suite-measurement.ts +23 -4
- package/lib/suite-run-reading.ts +411 -1
- package/lib/suite-timeout-margin.ts +193 -0
- package/lib/test-case-bodies.ts +883 -0
- package/lib/test-case-deletion.ts +653 -0
- package/lib/test-census.ts +97 -29
- package/lib/test-command-positions.ts +274 -0
- package/lib/test-command.ts +133 -0
- package/lib/test-suite-lock.ts +325 -26
- package/lib/tool-probe.ts +142 -0
- package/lib/top-level-cd.ts +393 -0
- package/lib/tracked-vs-ignored.ts +234 -0
- package/lib/tree-entries.ts +89 -0
- package/lib/ts-ast.ts +200 -0
- package/lib/validators/spec-validators.ts +3 -2
- package/lib/walk-test-files.ts +97 -0
- package/lib/workflow-actions.ts +263 -0
- package/lib/worktree-collect.ts +177 -5
- package/lib/worktree-repo.ts +36 -1
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +45 -0
- package/scripts/audit-specs.ts +247 -12
- package/scripts/audit-transcript.ts +2 -1
- package/scripts/convert-spec.ts +2 -1
- package/scripts/create-sc.ts +37 -7
- package/scripts/create-spec.ts +2 -1
- package/scripts/create-success.ts +2 -1
- package/scripts/extract-constraints.ts +197 -26
- package/scripts/generate-code-map.ts +62 -28
- package/scripts/scaffold-project.ts +12 -2
- package/scripts/split-spec.ts +2 -1
- package/scripts/sync-sc-status.ts +127 -14
- package/scripts/sync-spec-tests.ts +2 -1
package/lib/conformity.ts
CHANGED
|
@@ -5,21 +5,73 @@
|
|
|
5
5
|
* import { runScaffoldConformity, runSpecDiscovery, runSpecDrift } from "rungate/lib/conformity";
|
|
6
6
|
* runScaffoldConformity(import.meta.dir + "/..");
|
|
7
7
|
*/
|
|
8
|
-
import { describe, test, expect } from "bun:test";
|
|
9
|
-
import { existsSync, readFileSync, readdirSync, realpathSync, statSync, writeFileSync } from "fs";
|
|
8
|
+
import { beforeAll, describe, test, expect } from "bun:test";
|
|
9
|
+
import { existsSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, statSync, writeFileSync } from "fs";
|
|
10
|
+
import { tmpdir } from "os";
|
|
10
11
|
import type { Dirent } from "fs";
|
|
11
12
|
import { collectSpecFiles } from "./spec-files";
|
|
13
|
+
import {
|
|
14
|
+
ambiguousSpecNames,
|
|
15
|
+
readSpecRequirements,
|
|
16
|
+
requirementViolations,
|
|
17
|
+
specIsReferenced,
|
|
18
|
+
testableViolations,
|
|
19
|
+
} from "./spec-requirements";
|
|
20
|
+
import { lastCommitDates } from "./git-dates";
|
|
12
21
|
import { spawnSync } from "child_process";
|
|
13
22
|
import { join, dirname, sep } from "path";
|
|
14
23
|
import { fileURLToPath } from "url";
|
|
15
24
|
import { deriveDirectoryName } from "../scripts/split-spec";
|
|
16
|
-
import {
|
|
25
|
+
import {
|
|
26
|
+
type BehavioralCacheState,
|
|
27
|
+
type BehavioralCensus,
|
|
28
|
+
type BehavioralReadingKind,
|
|
29
|
+
behavioralCachePath,
|
|
30
|
+
behavioralCensus,
|
|
31
|
+
behavioralCensusLine,
|
|
32
|
+
behavioralReading,
|
|
33
|
+
readCache,
|
|
34
|
+
} from "./behavioral-cache";
|
|
17
35
|
import { tryLoadRungateConfig } from "./config-loader";
|
|
36
|
+
import {
|
|
37
|
+
type ProtectionProbe,
|
|
38
|
+
githubProtectionProbe,
|
|
39
|
+
protectionSubject,
|
|
40
|
+
readMergeSerialisation,
|
|
41
|
+
readPushCi,
|
|
42
|
+
} from "./branch-protection";
|
|
18
43
|
import { stripJsComments } from "./strip-js-comments";
|
|
44
|
+
import { remedyMessage } from "./check-remedy";
|
|
19
45
|
import { parseScDeclarations } from "./sc-ids";
|
|
20
46
|
import { isHarnessItself } from "./harness-identity";
|
|
47
|
+
import {
|
|
48
|
+
FALLOW_BUDGET_MS,
|
|
49
|
+
fallowAvailable,
|
|
50
|
+
runFallowJson,
|
|
51
|
+
warmFallow,
|
|
52
|
+
type FallowReading,
|
|
53
|
+
type FallowUnavailable,
|
|
54
|
+
} from "./fallow-availability";
|
|
55
|
+
import {
|
|
56
|
+
briefReferenceReading,
|
|
57
|
+
unmeasuredBriefReferenceNotice,
|
|
58
|
+
unresolvableBriefReferenceMessage,
|
|
59
|
+
} from "./brief-references";
|
|
21
60
|
import { tsconfigVerdict, TS_SOURCE_DIRS } from "./typescript-project";
|
|
22
61
|
import { readSuccessDoc, readSuccessTemplate, successViolations } from "./success-doc";
|
|
62
|
+
import {
|
|
63
|
+
AI_REVIEW_AUTH_PRECEDENCE,
|
|
64
|
+
AI_REVIEW_AUTH_STEP_ID,
|
|
65
|
+
AI_REVIEW_JOB,
|
|
66
|
+
AI_REVIEW_MODES,
|
|
67
|
+
AI_REVIEW_MODE_NONE,
|
|
68
|
+
AI_REVIEW_MODE_VERTEX,
|
|
69
|
+
AI_REVIEW_REQUIRE_VAR,
|
|
70
|
+
AI_REVIEW_VERTEX_AUTH_STEP_ID,
|
|
71
|
+
AI_REVIEW_WORKFLOW_FILE,
|
|
72
|
+
} from "./scaffold/ai-review-step";
|
|
73
|
+
import { workflowOverrides } from "./scaffold/managed-workflow";
|
|
74
|
+
import { workflowDrift } from "./scaffold/workflow-drift";
|
|
23
75
|
|
|
24
76
|
// ── Shared utilities ────────────────────────────────────────
|
|
25
77
|
|
|
@@ -170,6 +222,11 @@ export function clearFindings(): void {
|
|
|
170
222
|
_candidates.length = 0;
|
|
171
223
|
_staleness.length = 0;
|
|
172
224
|
_behavioralCount = 0;
|
|
225
|
+
for (const k of Object.keys(_behavioralReadings) as BehavioralReadingKind[]) {
|
|
226
|
+
_behavioralReadings[k] = 0;
|
|
227
|
+
}
|
|
228
|
+
_behavioralCacheStates = [];
|
|
229
|
+
_behavioralUnmeasured.length = 0;
|
|
173
230
|
}
|
|
174
231
|
|
|
175
232
|
export function addBehavioralCount(count: number): void {
|
|
@@ -180,6 +237,56 @@ export function getBehavioralCount(): number {
|
|
|
180
237
|
return _behavioralCount;
|
|
181
238
|
}
|
|
182
239
|
|
|
240
|
+
/**
|
|
241
|
+
* Every ticked `(behavioral)` criterion the run saw, and what the cache said
|
|
242
|
+
* about it (#758).
|
|
243
|
+
*
|
|
244
|
+
* `_behavioralCount` counts the ones that were ROUTED — the ones with no
|
|
245
|
+
* fresh reading — and a criterion that was measured and passed never reaches
|
|
246
|
+
* it. So `behavioralCount: 0` is produced by a run in which everything was
|
|
247
|
+
* measured AND by a run in which there are no behavioral criteria at all,
|
|
248
|
+
* and the number that matters — how many went unread — was never written
|
|
249
|
+
* down anywhere.
|
|
250
|
+
*/
|
|
251
|
+
const _behavioralReadings: Record<BehavioralReadingKind, number> = {
|
|
252
|
+
"fresh-pass": 0, "fresh-fail": 0, stale: 0, absent: 0,
|
|
253
|
+
};
|
|
254
|
+
let _behavioralCacheStates: BehavioralCacheState[] = [];
|
|
255
|
+
const _behavioralUnmeasured: string[] = [];
|
|
256
|
+
|
|
257
|
+
export function addBehavioralCensus(c: BehavioralCensus): void {
|
|
258
|
+
for (const k of Object.keys(_behavioralReadings) as BehavioralReadingKind[]) {
|
|
259
|
+
_behavioralReadings[k] += c.counts[k];
|
|
260
|
+
}
|
|
261
|
+
if (!_behavioralCacheStates.includes(c.cacheState)) _behavioralCacheStates.push(c.cacheState);
|
|
262
|
+
_behavioralUnmeasured.push(...c.unmeasured);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/**
|
|
266
|
+
* The run's whole behavioral reading, as a figure.
|
|
267
|
+
*
|
|
268
|
+
* `reading` is derived here and nowhere else, from the same lists the counts
|
|
269
|
+
* are projections of — a second boolean beside it would always agree with it.
|
|
270
|
+
* An empty population is UNMEASURED, because zero unread criteria out of zero
|
|
271
|
+
* criteria is the number a scan that collected nothing also produces.
|
|
272
|
+
*/
|
|
273
|
+
export function getBehavioralReading(): {
|
|
274
|
+
reading: "MEASURED" | "UNMEASURED";
|
|
275
|
+
counts: Record<BehavioralReadingKind, number>;
|
|
276
|
+
unmeasured: string[];
|
|
277
|
+
cacheStates: BehavioralCacheState[];
|
|
278
|
+
total: number;
|
|
279
|
+
} {
|
|
280
|
+
const total = Object.values(_behavioralReadings).reduce((a, b) => a + b, 0);
|
|
281
|
+
return {
|
|
282
|
+
reading: total > 0 && _behavioralUnmeasured.length === 0 ? "MEASURED" : "UNMEASURED",
|
|
283
|
+
counts: { ..._behavioralReadings },
|
|
284
|
+
unmeasured: [..._behavioralUnmeasured],
|
|
285
|
+
cacheStates: [..._behavioralCacheStates],
|
|
286
|
+
total,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
|
|
183
290
|
// ── Auto-flip: track passing SCs and flip checkboxes ────────
|
|
184
291
|
|
|
185
292
|
interface PassingSC {
|
|
@@ -248,6 +355,7 @@ export function writeFindingsReport(root: string): string {
|
|
|
248
355
|
warnings: _findings.filter(f => f.severity === "WARN").length,
|
|
249
356
|
candidateCount: _candidates.length,
|
|
250
357
|
behavioralCount: _behavioralCount,
|
|
358
|
+
behavioral: getBehavioralReading(),
|
|
251
359
|
staleCount: _staleness.length,
|
|
252
360
|
findings: _findings,
|
|
253
361
|
constraintCandidates: _candidates,
|
|
@@ -312,8 +420,12 @@ export function getBehavioralRouting(sc: ParsedSC): string | null {
|
|
|
312
420
|
* Stale entries (>= 7 days old) return null so they're treated as unmatchable.
|
|
313
421
|
*/
|
|
314
422
|
export function behavioralPattern(sc: ParsedSC, cachePath: string): ((root: string) => void) | null {
|
|
315
|
-
const
|
|
316
|
-
|
|
423
|
+
const entry = readCache(cachePath)[sc.id];
|
|
424
|
+
// One decider for "is there a fresh reading", shared with the census that
|
|
425
|
+
// reports how many there are (#758). Asking `readFreshCache` here and
|
|
426
|
+
// `behavioralReading` there would be two guards over one question, always
|
|
427
|
+
// agreeing, and the count would be unfalsifiable against the routing.
|
|
428
|
+
if (!behavioralReading(entry).startsWith("fresh")) return null;
|
|
317
429
|
if (!entry) return null;
|
|
318
430
|
|
|
319
431
|
return (_root: string) => {
|
|
@@ -1223,7 +1335,28 @@ export type SCPlanEntry =
|
|
|
1223
1335
|
*/
|
|
1224
1336
|
export function planSCTests(scs: ParsedSC[], projectRoot?: string): SCPlanEntry[] {
|
|
1225
1337
|
return scs.map((sc): SCPlanEntry => {
|
|
1226
|
-
if (isBehavioralSC(sc))
|
|
1338
|
+
if (isBehavioralSC(sc)) {
|
|
1339
|
+
// Ask the cache BEFORE classifying. This line used to classify first,
|
|
1340
|
+
// which made `behavioralPattern` — the function that turns a recorded
|
|
1341
|
+
// runtime result into an assertion that throws — unreachable from any
|
|
1342
|
+
// real run: defined, exported, unit-tested, and called by nothing but
|
|
1343
|
+
// its own tests. Meanwhile `scripts/sync-sc-status.ts` DID read the
|
|
1344
|
+
// cache and would move a `passed: false` criterion into `failing`, so
|
|
1345
|
+
// the decider that TICKS a checkbox looked at the evidence and the
|
|
1346
|
+
// suite that decides the build's colour did not (#328).
|
|
1347
|
+
//
|
|
1348
|
+
// `behavioralPattern` returns null for an absent entry and for a stale
|
|
1349
|
+
// one — `behavioralReading` calls both `absent` and `stale` — so a
|
|
1350
|
+
// criterion with no current reading falls through to `behavioral` as
|
|
1351
|
+
// before: reported, ratcheted, UNMEASURED. A stale PASS must not
|
|
1352
|
+
// certify, because a measurement taken a fortnight ago read as one
|
|
1353
|
+
// taken now is the silent default this whole subsystem exists to avoid.
|
|
1354
|
+
const assertion = projectRoot
|
|
1355
|
+
? behavioralPattern(sc, behavioralCachePath(projectRoot))
|
|
1356
|
+
: null;
|
|
1357
|
+
if (assertion) return { kind: "assert", sc, assertion };
|
|
1358
|
+
return { kind: "behavioral", sc, routing: getBehavioralRouting(sc) };
|
|
1359
|
+
}
|
|
1227
1360
|
if (!sc.checked) return { kind: "todo", sc };
|
|
1228
1361
|
const outcome = resolveMatcher(sc, projectRoot);
|
|
1229
1362
|
if (outcome.kind === "assert") return { kind: "assert", sc, assertion: outcome.assertion };
|
|
@@ -1283,6 +1416,25 @@ export function runScaffoldConformity(root: string, opts?: { extraSpecDirs?: str
|
|
|
1283
1416
|
// and it was not hypothetical: the planted-project test in
|
|
1284
1417
|
// test/matcher-registry-parity.test.ts reported its criterion as
|
|
1285
1418
|
// `unmatched` until this argument was supplied.
|
|
1419
|
+
// The census over EVERY ticked behavioral criterion in this spec,
|
|
1420
|
+
// measured or not (#758).
|
|
1421
|
+
//
|
|
1422
|
+
// It cannot be read off `behavioral` below: that list is appended to
|
|
1423
|
+
// only when the plan routed a criterion, which happens only when the
|
|
1424
|
+
// cache had nothing fresh — so it is already the unmeasured half and
|
|
1425
|
+
// carries no denominator. A criterion that was measured and passed is
|
|
1426
|
+
// an ordinary `assert` entry and is invisible to it.
|
|
1427
|
+
//
|
|
1428
|
+
// No second decider: `behavioralCensus` classifies through
|
|
1429
|
+
// `behavioralReading`, which is the function `planSCTests` routed on
|
|
1430
|
+
// one line above.
|
|
1431
|
+
const tickedBehavioral = metadata.scs.filter(sc => sc.checked && isBehavioralSC(sc));
|
|
1432
|
+
if (tickedBehavioral.length > 0) {
|
|
1433
|
+
const census = behavioralCensus(tickedBehavioral.map(sc => sc.id), behavioralCachePath(root));
|
|
1434
|
+
addBehavioralCensus(census);
|
|
1435
|
+
console.log(`${specFile} — ${behavioralCensusLine(census)}`);
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1286
1438
|
for (const entry of planSCTests(metadata.scs, root)) {
|
|
1287
1439
|
const { sc } = entry;
|
|
1288
1440
|
if (entry.kind === "behavioral") {
|
|
@@ -1351,14 +1503,30 @@ export function runScaffoldConformity(root: string, opts?: { extraSpecDirs?: str
|
|
|
1351
1503
|
// to — 54 of them on the day they first became visible (#209).
|
|
1352
1504
|
//
|
|
1353
1505
|
// Failing 54 specs at once would get the whole file skipped, so the
|
|
1354
|
-
// teeth live in a
|
|
1355
|
-
// test/conformity-sc-binding.test.ts
|
|
1356
|
-
//
|
|
1506
|
+
// teeth live in a repo-wide ratchet instead. That ratchet has MOVED:
|
|
1507
|
+
// `UNMATCHED_CHECKED_RATCHET` in test/conformity-sc-binding.test.ts
|
|
1508
|
+
// is gone (#441, and that file's line 44 says so), replaced by the
|
|
1509
|
+
// two-sided bounds named below. Here we name the SCs, per spec, so
|
|
1510
|
+
// they are findable.
|
|
1511
|
+
//
|
|
1512
|
+
// A report, not a test (#211). This slot held
|
|
1513
|
+
// `expect(unmatched.length).toBeGreaterThan(0)` INSIDE
|
|
1514
|
+
// `if (unmatched.length > 0)` — true by construction, with no input
|
|
1515
|
+
// on which it could go red. That is #328's defect verbatim, twenty
|
|
1516
|
+
// lines below the branch #328 repaired, and it was minting one
|
|
1517
|
+
// passing test per strict spec out of a number nothing asserts.
|
|
1518
|
+
//
|
|
1519
|
+
// The repair is #328's: the block says what it found and stops
|
|
1520
|
+
// claiming to have checked it. The teeth stay where #441 put them —
|
|
1521
|
+
// test/sc-binding-split.test.ts bounds UNCLAIMED_RATCHET above and
|
|
1522
|
+
// CLAIMED_UNBINDABLE in both directions — deliberately NOT a
|
|
1523
|
+
// per-spec ratchet to zero, because that number measures the
|
|
1524
|
+
// matcher's vocabulary and the only way to lower it is to make the
|
|
1525
|
+
// specs say less.
|
|
1357
1526
|
const level = metadata.compliance === "permissive" ? "WARN" : "WARN (strict)";
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
});
|
|
1527
|
+
console.warn(
|
|
1528
|
+
`${level}: ${unmatched.length} checked SCs have no pattern matcher in ${specFile}:\n ${unmatched.join("\n ")}`,
|
|
1529
|
+
);
|
|
1362
1530
|
}
|
|
1363
1531
|
// Deliberately BELOW the unmatched branch, not between it and the
|
|
1364
1532
|
// behavioral report. #328's guard slices the source from the
|
|
@@ -1441,25 +1609,95 @@ export function runSpecDiscovery(root: string) {
|
|
|
1441
1609
|
expect(missing).toEqual([]);
|
|
1442
1610
|
});
|
|
1443
1611
|
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1612
|
+
const countTestable = () => collectSpecFiles(specsDir).filter(f => {
|
|
1613
|
+
const fm = parseFrontmatter(readFileSync(join(specsDir, f), "utf-8"));
|
|
1614
|
+
return fm?.testable === "true";
|
|
1615
|
+
}).length;
|
|
1616
|
+
|
|
1617
|
+
// #211. This test had three paths and no failing input:
|
|
1618
|
+
//
|
|
1619
|
+
// specs/ absent -> `if (!existsSync(specsDir)) return;` -> PASS
|
|
1620
|
+
// specs/ with 0 testable -> addFinding(WARN) + console.warn -> PASS
|
|
1621
|
+
// specs/ with N testable -> expect(N).toBeGreaterThan(0) -> PASS
|
|
1622
|
+
//
|
|
1623
|
+
// The third is #328's tautology — the assertion sat in the `else` of
|
|
1624
|
+
// `testable.length === 0`, so the only branch that reached it already knew
|
|
1625
|
+
// the answer. The first two are worse than vacuous: a test NAMED "at least
|
|
1626
|
+
// one testable spec exists" reported green on the two trees where no
|
|
1627
|
+
// testable spec exists. A tree that could not be measured had not passed,
|
|
1628
|
+
// and a gap that was tolerated had not been checked; both were being read
|
|
1629
|
+
// as evidence.
|
|
1630
|
+
//
|
|
1631
|
+
// The zero-testable grace is kept, because it is load-bearing: a scaffolded
|
|
1632
|
+
// consumer gets `testable: no` (lib/scaffold/write-steps.ts), so failing
|
|
1633
|
+
// here would turn every fresh consumer red on day one. It is kept as
|
|
1634
|
+
// `test.todo` — the honest report of a known-false property — rather than
|
|
1635
|
+
// as a pass.
|
|
1636
|
+
//
|
|
1637
|
+
// The probe is TOTAL, and #1027 left it partial. `countTestable` does a
|
|
1638
|
+
// `readFileSync` per spec file and ran bare in the `describe` body, so a
|
|
1639
|
+
// file that vanished between `collectSpecFiles` and the read — or a
|
|
1640
|
+
// dangling symlink, or an unreadable mode — threw at REGISTRATION time.
|
|
1641
|
+
// A throw there does not fail this check: it aborts registration of
|
|
1642
|
+
// everything after it in the file, and a check that never registered
|
|
1643
|
+
// reports as not existing rather than as failing.
|
|
1644
|
+
//
|
|
1645
|
+
// Second order, and the worse half: `writeFindingsReport` runs from an
|
|
1646
|
+
// `afterAll` that an aborted registration never reaches, so the previous
|
|
1647
|
+
// run's `.rungate/conformity-findings.json` stays on disk and
|
|
1648
|
+
// `readConformityFindings` (gates/gate-executor.ts) consumes it as
|
|
1649
|
+
// current — byte-indistinguishable from a fresh clean one.
|
|
1650
|
+
//
|
|
1651
|
+
// That is the same sentence as the comment above, reached by exception
|
|
1652
|
+
// instead of by reading: a tree that could not be measured has not
|
|
1653
|
+
// passed. So the read is caught here and reported as UNMEASURED.
|
|
1654
|
+
let measured: number | null = null;
|
|
1655
|
+
let probeError: unknown = null;
|
|
1656
|
+
try {
|
|
1657
|
+
measured = existsSync(specsDir) ? countTestable() : null;
|
|
1658
|
+
} catch (e) {
|
|
1659
|
+
probeError = e;
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
if (probeError !== null) {
|
|
1663
|
+
const why = probeError instanceof Error ? probeError.message : String(probeError);
|
|
1664
|
+
addFinding({
|
|
1665
|
+
ruleId: "SPEC-DISCOVERY-UNMEASURED",
|
|
1666
|
+
severity: "FAIL",
|
|
1667
|
+
file: "specs/",
|
|
1668
|
+
message: `UNMEASURED: a spec under specs/ could not be read, so testability was never measured — ${why}`,
|
|
1669
|
+
fixCommand: "Repair or remove the unreadable file named in the message, then re-run",
|
|
1449
1670
|
});
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1671
|
+
// A FAILING test, deliberately not the `test.todo` below. That todo is
|
|
1672
|
+
// the grace for a tree with nothing marked testable — a fact this run
|
|
1673
|
+
// READ. A tree it could not read is a different fact and must not
|
|
1674
|
+
// borrow that grace, which is exactly what a per-file
|
|
1675
|
+
// `catch { return false }` inside the filter would have let it do.
|
|
1676
|
+
test("at least one testable spec exists (specs/ could not be read)", () => {
|
|
1677
|
+
expect(
|
|
1678
|
+
probeError,
|
|
1679
|
+
`specs/ could not be read, so testability was never measured: ${why}`,
|
|
1680
|
+
).toBeNull();
|
|
1681
|
+
});
|
|
1682
|
+
} else if (measured === 0) {
|
|
1683
|
+
addFinding({
|
|
1684
|
+
ruleId: "SPEC-DISCOVERY-TESTABLE",
|
|
1685
|
+
severity: "WARN",
|
|
1686
|
+
file: "specs/",
|
|
1687
|
+
message: "No spec has testable: true. Add testable: true to a spec when ready for conformity testing.",
|
|
1688
|
+
fixCommand: `Add "testable: true" to frontmatter of a spec in specs/`,
|
|
1689
|
+
});
|
|
1690
|
+
test.todo("at least one testable spec exists (none marked testable: true yet)");
|
|
1691
|
+
} else {
|
|
1692
|
+
test("at least one testable spec exists", () => {
|
|
1693
|
+
// Deliberately duplicates the "specs/ directory exists" check above.
|
|
1694
|
+
// That one answers "is the directory there"; this one refuses to
|
|
1695
|
+
// answer its OWN question over a tree it could not read, which is the
|
|
1696
|
+
// early `return` this replaces. Unmeasured is a failure, not a pass.
|
|
1697
|
+
expect(existsSync(specsDir), "specs/ is absent, so testability was never measured").toBe(true);
|
|
1698
|
+
expect(countTestable()).toBeGreaterThan(0);
|
|
1699
|
+
});
|
|
1700
|
+
}
|
|
1463
1701
|
});
|
|
1464
1702
|
}
|
|
1465
1703
|
|
|
@@ -1526,7 +1764,11 @@ export function runSpecDrift(
|
|
|
1526
1764
|
}
|
|
1527
1765
|
}
|
|
1528
1766
|
|
|
1529
|
-
|
|
1767
|
+
// Not `allContent.includes(s)`. A bare basename credits a spec for a
|
|
1768
|
+
// test that names a DIFFERENT file — see `lib/spec-requirements.ts`,
|
|
1769
|
+
// where the collision and its one instance on this tree are measured.
|
|
1770
|
+
const ambiguous = ambiguousSpecNames(root, collectSpecFiles(specsDir));
|
|
1771
|
+
const orphaned = testable.filter(s => !specIsReferenced(allContent, s, ambiguous));
|
|
1530
1772
|
|
|
1531
1773
|
// Was `expect(true).toBe(true)` after a console.warn, so the mechanical
|
|
1532
1774
|
// enforcement of CLAUDE.md's "SCs without tests are wishes" could never
|
|
@@ -1562,12 +1804,35 @@ export function runSpecDrift(
|
|
|
1562
1804
|
}
|
|
1563
1805
|
|
|
1564
1806
|
const newlyOrphaned = orphaned.filter(s => !untestedAllowlist.includes(s));
|
|
1565
|
-
expect(
|
|
1807
|
+
expect(
|
|
1808
|
+
newlyOrphaned,
|
|
1809
|
+
remedyMessage({
|
|
1810
|
+
root,
|
|
1811
|
+
check: "DRIFT-2",
|
|
1812
|
+
violations: newlyOrphaned,
|
|
1813
|
+
fix:
|
|
1814
|
+
"name the spec in a test — a `test(\"SC-N: ...\")` title counts as a declaration. " +
|
|
1815
|
+
"Do NOT add it to .claude/conformity-allowlists.json: DRIFT-3 diffs that file " +
|
|
1816
|
+
"against origin/main and fails any array that grew, so the obvious escape hatch " +
|
|
1817
|
+
"turns one red check into two.",
|
|
1818
|
+
}),
|
|
1819
|
+
).toEqual([]);
|
|
1566
1820
|
|
|
1567
1821
|
const staleAllowlist = untestedAllowlist.filter(
|
|
1568
1822
|
s => testable.includes(s) && !orphaned.includes(s),
|
|
1569
1823
|
);
|
|
1570
|
-
expect(
|
|
1824
|
+
expect(
|
|
1825
|
+
staleAllowlist,
|
|
1826
|
+
remedyMessage({
|
|
1827
|
+
root,
|
|
1828
|
+
check: "DRIFT-2 (stale exemption)",
|
|
1829
|
+
violations: staleAllowlist,
|
|
1830
|
+
fix:
|
|
1831
|
+
"these specs now HAVE a referencing test, so delete their entries from " +
|
|
1832
|
+
"untestedSpecs in .claude/conformity-allowlists.json. Shrinking the list is " +
|
|
1833
|
+
"always allowed; only growth is refused.",
|
|
1834
|
+
}),
|
|
1835
|
+
).toEqual([]);
|
|
1571
1836
|
});
|
|
1572
1837
|
|
|
1573
1838
|
test("DRIFT-3: the exemption lists did not GROW", () => {
|
|
@@ -1612,6 +1877,23 @@ export function runSpecDrift(
|
|
|
1612
1877
|
`exemption a deliberate recorded decision rather than a diff.`,
|
|
1613
1878
|
).toEqual([]);
|
|
1614
1879
|
});
|
|
1880
|
+
|
|
1881
|
+
test("DRIFT-4: Every spec states criteria, or declares that it states none", () => {
|
|
1882
|
+
if (!existsSync(specsDir)) return checkSkipped("DRIFT-4", "no specs/ directory");
|
|
1883
|
+
const specs = collectSpecFiles(specsDir);
|
|
1884
|
+
if (specs.length === 0) return checkSkipped("DRIFT-4", "no spec files");
|
|
1885
|
+
|
|
1886
|
+
// Unlike DRIFT-2 this has no ratchet and does not need one. The
|
|
1887
|
+
// exemption is a line in the document's own frontmatter, so a consumer
|
|
1888
|
+
// on upgrade can clear a violation by declaring it where the reader it
|
|
1889
|
+
// affects will see it — no list to adopt, nothing inherited from
|
|
1890
|
+
// rungate's backlog, and nothing DRIFT-3 has to police.
|
|
1891
|
+
const violations = requirementViolations(specs.map(s => readSpecRequirements(specsDir, s)));
|
|
1892
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
1893
|
+
|
|
1894
|
+
const badFlags = testableViolations(specsDir, specs);
|
|
1895
|
+
expect(badFlags, badFlags.join("\n")).toEqual([]);
|
|
1896
|
+
});
|
|
1615
1897
|
});
|
|
1616
1898
|
}
|
|
1617
1899
|
|
|
@@ -1622,67 +1904,11 @@ export function runSpecDrift(
|
|
|
1622
1904
|
* `runDocHygiene(ROOT, findings)`.
|
|
1623
1905
|
*/
|
|
1624
1906
|
/**
|
|
1625
|
-
*
|
|
1626
|
-
*
|
|
1627
|
-
*
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
* being faster. A detector that got quicker by finding less would pass every
|
|
1631
|
-
* timing test and none of the ones that matter.
|
|
1632
|
-
*
|
|
1633
|
-
* Walks newest-first, so the first time a path appears is its latest commit
|
|
1634
|
-
* and later entries are ignored. Paths deleted or renamed earlier in history
|
|
1635
|
-
* also appear; they are never looked up, which is cheaper than filtering them.
|
|
1636
|
-
*
|
|
1637
|
-
* `pathspecs` must be the SET OF PATHS THE CALLER WILL LOOK UP, not the
|
|
1638
|
-
* directories they mostly live in. The first version of this took the six doc
|
|
1639
|
-
* directories, which is where the staleness scan finds most of its files — but
|
|
1640
|
-
* the scan also walks root-level markdown, and `git log -- specs docs` cannot
|
|
1641
|
-
* report a commit for `AGENTS.md`. Six files silently dropped out of the
|
|
1642
|
-
* lookup and `if (!dateStr) continue` read that as "newly added, so not
|
|
1643
|
-
* stale". A detector narrower than what it detects, shipped inside the fix for
|
|
1644
|
-
* a different instance of the same thing. Passing the lookup set makes the
|
|
1645
|
-
* pathspec and the keys the same list by construction, so they cannot drift.
|
|
1646
|
-
*
|
|
1647
|
-
* Fails CLOSED. `spawnSync` does not throw when git exits non-zero, so a
|
|
1648
|
-
* broken invocation would otherwise return an empty map and every file would
|
|
1649
|
-
* read as newly added — total blindness that looks exactly like a clean run.
|
|
1650
|
-
*/
|
|
1651
|
-
export function lastCommitDates(root: string, pathspecs: string[]): Map<string, string> {
|
|
1652
|
-
const lastCommit = new Map<string, string>();
|
|
1653
|
-
// Not a git repository: nothing is tracked, so no path has a commit date.
|
|
1654
|
-
// That is a real answer, not a failure, and scaffolded projects hit it.
|
|
1655
|
-
if (!existsSync(join(root, ".git"))) return lastCommit;
|
|
1656
|
-
if (pathspecs.length === 0) return lastCommit;
|
|
1657
|
-
{
|
|
1658
|
-
const log = spawnSync(
|
|
1659
|
-
"git",
|
|
1660
|
-
["-C", root, "log", "--format=%H %ci", "--name-only", "--", ...pathspecs],
|
|
1661
|
-
{ timeout: 60_000, maxBuffer: 64 * 1024 * 1024 },
|
|
1662
|
-
);
|
|
1663
|
-
if (log.error) {
|
|
1664
|
-
throw new Error(`lastCommitDates: git log could not run in ${root}: ${log.error.message}`);
|
|
1665
|
-
}
|
|
1666
|
-
if (log.status !== 0) {
|
|
1667
|
-
throw new Error(
|
|
1668
|
-
`lastCommitDates: git log exited ${log.status} in ${root}: ` +
|
|
1669
|
-
`${log.stderr?.toString().trim() || "(no stderr)"}`,
|
|
1670
|
-
);
|
|
1671
|
-
}
|
|
1672
|
-
let currentDate = "";
|
|
1673
|
-
for (const line of log.stdout.toString().split("\n")) {
|
|
1674
|
-
const header = /^[0-9a-f]{7,40} (\d{4}-\d{2}-\d{2})/.exec(line);
|
|
1675
|
-
if (header) {
|
|
1676
|
-
currentDate = header[1];
|
|
1677
|
-
continue;
|
|
1678
|
-
}
|
|
1679
|
-
const path = line.trim();
|
|
1680
|
-
if (!path || !currentDate) continue;
|
|
1681
|
-
if (!lastCommit.has(path)) lastCommit.set(path, currentDate);
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
return lastCommit;
|
|
1685
|
-
}
|
|
1907
|
+
* Re-exported from `lib/git-dates.ts` (#508). The body moved to a leaf so a
|
|
1908
|
+
* CLI outside the test runner can call it; this stays the name SC-842 and
|
|
1909
|
+
* every existing caller use.
|
|
1910
|
+
*/
|
|
1911
|
+
export { lastCommitDates };
|
|
1686
1912
|
|
|
1687
1913
|
export function runDocHygiene(
|
|
1688
1914
|
root: string,
|
|
@@ -1821,10 +2047,34 @@ export function runDocHygiene(
|
|
|
1821
2047
|
}
|
|
1822
2048
|
|
|
1823
2049
|
const newOrphans = orphans.filter(o => !orphanAllowlist.includes(o));
|
|
1824
|
-
expect(
|
|
2050
|
+
expect(
|
|
2051
|
+
newOrphans,
|
|
2052
|
+
remedyMessage({
|
|
2053
|
+
root,
|
|
2054
|
+
check: "HYGIENE-3",
|
|
2055
|
+
violations: newOrphans,
|
|
2056
|
+
fix:
|
|
2057
|
+
"make something genuinely reference the file — a routing row, a spec, an import. " +
|
|
2058
|
+
"Do NOT add it to .claude/conformity-allowlists.json: DRIFT-3 diffs that file " +
|
|
2059
|
+
"against origin/main and fails any array that grew, so the obvious escape hatch " +
|
|
2060
|
+
"turns one red check into two. If the file is history rather than guidance, " +
|
|
2061
|
+
"move it under reference/ instead.",
|
|
2062
|
+
}),
|
|
2063
|
+
).toEqual([]);
|
|
1825
2064
|
|
|
1826
2065
|
const staleAllowlist = orphanAllowlist.filter(o => !orphans.includes(o));
|
|
1827
|
-
expect(
|
|
2066
|
+
expect(
|
|
2067
|
+
staleAllowlist,
|
|
2068
|
+
remedyMessage({
|
|
2069
|
+
root,
|
|
2070
|
+
check: "HYGIENE-3 (stale exemption)",
|
|
2071
|
+
violations: staleAllowlist,
|
|
2072
|
+
fix:
|
|
2073
|
+
"these files are now referenced, so delete their entries from orphanFiles in " +
|
|
2074
|
+
".claude/conformity-allowlists.json. Shrinking the list is always allowed; " +
|
|
2075
|
+
"only growth is refused.",
|
|
2076
|
+
}),
|
|
2077
|
+
).toEqual([]);
|
|
1828
2078
|
});
|
|
1829
2079
|
|
|
1830
2080
|
test("HYGIENE-4: Every spec appears in the specs routing table", () => {
|
|
@@ -1842,7 +2092,18 @@ export function runDocHygiene(
|
|
|
1842
2092
|
// SPEC-TEMPLATE.md documents how to write a spec; it governs nothing.
|
|
1843
2093
|
.filter(f => f !== "SPEC-TEMPLATE.md");
|
|
1844
2094
|
const unlisted = specFiles.filter(f => !routing.includes(f));
|
|
1845
|
-
expect(
|
|
2095
|
+
expect(
|
|
2096
|
+
unlisted,
|
|
2097
|
+
remedyMessage({
|
|
2098
|
+
root,
|
|
2099
|
+
check: "HYGIENE-4",
|
|
2100
|
+
violations: unlisted,
|
|
2101
|
+
fix:
|
|
2102
|
+
"add a row naming each file to .claude/rules/specs-routing.md — the table this " +
|
|
2103
|
+
"check reads. A spec with no `- [ ]` criteria needs its row to say so explicitly. " +
|
|
2104
|
+
"There is no exemption list for this one.",
|
|
2105
|
+
}),
|
|
2106
|
+
).toEqual([]);
|
|
1846
2107
|
});
|
|
1847
2108
|
|
|
1848
2109
|
test("HYGIENE-5: reference/ contents not in active routing", () => {
|
|
@@ -2003,7 +2264,47 @@ export function runDocHygiene(
|
|
|
2003
2264
|
);
|
|
2004
2265
|
}
|
|
2005
2266
|
|
|
2006
|
-
|
|
2267
|
+
// The leniency above is about the BACKLOG. It was never meant to be
|
|
2268
|
+
// leniency about whether the scan happened, and `expect(true).toBe(true)`
|
|
2269
|
+
// made it both: with no assertion on the reading, the only red this
|
|
2270
|
+
// check could produce was a timeout — which is exactly the red it
|
|
2271
|
+
// produced, four times, until #508 made the staleness scan one git
|
|
2272
|
+
// call instead of ninety-two.
|
|
2273
|
+
//
|
|
2274
|
+
// So the reading is asserted and the backlog still is not. `0
|
|
2275
|
+
// candidates` over 91 documents and `0 candidates` over 4 are the same
|
|
2276
|
+
// line of output, and nothing here could previously tell them apart.
|
|
2277
|
+
console.log(
|
|
2278
|
+
`HYGIENE-6 coverage: read ${result.coverage.read} of ${result.coverage.planned} planned documents ` +
|
|
2279
|
+
`from ${result.scan.roots.length} directories (${result.scan.source})`,
|
|
2280
|
+
);
|
|
2281
|
+
|
|
2282
|
+
// #545/#544. The failure intake and the promotion decision are reported
|
|
2283
|
+
// here and asserted nowhere: HYGIENE-6 is advisory by declaration, and a
|
|
2284
|
+
// new refusal in it would turn a backlog report into a gate.
|
|
2285
|
+
//
|
|
2286
|
+
// What it is NOT allowed to be is silent. The intake reads `git log`,
|
|
2287
|
+
// and `actions/checkout@v4` with no `fetch-depth` clones to depth 1, so
|
|
2288
|
+
// on a runner this line says UNMEASURED/SHALLOW_HISTORY — which is the
|
|
2289
|
+
// state, printed, rather than a zero that reads as "no failures found".
|
|
2290
|
+
const { promotionSummary } = await import("./rule-promotion");
|
|
2291
|
+
console.log(
|
|
2292
|
+
`HYGIENE-6 intake: ${result.evidence.reading}, ` +
|
|
2293
|
+
`${result.evidence.candidates.length} evidence-backed candidates ` +
|
|
2294
|
+
`over ${result.evidence.commitsScanned} commits` +
|
|
2295
|
+
(result.evidence.unmeasuredReasons.length > 0
|
|
2296
|
+
? ` — ${result.evidence.unmeasuredReasons.join("; ")}`
|
|
2297
|
+
: ""),
|
|
2298
|
+
);
|
|
2299
|
+
console.log(`HYGIENE-6 ${promotionSummary(result.promotion)}`);
|
|
2300
|
+
expect(
|
|
2301
|
+
result.coverage.planned,
|
|
2302
|
+
"the documentation scan planned no files, so this check measured nothing",
|
|
2303
|
+
).toBeGreaterThan(0);
|
|
2304
|
+
expect(
|
|
2305
|
+
result.coverage.unread,
|
|
2306
|
+
"planned documents could not be read, so the candidate list above is partial",
|
|
2307
|
+
).toEqual([]);
|
|
2007
2308
|
});
|
|
2008
2309
|
|
|
2009
2310
|
test("HYGIENE-11: Stale unreferenced files warn with archival suggestion", () => {
|
|
@@ -2369,24 +2670,35 @@ export function runAgentFileValidation(root: string) {
|
|
|
2369
2670
|
|
|
2370
2671
|
test("AGENT-6: All file paths referenced in briefs exist", () => {
|
|
2371
2672
|
if (!existsSync(agentsDir)) return;
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2673
|
+
// #884: a reference into a package that is not installed here is
|
|
2674
|
+
// UNMEASURED, not broken. A ship worktree has no install, and reporting
|
|
2675
|
+
// its missing corpus as the project's test health failed `tests-pass` on
|
|
2676
|
+
// every run. The three outcomes live in `lib/brief-references.ts`.
|
|
2677
|
+
const reading = briefReferenceReading(root, agentsDir);
|
|
2678
|
+
|
|
2679
|
+
for (const u of reading.unresolvable) {
|
|
2680
|
+
addFinding({
|
|
2681
|
+
ruleId: "AGENT-REF-UNMEASURED",
|
|
2682
|
+
severity: "WARN",
|
|
2683
|
+
file: `.claude/agents/${u.brief}`,
|
|
2684
|
+
message: unresolvableBriefReferenceMessage(u),
|
|
2685
|
+
fixCommand: `Install dependencies in this checkout and re-run — "${u.packageRoot}" is absent, so AGENT-6 could not check this reference`,
|
|
2686
|
+
});
|
|
2687
|
+
}
|
|
2688
|
+
for (const b of reading.broken) {
|
|
2689
|
+
addFinding({
|
|
2690
|
+
ruleId: "AGENT-BROKEN-REF",
|
|
2691
|
+
severity: "FAIL",
|
|
2692
|
+
file: `.claude/agents/${b.brief}`,
|
|
2693
|
+
message: `Brief references "${b.ref}" but file does not exist`,
|
|
2694
|
+
fixCommand: `Check prompts/ directory for correct filename — may be a typo or renamed file`,
|
|
2695
|
+
});
|
|
2389
2696
|
}
|
|
2697
|
+
|
|
2698
|
+
const notice = unmeasuredBriefReferenceNotice(reading);
|
|
2699
|
+
if (notice !== null) console.error(notice);
|
|
2700
|
+
|
|
2701
|
+
const broken = reading.broken.map(b => `${b.brief} → ${b.ref}`);
|
|
2390
2702
|
if (broken.length > 0) {
|
|
2391
2703
|
console.error(`Broken references in agent briefs:\n ${broken.join("\n ")}`);
|
|
2392
2704
|
}
|
|
@@ -2464,92 +2776,94 @@ interface FallowResult {
|
|
|
2464
2776
|
// fail as timeouts instead of skipping, which blocks the pre-push conformity
|
|
2465
2777
|
// gate on a tool nobody installed.
|
|
2466
2778
|
//
|
|
2467
|
-
// Resolve it locally instead, with no network path.
|
|
2468
|
-
//
|
|
2469
|
-
//
|
|
2470
|
-
//
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
);
|
|
2779
|
+
// Resolve it locally instead, with no network path. The predicate moved to
|
|
2780
|
+
// `lib/fallow-availability.ts` in #581 and is re-exported here rather than
|
|
2781
|
+
// copied: this module imports `bun:test` at module scope, so a CLI cannot
|
|
2782
|
+
// import it, and `scripts/generate-code-map.ts` is a CLI that needed the
|
|
2783
|
+
// same answer. It had a second answer — a bare `npx fallow` — for four
|
|
2784
|
+
// months, which is the defect #581 is about.
|
|
2785
|
+
export { fallowAvailable };
|
|
2786
|
+
|
|
2787
|
+
/**
|
|
2788
|
+
* Shared skip notice — a skipped check must say so, not read as a pass, and
|
|
2789
|
+
* must say WHICH silence it is (#935).
|
|
2790
|
+
*
|
|
2791
|
+
* It used to take only the check name and print one sentence: *"fallow is not
|
|
2792
|
+
* installed (not a declared dependency)"*. That sentence was printed for a
|
|
2793
|
+
* run the clock killed, on trees where fallow is an exactly-pinned declared
|
|
2794
|
+
* dependency, because `runFallowCommand` returned `null` for both. The
|
|
2795
|
+
* reading is passed in now, so the notice is the reading's own sentence
|
|
2796
|
+
* rather than a guess about which of three things happened.
|
|
2797
|
+
*/
|
|
2798
|
+
function fallowSkipped(check: string, why: FallowUnavailable): void {
|
|
2799
|
+
console.warn(`SKIP ${check}: ${why.reason}. This check did NOT run.`);
|
|
2489
2800
|
}
|
|
2490
2801
|
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
try {
|
|
2501
|
-
return JSON.parse(stdout);
|
|
2502
|
-
} catch {
|
|
2503
|
-
return null;
|
|
2504
|
-
}
|
|
2802
|
+
/**
|
|
2803
|
+
* The reading, or the refusal that says why there isn't one.
|
|
2804
|
+
*
|
|
2805
|
+
* Delegates to `lib/fallow-availability.ts` — one spawn in the tree, one
|
|
2806
|
+
* budget, and the timeout classified before the empty-stdout branch. This
|
|
2807
|
+
* function used to hold its own copy of all three (#935).
|
|
2808
|
+
*/
|
|
2809
|
+
function runFallowCommand(root: string, args: string[]): FallowReading<FallowResult> {
|
|
2810
|
+
return runFallowJson<FallowResult>(root, args);
|
|
2505
2811
|
}
|
|
2506
2812
|
|
|
2507
2813
|
export function runFallowCheck(root: string, opts?: { skipUnusedExports?: boolean; warnOnly?: boolean }) {
|
|
2508
2814
|
const fail = !opts?.warnOnly;
|
|
2509
2815
|
describe("Fallow: static analysis", () => {
|
|
2816
|
+
// Before the four checks, not inside the first of them. fallow builds its
|
|
2817
|
+
// index on first use — ~1.5s cold against ~455ms warm — and whichever
|
|
2818
|
+
// check ran first paid for it alone. That made FALLOW-1 look slow and the
|
|
2819
|
+
// other three look fast, which sent two investigations after FALLOW-1's
|
|
2820
|
+
// own query instead of after the cold start (#935).
|
|
2821
|
+
beforeAll(() => warmFallow(root), FALLOW_BUDGET_MS);
|
|
2822
|
+
|
|
2510
2823
|
test("FALLOW-1: No unused files", () => {
|
|
2511
|
-
const
|
|
2512
|
-
if (
|
|
2824
|
+
const reading = runFallowCommand(root, ["dead-code", "--unused-files"]);
|
|
2825
|
+
if ("unavailable" in reading) { fallowSkipped("FALLOW-1 (unused files)", reading.unavailable); return; }
|
|
2826
|
+
const result = reading.data;
|
|
2513
2827
|
const unused = result.unused_files.map(f => f.path).filter(p => !p.includes("test/fixtures/") && !p.includes("reference/"));
|
|
2514
2828
|
if (unused.length > 0) {
|
|
2515
2829
|
console.warn(`Unused files (${unused.length}):\n ${unused.slice(0, 10).join("\n ")}${unused.length > 10 ? `\n ... and ${unused.length - 10} more` : ""}`);
|
|
2516
2830
|
}
|
|
2517
2831
|
if (fail) expect(unused).toEqual([]);
|
|
2518
|
-
});
|
|
2832
|
+
}, FALLOW_BUDGET_MS);
|
|
2519
2833
|
|
|
2520
2834
|
if (!opts?.skipUnusedExports) {
|
|
2521
2835
|
test("FALLOW-2: No unused exports", () => {
|
|
2522
|
-
const
|
|
2523
|
-
if (
|
|
2524
|
-
const unused = (
|
|
2836
|
+
const reading = runFallowCommand(root, ["dead-code", "--unused-exports"]);
|
|
2837
|
+
if ("unavailable" in reading) { fallowSkipped("FALLOW-2 (unused exports)", reading.unavailable); return; }
|
|
2838
|
+
const unused = (reading.data.unused_exports || [])
|
|
2525
2839
|
.filter(e => !e.path.includes("test/fixtures/") && !e.path.includes("reference/"))
|
|
2526
2840
|
.map(e => `${e.path}:${e.line ?? "?"} ${e.export_name}`);
|
|
2527
2841
|
if (unused.length > 0) {
|
|
2528
2842
|
console.warn(`Unused exports (${unused.length}):\n ${unused.slice(0, 10).join("\n ")}${unused.length > 10 ? `\n ... and ${unused.length - 10} more` : ""}`);
|
|
2529
2843
|
}
|
|
2530
2844
|
if (fail) expect(unused).toEqual([]);
|
|
2531
|
-
});
|
|
2845
|
+
}, FALLOW_BUDGET_MS);
|
|
2532
2846
|
}
|
|
2533
2847
|
|
|
2534
2848
|
test("FALLOW-3: No unused dependencies", () => {
|
|
2535
|
-
const
|
|
2536
|
-
if (
|
|
2537
|
-
const unused =
|
|
2849
|
+
const reading = runFallowCommand(root, ["dead-code", "--unused-deps"]);
|
|
2850
|
+
if ("unavailable" in reading) { fallowSkipped("FALLOW-3 (unused dependencies)", reading.unavailable); return; }
|
|
2851
|
+
const unused = reading.data.unused_dependencies?.map(d => d.name) || [];
|
|
2538
2852
|
if (unused.length > 0) {
|
|
2539
2853
|
console.warn(`Unused dependencies (${unused.length}):\n ${unused.join("\n ")}`);
|
|
2540
2854
|
}
|
|
2541
2855
|
if (fail) expect(unused).toEqual([]);
|
|
2542
|
-
});
|
|
2856
|
+
}, FALLOW_BUDGET_MS);
|
|
2543
2857
|
|
|
2544
2858
|
test("FALLOW-4: No circular dependencies", () => {
|
|
2545
|
-
const
|
|
2546
|
-
if (
|
|
2547
|
-
const circles =
|
|
2859
|
+
const reading = runFallowCommand(root, ["dead-code"]);
|
|
2860
|
+
if ("unavailable" in reading) { fallowSkipped("FALLOW-4 (circular dependencies)", reading.unavailable); return; }
|
|
2861
|
+
const circles = reading.data.circular_dependencies || [];
|
|
2548
2862
|
if (circles.length > 0) {
|
|
2549
2863
|
console.warn(`Circular dependencies (${circles.length}):\n ${circles.slice(0, 5).map(c => c.path || JSON.stringify(c.chain ?? c)).join("\n ")}${circles.length > 5 ? `\n ... and ${circles.length - 5} more` : ""}`);
|
|
2550
2864
|
}
|
|
2551
2865
|
if (fail) expect(circles).toEqual([]);
|
|
2552
|
-
});
|
|
2866
|
+
}, FALLOW_BUDGET_MS);
|
|
2553
2867
|
});
|
|
2554
2868
|
}
|
|
2555
2869
|
|
|
@@ -2976,7 +3290,19 @@ function walkSources(dir: string, rel: string, out: Array<{ path: string; text:
|
|
|
2976
3290
|
walkSources(abs, relPath, out);
|
|
2977
3291
|
continue;
|
|
2978
3292
|
}
|
|
2979
|
-
|
|
3293
|
+
// `SOURCE_EXTENSIONS`, not `/\.(ts|js)$/`. The evidence walk counts
|
|
3294
|
+
// `.tsx .jsx .mjs .cjs .mts .cts` as source and this scan could not read
|
|
3295
|
+
// any of them, so the invariant stated above — the walk must not claim
|
|
3296
|
+
// source the scan would never read — was violated for six extensions
|
|
3297
|
+
// (#506).
|
|
3298
|
+
//
|
|
3299
|
+
// Measured: a project whose only code is `lib/app.tsx` holding a live
|
|
3300
|
+
// direct config read scans ZERO files, and CONFIG-1 refuses with *"This
|
|
3301
|
+
// project DOES contain source — lib/app.tsx — none of it in the
|
|
3302
|
+
// directories this check reads"*. `lib/` is the first directory this
|
|
3303
|
+
// check reads. The sentence is false, the refusal is unclearable, and
|
|
3304
|
+
// the read goes uncaught — three failures from one missing extension.
|
|
3305
|
+
if (!SOURCE_EXTENSIONS.test(e.name) || /\.test\.ts$/.test(e.name)) continue;
|
|
2980
3306
|
out.push({ path: relPath, text: readFileSync(abs, "utf-8") });
|
|
2981
3307
|
}
|
|
2982
3308
|
}
|
|
@@ -2989,6 +3315,16 @@ export interface ConfigAccessScan {
|
|
|
2989
3315
|
filesScanned: number;
|
|
2990
3316
|
/** Bytes read. Zero with a non-zero file count means empty sources. */
|
|
2991
3317
|
bytesScanned: number;
|
|
3318
|
+
/**
|
|
3319
|
+
* Files read by the out-of-reach fallback rather than by walking
|
|
3320
|
+
* `CONFIG_SCAN_DIRS` (#506). Empty on every ordinary scan.
|
|
3321
|
+
*
|
|
3322
|
+
* Recorded rather than merged invisibly because `coverageShortfall` is
|
|
3323
|
+
* about to be asked how many source files went unread, and these are
|
|
3324
|
+
* precisely the ones that would otherwise be counted as unread a line
|
|
3325
|
+
* after being read.
|
|
3326
|
+
*/
|
|
3327
|
+
fallbackFiles: string[];
|
|
2992
3328
|
}
|
|
2993
3329
|
|
|
2994
3330
|
/**
|
|
@@ -3010,7 +3346,93 @@ export function scanConfigAccess(root: string): ConfigAccessScan {
|
|
|
3010
3346
|
const n = classifyConfigSites(f.text).filter(site => site.kind === "use").length;
|
|
3011
3347
|
if (n > 0) counts[f.path] = n;
|
|
3012
3348
|
}
|
|
3013
|
-
return { counts, filesScanned: files.length, bytesScanned };
|
|
3349
|
+
return { counts, filesScanned: files.length, bytesScanned, fallbackFiles: [] };
|
|
3350
|
+
}
|
|
3351
|
+
|
|
3352
|
+
/**
|
|
3353
|
+
* CONFIG-1's reading, with the files it could not reach added when — and
|
|
3354
|
+
* only when — it reached nothing at all (#506).
|
|
3355
|
+
*
|
|
3356
|
+
* ## The refusal this replaces could not be cleared
|
|
3357
|
+
*
|
|
3358
|
+
* A documentation site whose only code-extension file is `docs/assets/
|
|
3359
|
+
* search.js` got a CONFIG-1 FAIL with no action available but deleting the
|
|
3360
|
+
* asset. `docs/` is not in `CONFIG_SCAN_DIRS`, and that list is fixed in the
|
|
3361
|
+
* harness, so no change inside the project could make the check read it.
|
|
3362
|
+
*
|
|
3363
|
+
* ## Why not a `docs/` exclusion, an allowlist, or a lower severity
|
|
3364
|
+
*
|
|
3365
|
+
* A bundled `search.js` and a runnable `docs/examples/client.ts` are the
|
|
3366
|
+
* same bytes to any heuristic available here, so excluding the directory is
|
|
3367
|
+
* the narrowing #506's body warns is most likely to buy green with
|
|
3368
|
+
* blindness. An allowlist in the consumer's config makes the file's name
|
|
3369
|
+
* evidence that somebody considered it. And dropping the severity to
|
|
3370
|
+
* UNMEASURED was the first thing I built and the wrong thing: it leaves a
|
|
3371
|
+
* live `join(process.cwd(), ".claude", "rungate.json")` in `bin/cli.ts`
|
|
3372
|
+
* unreported, which is one of the five layouts #476's security review
|
|
3373
|
+
* specifically fixed.
|
|
3374
|
+
*
|
|
3375
|
+
* Reading the files has none of those costs. The walk has ALREADY located
|
|
3376
|
+
* them and already applied every exclusion it applies — tests, type
|
|
3377
|
+
* declarations, root tool configs, dot-directories, build output, symlinks
|
|
3378
|
+
* leaving the project — so this re-reads that decision rather than making a
|
|
3379
|
+
* new one. A clean `search.js` becomes a PASS that MEASURED something; a
|
|
3380
|
+
* `docs/examples/client.ts` with a real direct read becomes a FAIL that
|
|
3381
|
+
* names the file.
|
|
3382
|
+
*
|
|
3383
|
+
* ## Why it is not "widen `CONFIG_SCAN_DIRS`" (#492/#501's option 3)
|
|
3384
|
+
*
|
|
3385
|
+
* It engages only when `filesScanned === 0` — the state where the check's
|
|
3386
|
+
* output today is an unsatisfiable refusal and therefore worth nothing. No
|
|
3387
|
+
* project that passes today can start failing, no ratchet moves, and
|
|
3388
|
+
* `coverageShortfall`'s domain (`filesScanned > 0`) is untouched, so the
|
|
3389
|
+
* guard #492 measured yesterday keeps the meaning it was measured with.
|
|
3390
|
+
* This closes neither #492 nor #501.
|
|
3391
|
+
*
|
|
3392
|
+
* ## Fail-closed on an unreadable file
|
|
3393
|
+
*
|
|
3394
|
+
* If ANY of them cannot be read, the fallback does not engage at all and
|
|
3395
|
+
* the original refusal stands, naming the source. Reading three of four and
|
|
3396
|
+
* reporting a PASS would be a scan that shrank silently — the second shape
|
|
3397
|
+
* in `.claude/rules/checks-must-be-able-to-fail.md` — and the window is
|
|
3398
|
+
* real: these paths come from a walk that finished before this call.
|
|
3399
|
+
*/
|
|
3400
|
+
export function scanConfigAccessWithReach(
|
|
3401
|
+
root: string,
|
|
3402
|
+
evidence: SourceEvidence,
|
|
3403
|
+
): ConfigAccessScan {
|
|
3404
|
+
const scan = scanConfigAccess(root);
|
|
3405
|
+
// Asked FIRST and separately. A fallback that merged into a non-empty
|
|
3406
|
+
// scan would change what every project is measured on, which is the
|
|
3407
|
+
// behavioural change this one is scoped not to be.
|
|
3408
|
+
if (scan.filesScanned > 0) return scan;
|
|
3409
|
+
if (evidence.unscannedFiles.length === 0) return scan;
|
|
3410
|
+
|
|
3411
|
+
const read: Array<{ path: string; text: string }> = [];
|
|
3412
|
+
for (const rel of evidence.unscannedFiles) {
|
|
3413
|
+
try {
|
|
3414
|
+
read.push({ path: rel, text: readFileSync(join(root, rel), "utf-8") });
|
|
3415
|
+
} catch {
|
|
3416
|
+
// Fail closed — see the docstring. The caller's refusal is the
|
|
3417
|
+
// correct outcome for a project this check could not finish reading.
|
|
3418
|
+
return scan;
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
|
|
3422
|
+
const counts = { ...scan.counts };
|
|
3423
|
+
let bytesScanned = scan.bytesScanned;
|
|
3424
|
+
for (const f of read) {
|
|
3425
|
+
bytesScanned += f.text.length;
|
|
3426
|
+
if (f.path === CONFIG_LOADER) continue;
|
|
3427
|
+
const n = classifyConfigSites(f.text).filter(site => site.kind === "use").length;
|
|
3428
|
+
if (n > 0) counts[f.path] = n;
|
|
3429
|
+
}
|
|
3430
|
+
return {
|
|
3431
|
+
counts,
|
|
3432
|
+
filesScanned: scan.filesScanned + read.length,
|
|
3433
|
+
bytesScanned,
|
|
3434
|
+
fallbackFiles: read.map(f => f.path),
|
|
3435
|
+
};
|
|
3014
3436
|
}
|
|
3015
3437
|
|
|
3016
3438
|
/**
|
|
@@ -3029,25 +3451,64 @@ export function scanConfigAccess(root: string): ConfigAccessScan {
|
|
|
3029
3451
|
* through both scans and asserts they disagree, so the collapse cannot be made
|
|
3030
3452
|
* quietly in the name of consistency.
|
|
3031
3453
|
*/
|
|
3454
|
+
/**
|
|
3455
|
+
* Every module in `workflows/`, in any language this project writes them in.
|
|
3456
|
+
*
|
|
3457
|
+
* It was `.js` only, non-recursively, because rungate's own workflows are
|
|
3458
|
+
* flat `.js` — they cannot import (#69). A consumer keeping TypeScript
|
|
3459
|
+
* workflows got CONFIG-2 reading zero files over a directory full of code,
|
|
3460
|
+
* which the empty-scan refusal then reported as `the scan read NO files`,
|
|
3461
|
+
* unsatisfiable without converting the workflows to `.js` (#501).
|
|
3462
|
+
*
|
|
3463
|
+
* `.d.ts` is excluded for the same reason the CONFIG-1 walk excludes it:
|
|
3464
|
+
* declarations contain no calls.
|
|
3465
|
+
*/
|
|
3466
|
+
const WORKFLOW_MODULE = /\.(js|mjs|cjs|ts|mts|cts)$/;
|
|
3467
|
+
|
|
3468
|
+
function workflowModules(dir: string, rel: string, out: string[]): void {
|
|
3469
|
+
let entries;
|
|
3470
|
+
try {
|
|
3471
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
3472
|
+
} catch {
|
|
3473
|
+
return;
|
|
3474
|
+
}
|
|
3475
|
+
for (const e of entries) {
|
|
3476
|
+
const childRel = rel ? `${rel}/${e.name}` : e.name;
|
|
3477
|
+
// RECURSIVE, and that is load-bearing rather than tidy. CONFIG-1
|
|
3478
|
+
// declines `workflows/` entirely on the grounds that CONFIG-2 owns it,
|
|
3479
|
+
// so a depth CONFIG-2 does not reach is a depth NEITHER check reads —
|
|
3480
|
+
// a blind spot the hand-off would create rather than one it inherits.
|
|
3481
|
+
if (e.isDirectory()) {
|
|
3482
|
+
if (e.name === "node_modules") continue;
|
|
3483
|
+
workflowModules(join(dir, e.name), childRel, out);
|
|
3484
|
+
continue;
|
|
3485
|
+
}
|
|
3486
|
+
if (!e.isFile()) continue;
|
|
3487
|
+
if (!WORKFLOW_MODULE.test(e.name) || e.name.endsWith(".d.ts")) continue;
|
|
3488
|
+
out.push(childRel);
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3491
|
+
|
|
3032
3492
|
export function scanPromptConfigAccess(root: string): ConfigAccessScan {
|
|
3033
3493
|
const dir = join(root, "workflows");
|
|
3034
3494
|
const counts: Record<string, number> = {};
|
|
3035
3495
|
let filesScanned = 0;
|
|
3036
3496
|
let bytesScanned = 0;
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3497
|
+
const entries: string[] = [];
|
|
3498
|
+
workflowModules(dir, "", entries);
|
|
3499
|
+
for (const f of entries.sort()) {
|
|
3500
|
+
let text: string;
|
|
3501
|
+
try {
|
|
3502
|
+
text = readFileSync(join(dir, f), "utf-8");
|
|
3503
|
+
} catch {
|
|
3504
|
+
continue;
|
|
3505
|
+
}
|
|
3045
3506
|
filesScanned++;
|
|
3046
3507
|
bytesScanned += text.length;
|
|
3047
3508
|
const n = (stripJsComments(text).match(configAccessPattern()) || []).length;
|
|
3048
3509
|
if (n > 0) counts[`workflows/${f}`] = n;
|
|
3049
3510
|
}
|
|
3050
|
-
return { counts, filesScanned, bytesScanned };
|
|
3511
|
+
return { counts, filesScanned, bytesScanned, fallbackFiles: [] };
|
|
3051
3512
|
}
|
|
3052
3513
|
|
|
3053
3514
|
/**
|
|
@@ -3290,7 +3751,7 @@ export function configAccessViolationRecords(
|
|
|
3290
3751
|
* refusal had never been reachable.
|
|
3291
3752
|
*/
|
|
3292
3753
|
/** Anything that can carry code. Deliberately WIDER than any scanner's filter. */
|
|
3293
|
-
const SOURCE_EXTENSIONS = /\.(ts|tsx|js|jsx|mjs|cjs|mts|cts)$/;
|
|
3754
|
+
export const SOURCE_EXTENSIONS = /\.(ts|tsx|js|jsx|mjs|cjs|mts|cts)$/;
|
|
3294
3755
|
|
|
3295
3756
|
/** Directories that hold code nobody wrote, or output nobody edits. */
|
|
3296
3757
|
const SOURCE_SKIP_DIRS = new Set([
|
|
@@ -3415,6 +3876,22 @@ export interface SourceEvidence {
|
|
|
3415
3876
|
* One number, one walk, no arithmetic between them.
|
|
3416
3877
|
*/
|
|
3417
3878
|
unscannedSourceCount: number;
|
|
3879
|
+
/**
|
|
3880
|
+
* The PATHS behind that count — every source file the walk counted that
|
|
3881
|
+
* lives outside `CONFIG_SCAN_DIRS` (#506).
|
|
3882
|
+
*
|
|
3883
|
+
* `unscannedSourceCount` is now `unscannedFiles.length` rather than a
|
|
3884
|
+
* second tally incremented beside it. Two expressions answering one
|
|
3885
|
+
* question are one decider wearing two hats, and a mutation of either
|
|
3886
|
+
* leaves the other holding every case up — the fourth shape in
|
|
3887
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`.
|
|
3888
|
+
*
|
|
3889
|
+
* It exists because naming these files was never the end of the job.
|
|
3890
|
+
* CONFIG-1 refused over `docs/assets/search.js` while the file sat right
|
|
3891
|
+
* there, already located by this walk and already filtered by every rule
|
|
3892
|
+
* the walk applies. Reading it is strictly more than refusing to.
|
|
3893
|
+
*/
|
|
3894
|
+
unscannedFiles: string[];
|
|
3418
3895
|
/**
|
|
3419
3896
|
* Top-level directories holding source that `CONFIG_SCAN_DIRS` does not
|
|
3420
3897
|
* walk — `"."` for a source file at the root (#492).
|
|
@@ -3461,9 +3938,47 @@ const NEVER_PROJECT_SOURCE = new Set([".git", "node_modules"]);
|
|
|
3461
3938
|
* dangling-symlink note further down already rules out for the same reason.
|
|
3462
3939
|
* A link that resolves to nothing is not somewhere a reader can go looking,
|
|
3463
3940
|
* which is the only thing the trailing slash claims.
|
|
3941
|
+
*
|
|
3942
|
+
* ## The dirent is authoritative for everything that is not a symlink (#543)
|
|
3943
|
+
*
|
|
3944
|
+
* `if (!e.isSymbolicLink()) return false` is a COST guard, not a correctness
|
|
3945
|
+
* one — `statSync` on a regular file answers the same thing the dirent
|
|
3946
|
+
* already did, so removing it changes no label on any real tree and was
|
|
3947
|
+
* **0 red of 92**. What it changes is syscalls. Before #518 an entry named
|
|
3948
|
+
* `.git`, `node_modules`, `dist`, `build`, `coverage`, `.next`, `.output`,
|
|
3949
|
+
* `vendor` or any dot-name cost zero; this function made it cost one, and
|
|
3950
|
+
* those are exactly the names people symlink onto network paths — a shared
|
|
3951
|
+
* `node_modules`, a pnpm store, `dist` on a build mount. `statSync` on a
|
|
3952
|
+
* stale NFS or autofs target blocks uninterruptibly and there is no timeout
|
|
3953
|
+
* to give it.
|
|
3954
|
+
*
|
|
3955
|
+
* A cost is not observable from a test, so the guard is pinned by the one
|
|
3956
|
+
* thing that is: a dirent that DISAGREES with the filesystem. Hand this
|
|
3957
|
+
* function "not a directory, not a symlink" together with a path that is a
|
|
3958
|
+
* directory, and today it answers `false`; without the guard the stat runs
|
|
3959
|
+
* and it answers `true`. That is the syscall, made visible.
|
|
3960
|
+
*
|
|
3961
|
+
* **Two residuals, stated rather than silently carried:**
|
|
3962
|
+
*
|
|
3963
|
+
* - A declined SYMLINK still costs one `statSync`, and that is the stale-
|
|
3964
|
+
* mount exposure. It is kept deliberately: dropping it writes a symlinked
|
|
3965
|
+
* `node_modules` down as a file, which is the commonest real instance of
|
|
3966
|
+
* the thing #518 fixed. `readlinkSync` would resolve the name without
|
|
3967
|
+
* touching the target, but a monorepo's `node_modules -> ../../node_modules`
|
|
3968
|
+
* is outside the project and still a directory, so the cheap answer is
|
|
3969
|
+
* wrong for the case that motivates the stat.
|
|
3970
|
+
* - That stat crosses the project boundary without consulting
|
|
3971
|
+
* `insideProject`. The walk does not follow the link — all three branches
|
|
3972
|
+
* `continue` — so containment holds, verified over a 64-shape matrix; the
|
|
3973
|
+
* residual is that the waiver message asserts one bit about a path
|
|
3974
|
+
* outside the project, reachable only by someone who could already plant
|
|
3975
|
+
* a symlink inside it.
|
|
3464
3976
|
*/
|
|
3465
|
-
function declinedIsDir(e: Dirent, abs: string): boolean {
|
|
3977
|
+
export function declinedIsDir(e: Dirent, abs: string): boolean {
|
|
3466
3978
|
if (e.isDirectory()) return true;
|
|
3979
|
+
// Not a correctness branch — see the note above. A dirent that says
|
|
3980
|
+
// "regular file" is the answer; going to the filesystem to ask again is a
|
|
3981
|
+
// syscall on a path that may be a stale mount.
|
|
3467
3982
|
if (!e.isSymbolicLink()) return false;
|
|
3468
3983
|
try {
|
|
3469
3984
|
return statSync(abs).isDirectory();
|
|
@@ -3482,8 +3997,10 @@ const SCRIPT_LIKE = /\.(py|rb|sh|bash|zsh|fish|pl|php|lua|r|go|rs|java|kt|swift|
|
|
|
3482
3997
|
|
|
3483
3998
|
export function projectSourceEvidence(root: string): SourceEvidence {
|
|
3484
3999
|
const examples: string[] = [];
|
|
3485
|
-
// #492: the COUNT and the WHERE, alongside the capped sample.
|
|
3486
|
-
|
|
4000
|
+
// #492: the COUNT and the WHERE, alongside the capped sample. #506 made
|
|
4001
|
+
// the count a projection of the paths — one list, so a file cannot be
|
|
4002
|
+
// counted as unread and then left out of the list of what to read.
|
|
4003
|
+
const unscannedFiles: string[] = [];
|
|
3487
4004
|
const unscannedDirs = new Set<string>();
|
|
3488
4005
|
const unreadable: string[] = [];
|
|
3489
4006
|
const skipped: string[] = [];
|
|
@@ -3502,7 +4019,7 @@ export function projectSourceEvidence(root: string): SourceEvidence {
|
|
|
3502
4019
|
} catch {
|
|
3503
4020
|
return {
|
|
3504
4021
|
found: false, examples: [], unreadable: ["."], skipped: [],
|
|
3505
|
-
unscannedSourceCount: 0, unscannedDirs: [],
|
|
4022
|
+
unscannedSourceCount: 0, unscannedFiles: [], unscannedDirs: [],
|
|
3506
4023
|
};
|
|
3507
4024
|
}
|
|
3508
4025
|
const insideProject = (p: string) => p === rootReal || p.startsWith(rootReal + sep);
|
|
@@ -3693,9 +4210,29 @@ export function projectSourceEvidence(root: string): SourceEvidence {
|
|
|
3693
4210
|
// the scan does not walk either — `index.ts` is an ordinary shape
|
|
3694
4211
|
// and leaving it out would be the same blind spot one level up.
|
|
3695
4212
|
const top = childRel.includes("/") ? childRel.slice(0, childRel.indexOf("/")) : ".";
|
|
4213
|
+
// Source another CHECK measures is not evidence that this one is
|
|
4214
|
+
// blind (#501). `workflows/w.ts` set `found`, so a project whose
|
|
4215
|
+
// only code is a workflow was told CONFIG-1 "DOES contain source …
|
|
4216
|
+
// none of it in the directories this check reads" — true, and
|
|
4217
|
+
// unsatisfiable, because CONFIG-1 declines that directory precisely
|
|
4218
|
+
// so CONFIG-2 can own it. Two red checks, neither fixable without
|
|
4219
|
+
// rewriting the project in another language.
|
|
4220
|
+
//
|
|
4221
|
+
// `CONFIG_1_DECLINES` and NOT `measuredElsewhere`, and the gap
|
|
4222
|
+
// between the two is deliberate: `test/` is excluded by ADR-002 D4
|
|
4223
|
+
// with nobody measuring it, so a project whose only source is a test
|
|
4224
|
+
// directory must still refuse — that is #500's own case, and
|
|
4225
|
+
// collapsing the two predicates would waive it.
|
|
4226
|
+
//
|
|
4227
|
+
// Recorded rather than skipped silently, so the UNMEASURED sentence
|
|
4228
|
+
// names the hand-off instead of implying the tree is empty.
|
|
4229
|
+
if (CONFIG_1_DECLINES.has(top)) {
|
|
4230
|
+
skipped.push(`${childRel} (measured by CONFIG-2, which reads ${top}/)`);
|
|
4231
|
+
continue;
|
|
4232
|
+
}
|
|
3696
4233
|
if (!CONFIG_SCAN_DIRS.includes(top) && !measuredElsewhere(top)) {
|
|
3697
4234
|
unscannedDirs.add(top);
|
|
3698
|
-
|
|
4235
|
+
unscannedFiles.push(childRel);
|
|
3699
4236
|
}
|
|
3700
4237
|
if (examples.length < 3) examples.push(childRel);
|
|
3701
4238
|
}
|
|
@@ -3735,7 +4272,9 @@ export function projectSourceEvidence(root: string): SourceEvidence {
|
|
|
3735
4272
|
examples,
|
|
3736
4273
|
unreadable,
|
|
3737
4274
|
skipped,
|
|
3738
|
-
|
|
4275
|
+
// A projection of the list, not a tally kept beside it (#506).
|
|
4276
|
+
unscannedSourceCount: unscannedFiles.length,
|
|
4277
|
+
unscannedFiles,
|
|
3739
4278
|
unscannedDirs: [...unscannedDirs].sort(),
|
|
3740
4279
|
};
|
|
3741
4280
|
}
|
|
@@ -3851,8 +4390,16 @@ export function coverageShortfall(scan: ConfigAccessScan, evidence: SourceEviden
|
|
|
3851
4390
|
// where that entry already closed it. Both measured at 0 red before the
|
|
3852
4391
|
// boundary cases were added (sec-492).
|
|
3853
4392
|
if (scan.filesScanned === 0) return null; // the empty-scan refusal owns this case
|
|
3854
|
-
|
|
3855
|
-
|
|
4393
|
+
// Subtracted, not zeroed, and the difference is a statement: these are
|
|
4394
|
+
// the SAME files. `fallbackFiles` is sourced from `unscannedFiles`, which
|
|
4395
|
+
// is also what this count projects, so the subtraction is inside one walk
|
|
4396
|
+
// and cannot go negative (#506). Naming a file as unread one line after
|
|
4397
|
+
// reading it is the kind of false sentence #500 was filed about.
|
|
4398
|
+
//
|
|
4399
|
+
// It cannot silence the ordinary path: `fallbackFiles` is non-empty only
|
|
4400
|
+
// when the first scan read NOTHING, which the guard above already owns.
|
|
4401
|
+
const unread = evidence.unscannedSourceCount - scan.fallbackFiles.length;
|
|
4402
|
+
if (unread <= 0) return null;
|
|
3856
4403
|
const where = evidence.unscannedDirs.map(d => (d === "." ? "the project root" : `${d}/`)).join(", ");
|
|
3857
4404
|
return (
|
|
3858
4405
|
`UNMEASURED: CONFIG-1 read ${scan.filesScanned} file(s), and ${unread} more source ` +
|
|
@@ -3949,8 +4496,10 @@ export function runSuccessDocValidation(root: string) {
|
|
|
3949
4496
|
export function runConfigAccessValidation(root: string) {
|
|
3950
4497
|
describe("Config access (ADR-002 D1)", () => {
|
|
3951
4498
|
test("CONFIG-1: only the loader reads the rungate config", () => {
|
|
3952
|
-
const scan = scanConfigAccess(root);
|
|
3953
4499
|
const evidence = projectSourceEvidence(root);
|
|
4500
|
+
// #506: when the scan dirs hold nothing, read the source the walk
|
|
4501
|
+
// found anyway rather than refusing over a file nobody can move.
|
|
4502
|
+
const scan = scanConfigAccessWithReach(root, evidence);
|
|
3954
4503
|
const unmeasured = unmeasuredScanReason(
|
|
3955
4504
|
scan,
|
|
3956
4505
|
evidence,
|
|
@@ -4005,6 +4554,540 @@ export function runConfigAccessValidation(root: string) {
|
|
|
4005
4554
|
});
|
|
4006
4555
|
}
|
|
4007
4556
|
|
|
4557
|
+
// ── AIREVIEW-1: the generated ai-review job (#630) ──────────────────────
|
|
4558
|
+
//
|
|
4559
|
+
// `test/ai-review-scaffold.test.ts` proves the BUILDER produces a script that
|
|
4560
|
+
// annotates UNMEASURED. This proves the FILE ON DISK does — which is a
|
|
4561
|
+
// different claim, and the one a consumer's pipeline depends on. A generator
|
|
4562
|
+
// that is correct and a workflow file that drifted from it look identical
|
|
4563
|
+
// from the generator's side.
|
|
4564
|
+
|
|
4565
|
+
export { AI_REVIEW_JOB, AI_REVIEW_WORKFLOW_FILE } from "./scaffold/ai-review-step";
|
|
4566
|
+
|
|
4567
|
+
/** What `aiReviewAuthScript` could establish about a project's ai-review job. */
|
|
4568
|
+
export interface AiReviewReading {
|
|
4569
|
+
/** The bash the generated auth-detect step runs, or null when none was read. */
|
|
4570
|
+
script: string | null;
|
|
4571
|
+
/**
|
|
4572
|
+
* Non-null when this project is not one the check can reach a verdict on —
|
|
4573
|
+
* it has no scaffolded CI at all. UNMEASURED, said out loud, because `0`,
|
|
4574
|
+
* `{}`, `null` and `SKIP` are indistinguishable from success.
|
|
4575
|
+
*/
|
|
4576
|
+
unmeasured: string | null;
|
|
4577
|
+
/**
|
|
4578
|
+
* Non-null when the project IS scaffolded and the job is absent, renamed,
|
|
4579
|
+
* conditional, or unreadable.
|
|
4580
|
+
*
|
|
4581
|
+
* Absence is a FAILURE and not UNMEASURED on purpose. The cheapest repair
|
|
4582
|
+
* for a red AIREVIEW-1 is to delete `.github/workflows/ai-review.yml`, and a
|
|
4583
|
+
* check that goes green under its own cheapest wrong fix certifies the
|
|
4584
|
+
* regression it exists to block (.claude/rules/checks-must-be-able-to-fail.md,
|
|
4585
|
+
* the fifth shape).
|
|
4586
|
+
*/
|
|
4587
|
+
failure: string | null;
|
|
4588
|
+
/**
|
|
4589
|
+
* What the same parse found about Workload Identity Federation (#631).
|
|
4590
|
+
*
|
|
4591
|
+
* Read HERE rather than by a second parse, because the question
|
|
4592
|
+
* `aiReviewUnmeasuredViolations` has to answer about a federated consumer
|
|
4593
|
+
* is "does the job that resolved `mode=vertex` also carry the step that
|
|
4594
|
+
* mints the credential" — two readings of one file, and two parses would be
|
|
4595
|
+
* two deciders with nothing comparing them.
|
|
4596
|
+
*/
|
|
4597
|
+
federation: AiReviewFederation;
|
|
4598
|
+
}
|
|
4599
|
+
|
|
4600
|
+
/** The federation evidence in a generated ai-review job. */
|
|
4601
|
+
export interface AiReviewFederation {
|
|
4602
|
+
/** Whether the job may mint an OIDC token at all. */
|
|
4603
|
+
idTokenWrite: boolean;
|
|
4604
|
+
/** The `google-github-actions/auth` step, or null when the job has none. */
|
|
4605
|
+
authStep: Record<string, any> | null;
|
|
4606
|
+
}
|
|
4607
|
+
|
|
4608
|
+
/** No evidence read. The direction this has to be wrong in: it refuses. */
|
|
4609
|
+
const NO_FEDERATION: AiReviewFederation = { idTokenWrite: false, authStep: null };
|
|
4610
|
+
|
|
4611
|
+
/**
|
|
4612
|
+
* Read the auth-detect script out of a project's generated ai-review workflow.
|
|
4613
|
+
*
|
|
4614
|
+
* Parses rather than greps: the property is "GitHub will run this script in a
|
|
4615
|
+
* job that cannot report skipped", and only a parse can say that. A `# if:` in
|
|
4616
|
+
* a comment is not a condition and a condition reformatted across lines still
|
|
4617
|
+
* is one, so a text match is wrong in both directions.
|
|
4618
|
+
*/
|
|
4619
|
+
export function aiReviewAuthScript(root: string): AiReviewReading {
|
|
4620
|
+
const dir = join(root, ".github", "workflows");
|
|
4621
|
+
const path = join(dir, AI_REVIEW_WORKFLOW_FILE);
|
|
4622
|
+
const none = { script: null, unmeasured: null, failure: null, federation: NO_FEDERATION };
|
|
4623
|
+
|
|
4624
|
+
if (!existsSync(path)) {
|
|
4625
|
+
if (!existsSync(join(dir, "ci.yml"))) {
|
|
4626
|
+
return {
|
|
4627
|
+
...none,
|
|
4628
|
+
unmeasured: `UNMEASURED: this project has no .github/workflows/ci.yml, so it has ` +
|
|
4629
|
+
`no scaffolded CI and no ${AI_REVIEW_WORKFLOW_FILE} to execute. Run the scaffold ` +
|
|
4630
|
+
`to get one; until then nothing here says an AI review passed.`,
|
|
4631
|
+
};
|
|
4632
|
+
}
|
|
4633
|
+
return {
|
|
4634
|
+
...none,
|
|
4635
|
+
failure: `.github/workflows/${AI_REVIEW_WORKFLOW_FILE} is absent while ci.yml is present, ` +
|
|
4636
|
+
`so this project took a scaffold that writes it and no longer has it. ` +
|
|
4637
|
+
`A deleted review job is not a passing review — re-run the scaffold.`,
|
|
4638
|
+
};
|
|
4639
|
+
}
|
|
4640
|
+
|
|
4641
|
+
let doc: unknown;
|
|
4642
|
+
try {
|
|
4643
|
+
doc = Bun.YAML.parse(readFileSync(path, "utf-8"));
|
|
4644
|
+
} catch (e) {
|
|
4645
|
+
return { ...none, failure: `${AI_REVIEW_WORKFLOW_FILE} does not parse as YAML: ${(e as Error).message}` };
|
|
4646
|
+
}
|
|
4647
|
+
const jobs = (doc as any)?.jobs;
|
|
4648
|
+
if (!jobs || typeof jobs !== "object") {
|
|
4649
|
+
return { ...none, failure: `${AI_REVIEW_WORKFLOW_FILE} declares no jobs: mapping` };
|
|
4650
|
+
}
|
|
4651
|
+
const job = jobs[AI_REVIEW_JOB];
|
|
4652
|
+
if (!job || typeof job !== "object") {
|
|
4653
|
+
return {
|
|
4654
|
+
...none,
|
|
4655
|
+
failure: `${AI_REVIEW_WORKFLOW_FILE} has no job named ${AI_REVIEW_JOB} ` +
|
|
4656
|
+
`(found: ${Object.keys(jobs).join(", ") || "none"})`,
|
|
4657
|
+
};
|
|
4658
|
+
}
|
|
4659
|
+
if ("if" in job) {
|
|
4660
|
+
return {
|
|
4661
|
+
...none,
|
|
4662
|
+
failure: `the ${AI_REVIEW_JOB} job carries a job-level \`if:\` — a job skipped by a ` +
|
|
4663
|
+
`condition reports \`skipped\`, and a skipped required check can be treated as ` +
|
|
4664
|
+
`satisfied. Resolve the condition inside \`run:\` instead.`,
|
|
4665
|
+
};
|
|
4666
|
+
}
|
|
4667
|
+
const steps = Array.isArray(job.steps) ? job.steps : [];
|
|
4668
|
+
const conditional = steps.filter((s: any) => s && typeof s === "object" && "if" in s);
|
|
4669
|
+
if (conditional.length > 0) {
|
|
4670
|
+
return {
|
|
4671
|
+
...none,
|
|
4672
|
+
failure: `${conditional.length} step(s) in the ${AI_REVIEW_JOB} job carry an \`if:\`, ` +
|
|
4673
|
+
`which reports \`skipped\` rather than a verdict`,
|
|
4674
|
+
};
|
|
4675
|
+
}
|
|
4676
|
+
const auth = steps.find((s: any) => s && s.id === AI_REVIEW_AUTH_STEP_ID);
|
|
4677
|
+
if (!auth || typeof auth.run !== "string") {
|
|
4678
|
+
return {
|
|
4679
|
+
...none,
|
|
4680
|
+
failure: `${AI_REVIEW_WORKFLOW_FILE}'s ${AI_REVIEW_JOB} job has no \`run:\` step with ` +
|
|
4681
|
+
`id ${AI_REVIEW_AUTH_STEP_ID}, so nothing resolves a credential mode`,
|
|
4682
|
+
};
|
|
4683
|
+
}
|
|
4684
|
+
// The federation evidence, off the SAME parse (#631). `permissions:` is
|
|
4685
|
+
// legal at workflow level and at job level and the job's wins, so both are
|
|
4686
|
+
// read — a check that looked at only one would report a correctly granted
|
|
4687
|
+
// job as ungranted, or the reverse.
|
|
4688
|
+
const permissions = (job as any).permissions ?? (doc as any)?.permissions;
|
|
4689
|
+
const idTokenWrite =
|
|
4690
|
+
permissions === "write-all" ||
|
|
4691
|
+
(!!permissions && typeof permissions === "object" && (permissions as any)["id-token"] === "write");
|
|
4692
|
+
const authStep =
|
|
4693
|
+
steps.find((s: any) => s && typeof s === "object" && s.id === AI_REVIEW_VERTEX_AUTH_STEP_ID) ?? null;
|
|
4694
|
+
|
|
4695
|
+
return {
|
|
4696
|
+
script: auth.run,
|
|
4697
|
+
unmeasured: null,
|
|
4698
|
+
failure: null,
|
|
4699
|
+
federation: { idTokenWrite, authStep },
|
|
4700
|
+
};
|
|
4701
|
+
}
|
|
4702
|
+
|
|
4703
|
+
/** One execution of a generated script, under an environment built from scratch. */
|
|
4704
|
+
function runAiReviewScript(script: string, env: Record<string, string>): { code: number; stdout: string } {
|
|
4705
|
+
const dir = mkdtempSync(join(tmpdir(), "rungate-ai-review-"));
|
|
4706
|
+
try {
|
|
4707
|
+
const sh = join(dir, "auth-detect.sh");
|
|
4708
|
+
writeFileSync(sh, script);
|
|
4709
|
+
const out = join(dir, "github_output");
|
|
4710
|
+
writeFileSync(out, "");
|
|
4711
|
+
// Built from scratch, NOT inherited. A developer or a runner with
|
|
4712
|
+
// ANTHROPIC_API_KEY exported would make the credential-free case resolve a
|
|
4713
|
+
// mode, and the whole check would go green for a reason unrelated to the
|
|
4714
|
+
// code — test/ai-review-scaffold.test.ts makes the same argument.
|
|
4715
|
+
const r = spawnSync("bash", [sh], {
|
|
4716
|
+
encoding: "utf-8",
|
|
4717
|
+
env: { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: dir, GITHUB_OUTPUT: out, ...env },
|
|
4718
|
+
});
|
|
4719
|
+
return { code: r.status ?? -1, stdout: `${r.stdout ?? ""}${r.stderr ?? ""}` };
|
|
4720
|
+
} finally {
|
|
4721
|
+
rmSync(dir, { recursive: true, force: true });
|
|
4722
|
+
}
|
|
4723
|
+
}
|
|
4724
|
+
|
|
4725
|
+
/** Warning annotations naming UNMEASURED, which is the state AC-4 counts. */
|
|
4726
|
+
function unmeasuredWarnings(stdout: string): string[] {
|
|
4727
|
+
return stdout.split("\n").filter(l => l.startsWith("::warning") && l.includes("UNMEASURED"));
|
|
4728
|
+
}
|
|
4729
|
+
|
|
4730
|
+
/**
|
|
4731
|
+
* The claims that apply when the generated script resolves a credential with
|
|
4732
|
+
* no environment at all (#631).
|
|
4733
|
+
*
|
|
4734
|
+
* The credential-free claim is unreachable for this consumer, so this is the
|
|
4735
|
+
* claim that replaces it rather than the absence of one. `mode=vertex` is a
|
|
4736
|
+
* statement that a review WILL run, and a job that makes it while carrying no
|
|
4737
|
+
* step that could mint the credential is a green check for a review that
|
|
4738
|
+
* cannot authenticate — AIREVIEW-1's own subject, reached through the new
|
|
4739
|
+
* mode. **It is also the cheapest wrong fix: accept any resolved mode and
|
|
4740
|
+
* return early.** These are the assertions that repair would skip.
|
|
4741
|
+
*/
|
|
4742
|
+
function aiReviewGeneratedCredentialViolations(
|
|
4743
|
+
mode: string,
|
|
4744
|
+
bare: { code: number; stdout: string },
|
|
4745
|
+
federation: AiReviewFederation,
|
|
4746
|
+
): string[] {
|
|
4747
|
+
const out: string[] = [];
|
|
4748
|
+
|
|
4749
|
+
if (!(AI_REVIEW_MODES as readonly string[]).includes(mode)) {
|
|
4750
|
+
out.push(
|
|
4751
|
+
`the ai-review step resolved \`${mode}\`, which is not one of the ${AI_REVIEW_MODES.length} ` +
|
|
4752
|
+
`modes it declares (${AI_REVIEW_MODES.join(", ")})`,
|
|
4753
|
+
);
|
|
4754
|
+
}
|
|
4755
|
+
|
|
4756
|
+
const stillWarned = unmeasuredWarnings(bare.stdout);
|
|
4757
|
+
if (stillWarned.length !== 0) {
|
|
4758
|
+
out.push(
|
|
4759
|
+
`the ai-review step resolved \`${mode}\` from the generated file and still emitted ` +
|
|
4760
|
+
`${stillWarned.length} UNMEASURED annotation(s) — a configured review that reports ` +
|
|
4761
|
+
`itself unmeasured makes the word mean nothing\n${bare.stdout.trim()}`,
|
|
4762
|
+
);
|
|
4763
|
+
}
|
|
4764
|
+
if (bare.code !== 0) {
|
|
4765
|
+
out.push(`the ai-review step exited ${bare.code} with \`${mode}\` resolved from the generated file`);
|
|
4766
|
+
}
|
|
4767
|
+
|
|
4768
|
+
if (mode !== AI_REVIEW_MODE_VERTEX) {
|
|
4769
|
+
// `oauth` and `api-key` come from secrets, and this run had none. A
|
|
4770
|
+
// script that resolved one anyway is reading a credential from somewhere
|
|
4771
|
+
// nobody declared.
|
|
4772
|
+
out.push(
|
|
4773
|
+
`the ai-review step resolved \`${mode}\` with an EMPTY environment. That mode is ` +
|
|
4774
|
+
`supposed to come from a repository secret, so a run with no secrets set must not ` +
|
|
4775
|
+
`reach it — something in the generated file is standing in for a credential`,
|
|
4776
|
+
);
|
|
4777
|
+
return out;
|
|
4778
|
+
}
|
|
4779
|
+
|
|
4780
|
+
const step = federation.authStep;
|
|
4781
|
+
if (step === null) {
|
|
4782
|
+
out.push(
|
|
4783
|
+
`the ai-review step resolved \`${AI_REVIEW_MODE_VERTEX}\` and the ${AI_REVIEW_JOB} job has no ` +
|
|
4784
|
+
`\`${AI_REVIEW_VERTEX_AUTH_STEP_ID}\` step, so nothing in it mints the credential the mode ` +
|
|
4785
|
+
`claims. A review that cannot authenticate is UNMEASURED, and this one reports a mode`,
|
|
4786
|
+
);
|
|
4787
|
+
return out;
|
|
4788
|
+
}
|
|
4789
|
+
if (!federation.idTokenWrite) {
|
|
4790
|
+
out.push(
|
|
4791
|
+
`the ${AI_REVIEW_JOB} job federates to Vertex without \`id-token: write\`, so the runner ` +
|
|
4792
|
+
`cannot mint the OIDC token the exchange consumes and the job resolves a mode it will ` +
|
|
4793
|
+
`never hold`,
|
|
4794
|
+
);
|
|
4795
|
+
}
|
|
4796
|
+
const inputs = (step.with ?? {}) as Record<string, unknown>;
|
|
4797
|
+
if ("credentials_json" in inputs) {
|
|
4798
|
+
out.push(
|
|
4799
|
+
`the ${AI_REVIEW_VERTEX_AUTH_STEP_ID} step passes \`credentials_json\`, which writes a ` +
|
|
4800
|
+
`permanent service-account key into the workspace the review agent then reads (#677 ` +
|
|
4801
|
+
`AC-15). Federation exists so that key does not have to be anywhere`,
|
|
4802
|
+
);
|
|
4803
|
+
}
|
|
4804
|
+
for (const required of ["workload_identity_provider", "project_id"]) {
|
|
4805
|
+
if (!(required in inputs)) {
|
|
4806
|
+
out.push(
|
|
4807
|
+
`the ${AI_REVIEW_VERTEX_AUTH_STEP_ID} step declares no \`${required}\`, so the exchange ` +
|
|
4808
|
+
`has nothing to present and the resolved mode is a claim about a credential that ` +
|
|
4809
|
+
`cannot be obtained`,
|
|
4810
|
+
);
|
|
4811
|
+
}
|
|
4812
|
+
}
|
|
4813
|
+
|
|
4814
|
+
return out;
|
|
4815
|
+
}
|
|
4816
|
+
|
|
4817
|
+
/**
|
|
4818
|
+
* Execute a generated auth-detect script and report what it got wrong.
|
|
4819
|
+
*
|
|
4820
|
+
* Two runs, and the second one is what makes the first mean anything: a script
|
|
4821
|
+
* that annotated UNMEASURED unconditionally would satisfy the credential-free
|
|
4822
|
+
* case while telling a consumer nothing, so the word has to be absent when a
|
|
4823
|
+
* credential IS configured.
|
|
4824
|
+
*/
|
|
4825
|
+
export function aiReviewUnmeasuredViolations(
|
|
4826
|
+
script: string,
|
|
4827
|
+
federation: AiReviewFederation = NO_FEDERATION,
|
|
4828
|
+
): string[] {
|
|
4829
|
+
const out: string[] = [];
|
|
4830
|
+
|
|
4831
|
+
const bare = runAiReviewScript(script, {});
|
|
4832
|
+
|
|
4833
|
+
// Which claim applies is a READING, not an assumption (#631). Every fixture
|
|
4834
|
+
// this check ever had took its credential from a secret, so emptying the
|
|
4835
|
+
// environment really did produce the credential-free state. A consumer who
|
|
4836
|
+
// declares `ci.vertex` gets `mode=vertex` written into the file, no
|
|
4837
|
+
// environment can move it, and demanding an UNMEASURED annotation of that
|
|
4838
|
+
// tree is demanding a state it cannot enter.
|
|
4839
|
+
const resolved = /^ai-review auth mode: (.*)$/m.exec(bare.stdout);
|
|
4840
|
+
if (!resolved) {
|
|
4841
|
+
out.push(
|
|
4842
|
+
`the ai-review step printed no \`ai-review auth mode:\` line, so which credential it ` +
|
|
4843
|
+
`resolved cannot be read at all. A mode nobody can read is UNMEASURED, and the ` +
|
|
4844
|
+
`credential-free claims below are being applied to it unverified`,
|
|
4845
|
+
);
|
|
4846
|
+
}
|
|
4847
|
+
const mode = resolved ? resolved[1].trim() : AI_REVIEW_MODE_NONE;
|
|
4848
|
+
|
|
4849
|
+
if (mode !== AI_REVIEW_MODE_NONE) {
|
|
4850
|
+
return [...out, ...aiReviewGeneratedCredentialViolations(mode, bare, federation)];
|
|
4851
|
+
}
|
|
4852
|
+
|
|
4853
|
+
const warnings = unmeasuredWarnings(bare.stdout);
|
|
4854
|
+
if (warnings.length !== 1) {
|
|
4855
|
+
out.push(
|
|
4856
|
+
`with no credential configured the ai-review step emitted ${warnings.length} UNMEASURED ` +
|
|
4857
|
+
`warning annotation(s), expected exactly 1 — a tree with no credential has not PASSED ` +
|
|
4858
|
+
`an AI review, it has not HAD one\n${bare.stdout.trim()}`,
|
|
4859
|
+
);
|
|
4860
|
+
}
|
|
4861
|
+
for (const { secret } of AI_REVIEW_AUTH_PRECEDENCE) {
|
|
4862
|
+
if (warnings.length === 1 && !warnings[0].includes(secret)) {
|
|
4863
|
+
out.push(`the UNMEASURED annotation does not name ${secret}, so nobody can act on it`);
|
|
4864
|
+
}
|
|
4865
|
+
}
|
|
4866
|
+
if (bare.code !== 0) {
|
|
4867
|
+
out.push(
|
|
4868
|
+
`the ai-review step exited ${bare.code} with no credential and no ` +
|
|
4869
|
+
`${AI_REVIEW_REQUIRE_VAR} set — the honest default for a tree that has never ` +
|
|
4870
|
+
`configured one is an annotation, not a red pipeline on the first scaffold`,
|
|
4871
|
+
);
|
|
4872
|
+
}
|
|
4873
|
+
|
|
4874
|
+
const credentialed = runAiReviewScript(script, { [AI_REVIEW_AUTH_PRECEDENCE[0].secret]: "x" });
|
|
4875
|
+
const stillWarned = credentialed.stdout.split("\n").filter(l => l.includes("UNMEASURED"));
|
|
4876
|
+
if (stillWarned.length !== 0) {
|
|
4877
|
+
out.push(
|
|
4878
|
+
`with ${AI_REVIEW_AUTH_PRECEDENCE[0].secret} configured the step still emitted ` +
|
|
4879
|
+
`${stillWarned.length} UNMEASURED line(s) — an unconditional annotation makes the ` +
|
|
4880
|
+
`word mean nothing\n${credentialed.stdout.trim()}`,
|
|
4881
|
+
);
|
|
4882
|
+
}
|
|
4883
|
+
if (credentialed.code !== 0) {
|
|
4884
|
+
out.push(`the ai-review step exited ${credentialed.code} with a credential configured`);
|
|
4885
|
+
}
|
|
4886
|
+
|
|
4887
|
+
return out;
|
|
4888
|
+
}
|
|
4889
|
+
|
|
4890
|
+
/**
|
|
4891
|
+
* AIREVIEW-1 (#630) — a scaffolded project reports UNMEASURED rather than a
|
|
4892
|
+
* passing review when no credential is configured.
|
|
4893
|
+
*
|
|
4894
|
+
* FAILS; does not warn, for every state except "this project has no CI at
|
|
4895
|
+
* all". See `.claude/rules/config-access.md` — an absent credential is
|
|
4896
|
+
* UNMEASURED, and UNMEASURED is not a pass.
|
|
4897
|
+
*/
|
|
4898
|
+
export function runAiReviewValidation(root: string) {
|
|
4899
|
+
describe("AI review workflow (#630)", () => {
|
|
4900
|
+
test("AIREVIEW-1: no credential is UNMEASURED, and a credential is not", () => {
|
|
4901
|
+
const reading = aiReviewAuthScript(root);
|
|
4902
|
+
if (reading.unmeasured) {
|
|
4903
|
+
addFinding({
|
|
4904
|
+
ruleId: "AIREVIEW-1",
|
|
4905
|
+
severity: "WARN",
|
|
4906
|
+
file: `.github/workflows/${AI_REVIEW_WORKFLOW_FILE}`,
|
|
4907
|
+
message: reading.unmeasured,
|
|
4908
|
+
});
|
|
4909
|
+
return;
|
|
4910
|
+
}
|
|
4911
|
+
const violations = reading.failure
|
|
4912
|
+
? [reading.failure]
|
|
4913
|
+
: aiReviewUnmeasuredViolations(reading.script!, reading.federation);
|
|
4914
|
+
for (const v of violations) {
|
|
4915
|
+
addFinding({
|
|
4916
|
+
ruleId: "AIREVIEW-1",
|
|
4917
|
+
severity: "FAIL",
|
|
4918
|
+
file: `.github/workflows/${AI_REVIEW_WORKFLOW_FILE}`,
|
|
4919
|
+
message: v,
|
|
4920
|
+
fixCommand: "bunx rungate scaffold .",
|
|
4921
|
+
});
|
|
4922
|
+
}
|
|
4923
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
4924
|
+
});
|
|
4925
|
+
});
|
|
4926
|
+
}
|
|
4927
|
+
|
|
4928
|
+
/**
|
|
4929
|
+
* PROTECT-1 (#909) — the repo's protection does not serialise every PR.
|
|
4930
|
+
*
|
|
4931
|
+
* `required_status_checks.strict = true` makes merging one PR invalidate all
|
|
4932
|
+
* the others. On a swarm that is the whole throughput limit, and it is
|
|
4933
|
+
* invisible: every PR is individually green and `BEHIND`. See
|
|
4934
|
+
* `lib/branch-protection.ts` for the measurement and for why the remedy
|
|
4935
|
+
* differs by `owner.type`.
|
|
4936
|
+
*
|
|
4937
|
+
* FAILS; does not warn, when the serialisation is established — the reason
|
|
4938
|
+
* this is in RunGate rather than a doc is that three consumers hit it
|
|
4939
|
+
* independently and nobody wrote it down, and a rule that is documented but
|
|
4940
|
+
* not enforced is the failure mode
|
|
4941
|
+
* `.claude/rules/checks-must-be-able-to-fail.md` is about.
|
|
4942
|
+
*
|
|
4943
|
+
* WARNS, and says UNMEASURED out loud, for everything it could not read. The
|
|
4944
|
+
* protection endpoint needs admin and the default Actions token has none, so
|
|
4945
|
+
* UNMEASURED is the ordinary outcome in a consumer's CI; it must still never
|
|
4946
|
+
* be mistaken for clean.
|
|
4947
|
+
*
|
|
4948
|
+
* `probe` and `subject` are seams and not conveniences. The probe is the same
|
|
4949
|
+
* binding production calls through, so a test replacing it exercises this
|
|
4950
|
+
* function rather than a parallel path, and nothing here reaches the network
|
|
4951
|
+
* when it is replaced.
|
|
4952
|
+
*/
|
|
4953
|
+
export function runBranchProtectionValidation(
|
|
4954
|
+
root: string,
|
|
4955
|
+
opts: { probe?: ProtectionProbe; subject?: (root: string) => string | null } = {},
|
|
4956
|
+
) {
|
|
4957
|
+
const probe = opts.probe ?? githubProtectionProbe;
|
|
4958
|
+
const subject = opts.subject ?? protectionSubject;
|
|
4959
|
+
|
|
4960
|
+
describe("Branch protection (#909)", () => {
|
|
4961
|
+
test("PROTECT-1: merging one PR does not invalidate every other open PR", async () => {
|
|
4962
|
+
const slug = subject(root);
|
|
4963
|
+
if (!slug) {
|
|
4964
|
+
addFinding({
|
|
4965
|
+
ruleId: "PROTECT-1",
|
|
4966
|
+
severity: "WARN",
|
|
4967
|
+
file: "",
|
|
4968
|
+
message:
|
|
4969
|
+
"UNMEASURED: this tree's origin remote names no owner/name, so there is no repo to " +
|
|
4970
|
+
"read protection for. Nothing was checked, which is not the same as nothing being wrong.",
|
|
4971
|
+
});
|
|
4972
|
+
return;
|
|
4973
|
+
}
|
|
4974
|
+
|
|
4975
|
+
const facts = await probe(slug);
|
|
4976
|
+
// Read against the branch the FACTS are about. Defaulting to "main"
|
|
4977
|
+
// here would answer a question about a branch the repo may not protect.
|
|
4978
|
+
const pushCi = facts.branch
|
|
4979
|
+
? readPushCi(root, facts.contexts, facts.branch)
|
|
4980
|
+
: { onPush: [], missing: [], unmeasured: "UNMEASURED push-CI: the default branch is unknown, so no trigger could be matched against it." };
|
|
4981
|
+
const reading = readMergeSerialisation(facts, pushCi);
|
|
4982
|
+
|
|
4983
|
+
if (reading.verdict === "UNMEASURED") {
|
|
4984
|
+
addFinding({ ruleId: "PROTECT-1", severity: "WARN", file: "", message: reading.reason });
|
|
4985
|
+
return;
|
|
4986
|
+
}
|
|
4987
|
+
if (reading.verdict === "PASS") return;
|
|
4988
|
+
|
|
4989
|
+
const message = `${reading.reason}\n\nREMEDY: ${reading.remedy}`;
|
|
4990
|
+
addFinding({
|
|
4991
|
+
ruleId: "PROTECT-1",
|
|
4992
|
+
severity: "FAIL",
|
|
4993
|
+
file: "",
|
|
4994
|
+
message,
|
|
4995
|
+
// Not a command that fixes it. This check reads and refuses; a
|
|
4996
|
+
// fixCommand that mutated protection would be the scaffold silently
|
|
4997
|
+
// rewriting a repo's security settings, which #909 puts out of scope.
|
|
4998
|
+
fixCommand: `gh api repos/${facts.slug}/branches/${facts.branch ?? "main"}/protection -q .required_status_checks`,
|
|
4999
|
+
});
|
|
5000
|
+
// Widened deliberately: control flow has already narrowed `verdict` to
|
|
5001
|
+
// "FAIL" here, so the comparison typechecks as a contradiction and the
|
|
5002
|
+
// assertion reads as dead. It is not dead — it is what turns the
|
|
5003
|
+
// recorded finding into a red suite — and the cast keeps the assertion
|
|
5004
|
+
// written against the value rather than against the narrowing.
|
|
5005
|
+
expect(reading.verdict as string, message).toBe("PASS");
|
|
5006
|
+
});
|
|
5007
|
+
});
|
|
5008
|
+
}
|
|
5009
|
+
|
|
5010
|
+
/**
|
|
5011
|
+
* WORKFLOW-DRIFT-1 (#915) — a generated workflow still holds the bytes the
|
|
5012
|
+
* generator wrote.
|
|
5013
|
+
*
|
|
5014
|
+
* The consumer-facing end of #722's census, which has been able to tell a
|
|
5015
|
+
* hand-edited generated workflow from an untouched one since the day it
|
|
5016
|
+
* shipped and which nothing a consumer runs has ever asked. `lib/conformity.ts`
|
|
5017
|
+
* exported fifteen `run*` functions and none of them was a workflow drift
|
|
5018
|
+
* check, so a hand-edit to a stamped `ci.yml`, `gates.yml` or `ai-review.yml`
|
|
5019
|
+
* was detectable and undetected.
|
|
5020
|
+
*
|
|
5021
|
+
* FAILS; does not warn, for a file whose every byte is the generator's —
|
|
5022
|
+
* `bunx rungate scaffold .` reverts that edit and nothing says so first.
|
|
5023
|
+
* WARNs, naming the reason, for the two states that are not a pass and not a
|
|
5024
|
+
* verdict either: a scan with nothing stamped to compare, and a file in which
|
|
5025
|
+
* `ci.workflowOverrides` has made part of the content the consumer's. See
|
|
5026
|
+
* `lib/scaffold/workflow-drift.ts` for why the second of those cannot be
|
|
5027
|
+
* resolved from a whole-body digest.
|
|
5028
|
+
*
|
|
5029
|
+
* NOT-OWNED workflows pass and are named. A consumer's own CI is theirs; a
|
|
5030
|
+
* check that refused it would be deleted within a day.
|
|
5031
|
+
*/
|
|
5032
|
+
export function runWorkflowDriftValidation(root: string) {
|
|
5033
|
+
describe("Generated workflow drift (#915)", () => {
|
|
5034
|
+
test("WORKFLOW-DRIFT-1: a generated workflow holds the bytes the generator wrote", () => {
|
|
5035
|
+
const overrides = workflowOverrides(tryLoadRungateConfig(root)?.ci);
|
|
5036
|
+
const reading = workflowDrift(root, overrides);
|
|
5037
|
+
|
|
5038
|
+
// Every cause, named. "UNMEASURED" with no reason beside it is the
|
|
5039
|
+
// silence this check exists to replace.
|
|
5040
|
+
//
|
|
5041
|
+
// Reported, and then execution CONTINUES (#958). UNMEASURED and DRIFTED
|
|
5042
|
+
// are not alternatives: `workflowDrift` pushes `UNREADABLE_WORKFLOW` and
|
|
5043
|
+
// moves on per file, so one unreadable workflow raises the whole reading
|
|
5044
|
+
// while a DIFFERENT file in the same directory lands in `drifted`. This
|
|
5045
|
+
// used to `return` here, which threw that drift away and rendered "I
|
|
5046
|
+
// could not fully look" as "I looked and it was fine" — #921's shape,
|
|
5047
|
+
// and a green suite over drift the check had already found.
|
|
5048
|
+
//
|
|
5049
|
+
// Drift found is a failure whatever happened to the rest of the scan.
|
|
5050
|
+
// The partiality is reported ALONGSIDE it, naming what could not be
|
|
5051
|
+
// read, rather than instead of it. The two readings are independent and
|
|
5052
|
+
// are no longer sequenced.
|
|
5053
|
+
for (const reason of reading.unmeasuredReasons) {
|
|
5054
|
+
addFinding({ ruleId: "WORKFLOW-DRIFT-1", severity: "WARN", file: "", message: reason });
|
|
5055
|
+
}
|
|
5056
|
+
|
|
5057
|
+
for (const f of reading.consumerOwned) {
|
|
5058
|
+
addFinding({
|
|
5059
|
+
ruleId: "WORKFLOW-DRIFT-1",
|
|
5060
|
+
severity: "WARN",
|
|
5061
|
+
file: `.github/workflows/${f.name}`,
|
|
5062
|
+
message: f.detail,
|
|
5063
|
+
});
|
|
5064
|
+
}
|
|
5065
|
+
for (const name of reading.notOwned) {
|
|
5066
|
+
addFinding({
|
|
5067
|
+
ruleId: "WORKFLOW-DRIFT-1",
|
|
5068
|
+
severity: "WARN",
|
|
5069
|
+
file: `.github/workflows/${name}`,
|
|
5070
|
+
message:
|
|
5071
|
+
"NOT-OWNED — rungate did not write this workflow and does not manage it. " +
|
|
5072
|
+
"Reported, not touched: what happens to it is yours",
|
|
5073
|
+
});
|
|
5074
|
+
}
|
|
5075
|
+
|
|
5076
|
+
const violations = reading.drifted.map(f => `${f.name}: ${f.detail}`);
|
|
5077
|
+
for (const f of reading.drifted) {
|
|
5078
|
+
addFinding({
|
|
5079
|
+
ruleId: "WORKFLOW-DRIFT-1",
|
|
5080
|
+
severity: "FAIL",
|
|
5081
|
+
file: `.github/workflows/${f.name}`,
|
|
5082
|
+
message: f.detail,
|
|
5083
|
+
fixCommand: "bunx rungate scaffold .",
|
|
5084
|
+
});
|
|
5085
|
+
}
|
|
5086
|
+
expect(violations, violations.join("\n")).toEqual([]);
|
|
5087
|
+
});
|
|
5088
|
+
});
|
|
5089
|
+
}
|
|
5090
|
+
|
|
4008
5091
|
/**
|
|
4009
5092
|
* CHECK-2 — config access inside workflow prompt text (ADR-002 D4).
|
|
4010
5093
|
*
|
|
@@ -4022,7 +5105,7 @@ export function runPromptConfigAccessValidation(root: string) {
|
|
|
4022
5105
|
// at the point it was read (#476).
|
|
4023
5106
|
const unmeasured = unmeasuredScanReason(
|
|
4024
5107
|
scan,
|
|
4025
|
-
{ found: existsSync(join(root, "workflows")), examples: [], unreadable: [], skipped: [], unscannedSourceCount: 0, unscannedDirs: [] },
|
|
5108
|
+
{ found: existsSync(join(root, "workflows")), examples: [], unreadable: [], skipped: [], unscannedSourceCount: 0, unscannedFiles: [], unscannedDirs: [] },
|
|
4026
5109
|
"CONFIG-2",
|
|
4027
5110
|
"workflows/ directory",
|
|
4028
5111
|
);
|