rungate 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/rungate.ts +70 -0
- package/lib/aes-calculator.ts +66 -0
- package/lib/agent-audit.ts +139 -0
- package/lib/agent-brief-validation.ts +359 -0
- package/lib/agents-md-refs.ts +90 -0
- package/lib/bash-file-read.ts +197 -0
- package/lib/behavioral-cache.ts +78 -0
- package/lib/branch-cleanup.ts +234 -0
- package/lib/brief-context-parser.ts +35 -0
- package/lib/canary.ts +117 -0
- package/lib/check-registry.ts +226 -0
- package/lib/ci-checkout.ts +400 -0
- package/lib/cli-commands.ts +137 -0
- package/lib/compliance-backfill.ts +220 -0
- package/lib/compliance-report.ts +504 -0
- package/lib/compliance.ts +361 -0
- package/lib/config-loader.ts +255 -0
- package/lib/conformity.ts +1818 -131
- package/lib/container-provenance.ts +230 -0
- package/lib/create-brief.ts +150 -0
- package/lib/create-sc.ts +272 -0
- package/lib/deep-merge.ts +69 -0
- package/lib/dev-liveness.ts +214 -0
- package/lib/directive-extractor.ts +210 -0
- package/lib/distribution-standing.ts +113 -0
- package/lib/doc-hygiene.ts +238 -0
- package/lib/eval-criteria.ts +6 -0
- package/lib/evidence-portability.ts +123 -0
- package/lib/evidence-prevalidator.ts +390 -0
- package/lib/evidence-required.ts +150 -0
- package/lib/evidence-satisfiability.ts +288 -0
- package/lib/frontmatter.ts +163 -0
- package/lib/gap-scanner.ts +348 -0
- package/lib/gate-enforcement.ts +366 -0
- package/lib/generators/agent-briefs.ts +136 -0
- package/lib/generators/agents-md.ts +506 -0
- package/lib/generators/code-map.ts +60 -0
- package/lib/generators/types.ts +184 -0
- package/lib/git-ref-normalizer.ts +107 -0
- package/lib/github.ts +539 -0
- package/lib/harness-identity.ts +63 -0
- package/lib/hill-climb.ts +164 -0
- package/lib/instruction-effectiveness.ts +131 -0
- package/lib/issue-closure-audit.ts +225 -0
- package/lib/issue-create-override.ts +343 -0
- package/lib/js-top-level-code.ts +250 -0
- package/lib/liveness-contradiction.ts +149 -0
- package/lib/organize.ts +505 -0
- package/lib/paths.ts +197 -6
- package/lib/post-fix-verify.ts +157 -0
- package/lib/prior-branch.ts +489 -0
- package/lib/project-state-shape.ts +166 -0
- package/lib/promote-outputs.ts +156 -0
- package/lib/prove-label-gate.ts +35 -0
- package/lib/reachability.ts +302 -0
- package/lib/release-preflight.ts +186 -0
- package/lib/rule-health-pipeline.ts +264 -0
- package/lib/rule-health-trend.ts +161 -0
- package/lib/rule-provenance.ts +149 -0
- package/lib/rule-registry.ts +128 -0
- package/lib/rungate-schema.ts +114 -4
- package/lib/sc-claim-forms.ts +179 -0
- package/lib/sc-cover.ts +150 -0
- package/lib/sc-guard.ts +157 -0
- package/lib/sc-ids.ts +344 -0
- package/lib/scaffold/defaults.ts +165 -0
- package/lib/scaffold/managed-workflow.ts +370 -0
- package/lib/scaffold/owned-files.ts +265 -0
- package/lib/scaffold/steps.ts +2680 -0
- package/lib/scanner.ts +701 -0
- package/lib/screenshot-upload.ts +73 -0
- package/lib/secret-patterns.ts +112 -0
- package/lib/security-verdict.ts +616 -0
- package/lib/signal-phrases.ts +1 -0
- package/lib/spec-change-conformity.ts +80 -0
- package/lib/spec-files.ts +62 -0
- package/lib/spec-registry.ts +133 -0
- package/lib/spec-updater.ts +96 -0
- package/lib/stale-cleanup.ts +224 -0
- package/lib/stale-issue-scanner.ts +36 -0
- package/lib/strip-js-comments.ts +269 -0
- package/lib/suite-measurement.ts +228 -0
- package/lib/task-completion-checks.ts +177 -0
- package/lib/test-suite-lock.ts +1136 -0
- package/lib/transcript-checker.ts +1140 -0
- package/lib/validators/spec-validators.ts +171 -0
- package/lib/verdict-capture.ts +73 -0
- package/lib/verified-push.ts +358 -0
- package/lib/workflow-security.ts +174 -0
- package/lib/worktree-cleanup.ts +250 -0
- package/lib/worktree-collect.ts +283 -0
- package/lib/worktree-isolation.ts +82 -0
- package/package.json +36 -3
- package/prompts/ac-adversary.md +14 -2
- package/prompts/aditi.md +5 -0
- package/prompts/blast-radius.md +5 -0
- package/prompts/container-verify.md +5 -0
- package/prompts/discovery.md +4 -0
- package/prompts/environment.md +5 -0
- package/prompts/evidence-hierarchy.md +5 -0
- package/prompts/evidence-validator.md +11 -0
- package/prompts/marcus.md +5 -0
- package/prompts/prevention.md +5 -0
- package/prompts/prove-reproducer.md +12 -0
- package/prompts/quinn-decision-tree.md +5 -0
- package/prompts/quinn-ui-brief.md +5 -0
- package/prompts/quinn.md +5 -0
- package/prompts/rca.md +5 -0
- package/prompts/read-before-write.md +4 -0
- package/prompts/regression.md +5 -0
- package/prompts/serena.md +5 -0
- package/scripts/audit-specs.ts +464 -0
- package/scripts/convert-spec.ts +293 -0
- package/scripts/create-sc.ts +92 -0
- package/scripts/create-spec.ts +166 -0
- package/scripts/extract-constraints.ts +4 -3
- package/scripts/generate-code-map.ts +1 -1
- package/scripts/scaffold-project.ts +138 -1426
- package/scripts/sync-spec-tests.ts +211 -126
- package/.claude/agents/aditi.md +0 -47
- package/.claude/agents/marcus.md +0 -50
- package/.claude/agents/quinn.md +0 -58
- package/.claude/agents/rook.md +0 -47
- package/.claude/agents/serena.md +0 -46
- package/.claude/rungate.json +0 -11
- package/.fallowrc.json +0 -28
- package/.github/copilot-instructions.md +0 -20
- package/.rungate/conformity-findings.json +0 -648
- package/AGENTS.md +0 -106
- package/CODE-MAP.md +0 -50
- package/HARNESS.md +0 -88
- package/bun.lock +0 -26
- package/bunfig.toml +0 -2
- package/config/brief-policies.json +0 -28
- package/config/ceremony-profiles.json +0 -212
- package/config/consumer-cascade-map.json +0 -24
- package/config/doc-cascade-map.json +0 -24
- package/config/eval.json +0 -84
- package/config/profiles/account-plan.yaml +0 -36
- package/config/profiles/campaign.yaml +0 -94
- package/config/prove-evidence.schema.json +0 -65
- package/config/spec-policies.json +0 -36
- package/config/workflow-schema.json +0 -281
- package/gates/SCHEMA-GUIDE.md +0 -121
- package/gates/ac-quality.test.ts +0 -106
- package/gates/adversarial.test.ts +0 -274
- package/gates/brief-assembler.test.ts +0 -429
- package/gates/brief-assembler.ts +0 -334
- package/gates/bunfig.toml +0 -2
- package/gates/chain.test.ts +0 -91
- package/gates/e2e-smoke.test.ts +0 -234
- package/gates/error-classifier.test.ts +0 -249
- package/gates/error-classifier.ts +0 -89
- package/gates/orchestrator.test.ts +0 -950
- package/gates/orchestrator.ts +0 -569
- package/gates/preload.ts +0 -77
- package/gates/prompts/ac-adversary.md +0 -20
- package/gates/prompts/evidence-validator.md +0 -22
- package/gates/prompts/prove-reproducer.md +0 -53
- package/gates/prove.test.ts +0 -164
- package/gates/run-gate.ts +0 -936
- package/gates/schema-parity.test.ts +0 -197
- package/gates/schema.ts +0 -362
- package/gates/self-heal.test.ts +0 -511
- package/gates/self-heal.ts +0 -121
- package/gates/ship-orchestrator.test.ts +0 -379
- package/gates/ship-orchestrator.ts +0 -363
- package/gates/test-fixtures/broken-no-environments.json +0 -60
- package/gates/test-fixtures/valid-standard.json +0 -54
- package/gates/test-utils.ts +0 -43
- package/gates/witness.ts +0 -220
- package/gates/workflow.test.ts +0 -1053
- package/hooks/AgentBriefGuard.hook.ts +0 -586
- package/hooks/AgentVerdictCapture.hook.ts +0 -134
- package/hooks/AutoVerifyGate.hook.ts +0 -76
- package/hooks/GateEnforcement.hook.ts +0 -273
- package/hooks/IssueCloseGuard.hook.ts +0 -116
- package/hooks/MergeGuard.hook.ts +0 -55
- package/hooks/StaleTTLCleanup.hook.ts +0 -133
- package/hooks/VerifyPhaseLock.hook.ts +0 -81
- package/hooks/WorkflowStateGuard.hook.ts +0 -37
- package/hooks/lib/agentDetection.ts +0 -70
- package/hooks/lib/findWorkflow.ts +0 -101
- package/hooks/lib/parseStdin.ts +0 -37
- package/hooks/lib/paths.ts +0 -111
- package/hooks/lib/utils.ts +0 -81
- package/reference/migration-manifest.json +0 -554
- package/reference/prompts/agent-aditi.md +0 -38
- package/reference/prompts/agent-marcus.md +0 -38
- package/reference/prompts/agent-quinn.md +0 -38
- package/reference/prompts/agent-rook.md +0 -37
- package/reference/prompts/agent-serena.md +0 -37
- package/reference/prompts/council-skill.md +0 -170
- package/reference/prompts/goal-skill.md +0 -235
- package/reference/prompts/prove-skill.md +0 -283
- package/reference/prompts/quinn-standard.md +0 -246
- package/reference/prompts/ship-convergence.md +0 -65
- package/reference/prompts/ship-escalation.md +0 -43
- package/reference/prompts/ship-iteration.md +0 -382
- package/reference/prompts/ship-pending-patches.md +0 -246
- package/reference/prompts/ship-reference.md +0 -250
- package/reference/prompts/ship-skill.md +0 -347
- package/reference/prompts/ship-templates.md +0 -215
- package/reference/scripts/batch-status.sh +0 -68
- package/reference/scripts/chain-compat-check.sh +0 -67
- package/reference/scripts/context-audit.sh +0 -314
- package/reference/scripts/dead-export-check.sh +0 -61
- package/reference/scripts/generate-qc-tables.sh +0 -92
- package/reference/scripts/harness-init.sh +0 -111
- package/reference/scripts/pai-054-baseline-check.sh +0 -40
- package/reference/scripts/prove-smoke-test.sh +0 -112
- package/reference/scripts/setup-keychain-salt.sh +0 -12
- package/reference/scripts/skill-optimizer-friday.sh +0 -88
- package/reference/scripts/skill-qc-test.sh +0 -351
- package/reference/scripts/skill-synthesize-tonight.sh +0 -66
- package/reference/specs/HARNESS-EXTRACTION-SPEC.md +0 -263
- package/reference/specs/harness-v3-bash-deletion-plan.md +0 -65
- package/reference/specs/harness-v3-migration-plan.md +0 -188
- package/scripts/decision-reconcile.sh +0 -74
- package/scripts/decision-reconcile.ts +0 -152
- package/scripts/git-hooks/pre-commit +0 -76
- package/scripts/git-hooks/pre-push +0 -271
- package/scripts/harness-skill-check.sh +0 -52
- package/scripts/lib/gap-emit.sh +0 -65
- package/scripts/lib/jq-update.sh +0 -30
- package/scripts/lib/resolve-slug.sh +0 -30
- package/scripts/lib/verify-spec-compliance.sh +0 -102
- package/scripts/prove-backfill.sh +0 -87
- package/scripts/scaffold-rungate-config.ts +0 -91
- package/scripts/self-containment-check.sh +0 -109
- package/scripts/skill-qc-validator.sh +0 -402
- package/scripts/slug-resolver.sh +0 -14
- package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +0 -2493
- package/specs/BOOTSTRAP-TEST-PLAN.md +0 -571
- package/specs/HARNESS-GATES.md +0 -256
- package/specs/HARNESS-SKILL-CHAIN.md +0 -286
- package/specs/HARNESS-SKILL-CONTRACT.md +0 -93
- package/specs/HARNESS-STANDARD.md +0 -488
- package/specs/SPEC-TEMPLATE.md +0 -53
- package/specs/bootstrap-data-flow.png +0 -0
- package/specs/harness-automation-matrix.md +0 -314
- package/specs/harness-automation-matrix.rules.json +0 -64
- package/test/anti.test.ts +0 -80
- package/test/contract-negative.test.ts +0 -86
- package/test/contract.test.ts +0 -158
- package/test/external-deps.test.ts +0 -48
- package/test/fixtures/golden-project/.env.example +0 -3
- package/test/fixtures/golden-project/Makefile +0 -7
- package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +0 -23
- package/test/fixtures/golden-project/docs/setup-guide.md +0 -10
- package/test/fixtures/golden-project/package.json +0 -10
- package/test/fixtures/golden-project/specs/api-spec.md +0 -20
- package/test/fixtures/golden-project/src/auth.ts +0 -8
- package/test/fixtures/golden-project/src/db.ts +0 -11
- package/test/fixtures/golden-project/src/index.ts +0 -19
- package/test/fixtures/golden-project/src/utils/format.ts +0 -3
- package/test/fixtures/golden-project/test/auth.test.ts +0 -15
- package/test/fixtures/golden-project/tsconfig.json +0 -12
- package/test/meta-sc-coverage.test.ts +0 -159
- package/test/phase-0.test.ts +0 -846
- package/test/phase-1.test.ts +0 -617
- package/test/phase-2.test.ts +0 -620
- package/test/phase-3.test.ts +0 -99
- package/test/scaffold-conformity.test.ts +0 -8
- package/test/schema-canary.test.ts +0 -51
- package/test/spec-compliance-auto.test.ts +0 -64
- package/test/spec-compliance.test.ts +0 -295
- package/test/spec-discovery.test.ts +0 -68
- package/test/spec-drift.test.ts +0 -5
- package/test/structure.test.ts +0 -165
- package/test/unit/agent-brief-guard-ship.test.ts +0 -293
- package/test/unit/extract-constraints.test.ts +0 -145
- package/test/unit/issue-close-guard.test.ts +0 -204
- package/test/unit/scaffold-project.test.ts +0 -417
- package/test/unit/skill-cooccurrence.test.ts +0 -177
- package/test/unit/skill-sequence-logger.test.ts +0 -244
- package/test/unit/workflow-state-guard.test.ts +0 -95
- package/workflows/batch-ship.js +0 -295
- package/workflows/council.js +0 -481
- package/workflows/prove.js +0 -560
- package/workflows/ship.js +0 -789
- package/workflows/verify.js +0 -236
package/lib/conformity.ts
CHANGED
|
@@ -6,8 +6,17 @@
|
|
|
6
6
|
* runScaffoldConformity(import.meta.dir + "/..");
|
|
7
7
|
*/
|
|
8
8
|
import { describe, test, expect } from "bun:test";
|
|
9
|
-
import { existsSync, readFileSync, readdirSync } from "fs";
|
|
10
|
-
import {
|
|
9
|
+
import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "fs";
|
|
10
|
+
import { collectSpecFiles } from "./spec-files";
|
|
11
|
+
import { spawnSync } from "child_process";
|
|
12
|
+
import { join, dirname } from "path";
|
|
13
|
+
import { fileURLToPath } from "url";
|
|
14
|
+
import { deriveDirectoryName } from "../scripts/split-spec";
|
|
15
|
+
import { readFreshCache } from "./behavioral-cache";
|
|
16
|
+
import { tryLoadRungateConfig } from "./config-loader";
|
|
17
|
+
import { stripJsComments } from "./strip-js-comments";
|
|
18
|
+
import { parseScDeclarations } from "./sc-ids";
|
|
19
|
+
import { isHarnessItself } from "./harness-identity";
|
|
11
20
|
|
|
12
21
|
// ── Shared utilities ────────────────────────────────────────
|
|
13
22
|
|
|
@@ -15,13 +24,89 @@ export function parseFrontmatter(content: string): Record<string, string> | null
|
|
|
15
24
|
const match = content.match(/^---\n([\s\S]*?)\n---/);
|
|
16
25
|
if (!match) return null;
|
|
17
26
|
const fields: Record<string, string> = {};
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
27
|
+
const lines = match[1].split("\n");
|
|
28
|
+
for (let i = 0; i < lines.length; i++) {
|
|
29
|
+
const kv = lines[i].match(/^(\w[\w-]*):\s*(.+)$/);
|
|
30
|
+
if (kv) {
|
|
31
|
+
fields[kv[1]] = kv[2].trim();
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
// Block-style key — `tiers:` with its value on the indented lines below.
|
|
35
|
+
// Without this branch the key reads as ABSENT, so a presence check on a
|
|
36
|
+
// field that is plainly there (marcus.md's `tiers`, SC-423) fails against
|
|
37
|
+
// correct input. The value is the flattened block, which is enough for the
|
|
38
|
+
// presence and substring checks the conformity matchers make of it.
|
|
39
|
+
const blockKey = lines[i].match(/^(\w[\w-]*):\s*$/);
|
|
40
|
+
if (!blockKey) continue;
|
|
41
|
+
const block: string[] = [];
|
|
42
|
+
while (i + 1 < lines.length && /^\s+\S/.test(lines[i + 1])) {
|
|
43
|
+
block.push(lines[++i].trim());
|
|
44
|
+
}
|
|
45
|
+
fields[blockKey[1]] = block.join("; ");
|
|
21
46
|
}
|
|
22
47
|
return fields;
|
|
23
48
|
}
|
|
24
49
|
|
|
50
|
+
/**
|
|
51
|
+
* SC-286: Resolve and validate file paths - reject traversal and absolute paths
|
|
52
|
+
* Returns resolved path or null if path is unsafe
|
|
53
|
+
*/
|
|
54
|
+
export function resolveAndContain(root: string, target: string): string | null {
|
|
55
|
+
// Reject path traversal
|
|
56
|
+
if (target.includes("../")) return null;
|
|
57
|
+
// Reject absolute paths
|
|
58
|
+
if (target.startsWith("/")) return null;
|
|
59
|
+
// Allow normal relative paths (including those with dots in filenames like .rungate)
|
|
60
|
+
return join(root, target);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Resolve the actual .git directory, handling worktrees where .git is a file
|
|
65
|
+
* containing "gitdir: /path/to/real/gitdir".
|
|
66
|
+
*/
|
|
67
|
+
function resolveGitDir(root: string): string {
|
|
68
|
+
const dotGit = join(root, ".git");
|
|
69
|
+
if (!existsSync(dotGit)) return dotGit;
|
|
70
|
+
const stat = statSync(dotGit);
|
|
71
|
+
if (stat.isDirectory()) return dotGit;
|
|
72
|
+
// Worktree: .git is a file with "gitdir: <path>"
|
|
73
|
+
const content = readFileSync(dotGit, "utf-8").trim();
|
|
74
|
+
const match = content.match(/^gitdir:\s*(.+)$/);
|
|
75
|
+
if (match) {
|
|
76
|
+
const gitdir = match[1].trim();
|
|
77
|
+
// gitdir may be absolute or relative
|
|
78
|
+
return gitdir.startsWith("/") ? gitdir : join(root, gitdir);
|
|
79
|
+
}
|
|
80
|
+
return dotGit;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Resolve a file path that starts with .git/ through the worktree-aware git dir.
|
|
85
|
+
* For non-.git paths, returns the normal join(root, file).
|
|
86
|
+
*/
|
|
87
|
+
function resolveFilePath(root: string, file: string): string {
|
|
88
|
+
if (file.startsWith(".git/")) {
|
|
89
|
+
const gitDir = resolveGitDir(root);
|
|
90
|
+
// For worktrees, hooks live in the main repo's hooks dir, not the worktree gitdir.
|
|
91
|
+
// The worktree gitdir (e.g. .git/worktrees/name/) doesn't have hooks/.
|
|
92
|
+
// Walk up to the main .git dir if we're in a worktree subdir.
|
|
93
|
+
const suffix = file.slice(".git/".length); // e.g. "hooks/pre-commit"
|
|
94
|
+
const resolved = join(gitDir, suffix);
|
|
95
|
+
if (existsSync(resolved)) return resolved;
|
|
96
|
+
// Try the common dir (main repo .git) for shared resources like hooks
|
|
97
|
+
const commonDirFile = join(gitDir, "commondir");
|
|
98
|
+
if (existsSync(commonDirFile)) {
|
|
99
|
+
const commonDir = readFileSync(commonDirFile, "utf-8").trim();
|
|
100
|
+
const commonResolved = commonDir.startsWith("/")
|
|
101
|
+
? join(commonDir, suffix)
|
|
102
|
+
: join(gitDir, commonDir, suffix);
|
|
103
|
+
if (existsSync(commonResolved)) return commonResolved;
|
|
104
|
+
}
|
|
105
|
+
return resolved;
|
|
106
|
+
}
|
|
107
|
+
return join(root, file);
|
|
108
|
+
}
|
|
109
|
+
|
|
25
110
|
export { SIGNAL_PHRASE_PATTERNS } from "./signal-phrases";
|
|
26
111
|
|
|
27
112
|
// ── Structured findings (machine-readable for agents) ──────
|
|
@@ -51,6 +136,7 @@ export interface StalenessEntry {
|
|
|
51
136
|
const _findings: ConformityFinding[] = [];
|
|
52
137
|
const _candidates: ConstraintCandidate[] = [];
|
|
53
138
|
const _staleness: StalenessEntry[] = [];
|
|
139
|
+
let _behavioralCount = 0;
|
|
54
140
|
|
|
55
141
|
export function addFinding(finding: ConformityFinding): void {
|
|
56
142
|
_findings.push(finding);
|
|
@@ -80,6 +166,70 @@ export function clearFindings(): void {
|
|
|
80
166
|
_findings.length = 0;
|
|
81
167
|
_candidates.length = 0;
|
|
82
168
|
_staleness.length = 0;
|
|
169
|
+
_behavioralCount = 0;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function addBehavioralCount(count: number): void {
|
|
173
|
+
_behavioralCount += count;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export function getBehavioralCount(): number {
|
|
177
|
+
return _behavioralCount;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// ── Auto-flip: track passing SCs and flip checkboxes ────────
|
|
181
|
+
|
|
182
|
+
interface PassingSC {
|
|
183
|
+
id: string;
|
|
184
|
+
specPath: string;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const _passingSCs: PassingSC[] = [];
|
|
188
|
+
|
|
189
|
+
export function recordPassingSC(id: string, specPath: string): void {
|
|
190
|
+
_passingSCs.push({ id, specPath });
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export function getPassingSCs(): readonly PassingSC[] {
|
|
194
|
+
return _passingSCs;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* Flip `- [ ] SC-N` to `- [x] SC-N` for every SC recorded as passing in this
|
|
199
|
+
* process.
|
|
200
|
+
*
|
|
201
|
+
* DELIBERATELY HAS NO AUTOMATIC CALLER (#86). It used to run from an afterAll
|
|
202
|
+
* inside runScaffoldConformity, which made a test run mutate the spec files
|
|
203
|
+
* that the next run's test collection reads from — the suite alternated
|
|
204
|
+
* between 57 and 99 tests and reported 0 fail either way.
|
|
205
|
+
*
|
|
206
|
+
* Routine syncing is handled by scripts/sync-sc-status.ts via the `posttest`
|
|
207
|
+
* hook, which re-derives pass state independently. Call this only from an
|
|
208
|
+
* explicitly-invoked command, never from a test lifecycle hook.
|
|
209
|
+
*/
|
|
210
|
+
export function flipPassingCheckboxes(): number {
|
|
211
|
+
const byFile = new Map<string, Set<string>>();
|
|
212
|
+
for (const { id, specPath } of _passingSCs) {
|
|
213
|
+
if (!byFile.has(specPath)) byFile.set(specPath, new Set());
|
|
214
|
+
byFile.get(specPath)!.add(id);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
let flipped = 0;
|
|
218
|
+
for (const [specPath, ids] of byFile) {
|
|
219
|
+
if (!existsSync(specPath)) continue;
|
|
220
|
+
let content = readFileSync(specPath, "utf-8");
|
|
221
|
+
let changed = false;
|
|
222
|
+
for (const id of ids) {
|
|
223
|
+
const unchecked = `- [ ] ${id}:`;
|
|
224
|
+
if (content.includes(unchecked)) {
|
|
225
|
+
content = content.replace(unchecked, `- [x] ${id}:`);
|
|
226
|
+
changed = true;
|
|
227
|
+
flipped++;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (changed) writeFileSync(specPath, content);
|
|
231
|
+
}
|
|
232
|
+
return flipped;
|
|
83
233
|
}
|
|
84
234
|
|
|
85
235
|
export function writeFindingsReport(root: string): string {
|
|
@@ -94,6 +244,7 @@ export function writeFindingsReport(root: string): string {
|
|
|
94
244
|
failures: _findings.filter(f => f.severity === "FAIL").length,
|
|
95
245
|
warnings: _findings.filter(f => f.severity === "WARN").length,
|
|
96
246
|
candidateCount: _candidates.length,
|
|
247
|
+
behavioralCount: _behavioralCount,
|
|
97
248
|
staleCount: _staleness.length,
|
|
98
249
|
findings: _findings,
|
|
99
250
|
constraintCandidates: _candidates,
|
|
@@ -102,32 +253,115 @@ export function writeFindingsReport(root: string): string {
|
|
|
102
253
|
return reportPath;
|
|
103
254
|
}
|
|
104
255
|
|
|
105
|
-
interface ParsedSC {
|
|
256
|
+
export interface ParsedSC {
|
|
106
257
|
id: string;
|
|
107
258
|
statement: string;
|
|
108
259
|
specFile: string;
|
|
260
|
+
/**
|
|
261
|
+
* Whether the SC's checkbox is ticked — `- [x]` rather than `- [ ]`.
|
|
262
|
+
*
|
|
263
|
+
* This drives the whole binding: a CHECKED SC claims to be done, so it is
|
|
264
|
+
* run as a real assertion and may fail the suite; an UNCHECKED one is work
|
|
265
|
+
* not started, so it is emitted as `test.todo`. Optional only because
|
|
266
|
+
* ParsedSC is constructed by hand in a dozen call sites that predate the
|
|
267
|
+
* field; `extractSCs` always sets it, and the planner treats a missing
|
|
268
|
+
* value as unchecked.
|
|
269
|
+
*/
|
|
270
|
+
checked?: boolean;
|
|
109
271
|
}
|
|
110
272
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
273
|
+
/** A ParsedSC as produced by extractSCs, where the checkbox state is known. */
|
|
274
|
+
export interface ExtractedSC extends ParsedSC {
|
|
275
|
+
checked: boolean;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// ── Behavioral SC detection ───────────────────────────────
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Detects whether an SC has the (behavioral) suffix, indicating it requires
|
|
282
|
+
* runtime verification (agent sessions, transcripts, auditing) and cannot
|
|
283
|
+
* be tested by static file checks.
|
|
284
|
+
*/
|
|
285
|
+
export function isBehavioralSC(sc: ParsedSC): boolean {
|
|
286
|
+
return /\(behavioral\)\s*$/.test(sc.statement);
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Counts behavioral SCs in a list.
|
|
291
|
+
*/
|
|
292
|
+
export function countBehavioralSCs(scs: ParsedSC[]): number {
|
|
293
|
+
return scs.filter(isBehavioralSC).length;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Returns the routing annotation for behavioral SCs.
|
|
298
|
+
* Behavioral SCs are verified via SESSION-AUDIT-SPEC's two feedback loops,
|
|
299
|
+
* not via static conformity checks.
|
|
300
|
+
*/
|
|
301
|
+
export function getBehavioralRouting(sc: ParsedSC): string | null {
|
|
302
|
+
if (!isBehavioralSC(sc)) return null;
|
|
303
|
+
return "Verified via SESSION-AUDIT-SPEC § Two Feedback Loops (runtime behavioral check)";
|
|
119
304
|
}
|
|
120
305
|
|
|
121
|
-
|
|
122
|
-
|
|
306
|
+
/**
|
|
307
|
+
* Returns an AssertionFn for a behavioral SC when cached results exist.
|
|
308
|
+
* Returns null when no fresh cache entry exists (treated as unmatchable).
|
|
309
|
+
* Stale entries (>= 7 days old) return null so they're treated as unmatchable.
|
|
310
|
+
*/
|
|
311
|
+
export function behavioralPattern(sc: ParsedSC, cachePath: string): ((root: string) => void) | null {
|
|
312
|
+
const cache = readFreshCache(cachePath);
|
|
313
|
+
const entry = cache[sc.id];
|
|
314
|
+
if (!entry) return null;
|
|
315
|
+
|
|
316
|
+
return (_root: string) => {
|
|
317
|
+
if (!entry.passed) {
|
|
318
|
+
throw new Error(`Behavioral SC ${sc.id} failed: ${entry.evidence}`);
|
|
319
|
+
}
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Pull every success criterion out of a spec, in BOTH checkbox states.
|
|
325
|
+
*
|
|
326
|
+
* This used to be `/^- \[ \] (SC-\w+)/` — unchecked only. Every SC marked
|
|
327
|
+
* done was therefore invisible to the engine: 261 of them in this repo,
|
|
328
|
+
* none asserted, and the suite reported 0 fail because it never looked.
|
|
329
|
+
* Narrowing this pattern again is caught by the floor test in
|
|
330
|
+
* test/phase-1-5.test.ts.
|
|
331
|
+
*
|
|
332
|
+
* The pattern itself moved to `lib/sc-ids.ts` (#330). It was one of four
|
|
333
|
+
* copies in the tree, each with a different idea of what an id looks like:
|
|
334
|
+
* this one admitted `SC-\w+`, the coverage lookup and the uniqueness guard
|
|
335
|
+
* only `SC-\d+`, so an anti-criterion was a declaration to this reader and
|
|
336
|
+
* invisible to the other two.
|
|
337
|
+
*/
|
|
338
|
+
export function extractSCs(content: string, specFile: string): ExtractedSC[] {
|
|
339
|
+
return parseScDeclarations(content, specFile);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
export interface SpecMetadata {
|
|
343
|
+
scs: ExtractedSC[];
|
|
344
|
+
compliance: "strict" | "permissive";
|
|
345
|
+
status: "active" | "draft";
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
export function collectTestableSpecs(root: string, extraSpecDirs?: string[]): Map<string, SpecMetadata> {
|
|
349
|
+
const specMap = new Map<string, SpecMetadata>();
|
|
123
350
|
|
|
124
351
|
const localSpecs = join(root, "specs");
|
|
125
352
|
if (existsSync(localSpecs)) {
|
|
126
|
-
for (const f of
|
|
353
|
+
for (const f of collectSpecFiles(localSpecs)) {
|
|
127
354
|
const content = readFileSync(join(localSpecs, f), "utf-8");
|
|
128
355
|
const fm = parseFrontmatter(content);
|
|
129
356
|
if (fm?.testable === "true") {
|
|
130
|
-
|
|
357
|
+
const compliance = fm?.compliance === "permissive" ? "permissive" : "strict";
|
|
358
|
+
const scs = extractSCs(content, f);
|
|
359
|
+
const hasCheckedSC = /- \[x\] SC-/i.test(content);
|
|
360
|
+
specMap.set(f, {
|
|
361
|
+
scs,
|
|
362
|
+
compliance,
|
|
363
|
+
status: hasCheckedSC ? "active" : "draft",
|
|
364
|
+
});
|
|
131
365
|
}
|
|
132
366
|
}
|
|
133
367
|
}
|
|
@@ -140,38 +374,349 @@ function collectTestableSpecs(root: string, extraSpecDirs?: string[]): ParsedSC[
|
|
|
140
374
|
if (fm?.testable !== "true") continue;
|
|
141
375
|
const governs = fm?.governs || "";
|
|
142
376
|
if (!governs.toLowerCase().includes("scaffold") && !governs.toLowerCase().includes("universal")) continue;
|
|
143
|
-
|
|
377
|
+
const compliance = fm?.compliance === "permissive" ? "permissive" : "strict";
|
|
378
|
+
const hasCheckedSC = /- \[x\] SC-/i.test(content);
|
|
379
|
+
const specFile = `${dir.split("/").pop()}/${f}`;
|
|
380
|
+
specMap.set(specFile, {
|
|
381
|
+
scs: extractSCs(content, specFile),
|
|
382
|
+
compliance,
|
|
383
|
+
status: hasCheckedSC ? "active" : "draft",
|
|
384
|
+
});
|
|
144
385
|
}
|
|
145
386
|
}
|
|
146
387
|
|
|
147
|
-
return
|
|
388
|
+
return specMap;
|
|
148
389
|
}
|
|
149
390
|
|
|
150
391
|
// ── Pattern matchers ────────────────────────────────────────
|
|
151
392
|
|
|
152
393
|
type AssertionFn = (root: string) => void;
|
|
394
|
+
type MatcherHandler = (sc: ParsedSC, match: RegExpMatchArray) => AssertionFn | null;
|
|
395
|
+
|
|
396
|
+
// Config loading with caching
|
|
397
|
+
const _registryCache = new Map<string, Array<{ name: string; regex: string }>>();
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Project-owned conformity exemptions, read from
|
|
401
|
+
* `<root>/.claude/conformity-allowlists.json`.
|
|
402
|
+
*
|
|
403
|
+
* A file rather than a constant because the callers that need it differ in
|
|
404
|
+
* editability: test/scaffold-conformity.test.ts is harness-managed and
|
|
405
|
+
* regenerated on re-scaffold, so a list written into it would be silently
|
|
406
|
+
* wiped. Keying on `root` is also what keeps a consumer from inheriting
|
|
407
|
+
* rungate's exemptions, which is the defect this replaced.
|
|
408
|
+
*
|
|
409
|
+
* Absent or unreadable means no exemptions adopted, and the checks report
|
|
410
|
+
* advisory rather than failing. That is deliberate: before #80 they could not
|
|
411
|
+
* fail at all, so a missing file must not break an existing consumer.
|
|
412
|
+
*/
|
|
413
|
+
function loadConformityAllowlists(root: string): {
|
|
414
|
+
untestedSpecs?: string[];
|
|
415
|
+
orphanFiles?: string[];
|
|
416
|
+
} {
|
|
417
|
+
try {
|
|
418
|
+
const p = join(root, ".claude", "conformity-allowlists.json");
|
|
419
|
+
if (!existsSync(p)) return {};
|
|
420
|
+
const parsed = JSON.parse(readFileSync(p, "utf-8"));
|
|
421
|
+
return {
|
|
422
|
+
untestedSpecs: Array.isArray(parsed.untestedSpecs) ? parsed.untestedSpecs : undefined,
|
|
423
|
+
orphanFiles: Array.isArray(parsed.orphanFiles) ? parsed.orphanFiles : undefined,
|
|
424
|
+
};
|
|
425
|
+
} catch {
|
|
426
|
+
return {};
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/**
|
|
431
|
+
* The exemption lists in a conformity-allowlists file, keyed by list name.
|
|
432
|
+
*
|
|
433
|
+
* Every array-valued key is read, not just the two `loadConformityAllowlists`
|
|
434
|
+
* consumes. A list added later must be covered by the growth check the day it
|
|
435
|
+
* appears, not the day someone remembers to add it here — the opposite
|
|
436
|
+
* arrangement is how an exemption mechanism acquires an unwatched corner.
|
|
437
|
+
*/
|
|
438
|
+
export function allowlistEntries(json: string): Record<string, string[]> {
|
|
439
|
+
const parsed = JSON.parse(json);
|
|
440
|
+
const lists: Record<string, string[]> = {};
|
|
441
|
+
for (const [key, value] of Object.entries(parsed)) {
|
|
442
|
+
if (Array.isArray(value)) lists[key] = value.map(String);
|
|
443
|
+
}
|
|
444
|
+
return lists;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* Entries present in `headJson` and absent from `baseJson` (#302).
|
|
449
|
+
*
|
|
450
|
+
* The existing ratchet enforces STALENESS — an entry that stops being a
|
|
451
|
+
* violation fails the build — and that half is real. It does not enforce
|
|
452
|
+
* GROWTH: a new entry for a genuinely new violation is not stale, so every
|
|
453
|
+
* check passes. An agent could introduce a violation and exempt it in the
|
|
454
|
+
* same commit, which is the one move this harness exists to prevent — an
|
|
455
|
+
* agent must not be able to pass a gate by editing the gate.
|
|
456
|
+
*
|
|
457
|
+
* Direction is the whole point. REMOVALS always pass: an agent that fixes a
|
|
458
|
+
* violation and delists it is the case the ratchet is for, and refusing that
|
|
459
|
+
* would trade one outage for another. Only additions are reported.
|
|
460
|
+
*
|
|
461
|
+
* Comparing against `origin/main` rather than the merge base is deliberate
|
|
462
|
+
* and safe in the direction that matters: entries added on main that a branch
|
|
463
|
+
* does not carry read as REMOVALS and pass, so the base can be stale without
|
|
464
|
+
* producing a false refusal.
|
|
465
|
+
*/
|
|
466
|
+
export function allowlistAdditions(baseJson: string, headJson: string): string[] {
|
|
467
|
+
const base = allowlistEntries(baseJson);
|
|
468
|
+
const head = allowlistEntries(headJson);
|
|
469
|
+
const added: string[] = [];
|
|
470
|
+
for (const [list, entries] of Object.entries(head)) {
|
|
471
|
+
const before = new Set(base[list] ?? []);
|
|
472
|
+
for (const entry of entries) {
|
|
473
|
+
if (!before.has(entry)) added.push(`${list}: ${entry}`);
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
return added;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/**
|
|
480
|
+
* Report a check that did not run, rather than returning silently.
|
|
481
|
+
*
|
|
482
|
+
* A bare `return` on a missing directory reads as a pass in every report, which
|
|
483
|
+
* is the same failure as the `expect(true).toBe(true)` bodies removed in #80 —
|
|
484
|
+
* the check contributes a green tick while verifying nothing. Wording matches
|
|
485
|
+
* the fallow skips so skipped checks are greppable as one class.
|
|
486
|
+
*/
|
|
487
|
+
function checkSkipped(check: string, reason: string): void {
|
|
488
|
+
console.warn(`SKIP ${check}: ${reason} — this check did NOT run.`);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
function loadBuiltInRegistry(): Array<{ name: string; regex: string }> {
|
|
492
|
+
const key = "__builtin__";
|
|
493
|
+
if (_registryCache.has(key)) return _registryCache.get(key)!;
|
|
494
|
+
const configPath = join(dirname(fileURLToPath(import.meta.url)), "..", "config", "matcher-registry.json");
|
|
495
|
+
const entries: Array<{ name: string; regex: string }> = JSON.parse(readFileSync(configPath, "utf-8"));
|
|
496
|
+
_registryCache.set(key, entries);
|
|
497
|
+
return entries;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
/**
|
|
501
|
+
* Load the matcher registry, optionally merging consumer-defined custom matchers.
|
|
502
|
+
*
|
|
503
|
+
* When projectRoot is provided, looks for .claude/rungate.json in that directory
|
|
504
|
+
* and merges any `customMatchers` array entries with the built-in registry.
|
|
505
|
+
* Consumer entries with the same `name` as a built-in entry override the built-in.
|
|
506
|
+
*/
|
|
507
|
+
/**
|
|
508
|
+
* `allowBuiltInOverride` is the difference between the AUTHORING view and the
|
|
509
|
+
* VERIFICATION view of the same config, and the two must not be the same.
|
|
510
|
+
*
|
|
511
|
+
* A consumer may name a custom matcher after a built-in, and `create-spec`
|
|
512
|
+
* honours it — that is a documented feature and extending the vocabulary is
|
|
513
|
+
* the point of custom matchers. But the conformity engine is verifying the
|
|
514
|
+
* very project that wrote that config, and `.claude/rungate.json` shadowing
|
|
515
|
+
* `content-contains` with a regex that matches nothing would switch the
|
|
516
|
+
* check off from inside the subject under test. The run would stay green and
|
|
517
|
+
* report nothing unusual, which is the defect class this repo exists to
|
|
518
|
+
* catch, arriving through its own extension point.
|
|
519
|
+
*
|
|
520
|
+
* So the engine takes `allowBuiltInOverride: false`: a consumer may ADD a
|
|
521
|
+
* matcher, never REPLACE one. A shadowing entry is dropped and named, rather
|
|
522
|
+
* than dropped quietly, because a consumer who meant it needs to know it did
|
|
523
|
+
* not take effect.
|
|
524
|
+
*
|
|
525
|
+
* This only became reachable when the runner started passing its project root
|
|
526
|
+
* (#212). Before that the engine ignored consumer config entirely, which is
|
|
527
|
+
* why the asymmetry had never had to be stated.
|
|
528
|
+
*/
|
|
529
|
+
function loadRegistry(
|
|
530
|
+
projectRoot?: string,
|
|
531
|
+
opts?: { allowBuiltInOverride?: boolean },
|
|
532
|
+
): Array<{ name: string; regex: string }> {
|
|
533
|
+
const allowOverride = opts?.allowBuiltInOverride ?? true;
|
|
534
|
+
const cacheKey = `${projectRoot ?? "__builtin__"}::${allowOverride ? "permissive" : "strict"}`;
|
|
535
|
+
if (_registryCache.has(cacheKey)) return _registryCache.get(cacheKey)!;
|
|
536
|
+
|
|
537
|
+
const builtIn = loadBuiltInRegistry();
|
|
538
|
+
if (!projectRoot) {
|
|
539
|
+
return builtIn;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
// Look for consumer custom matchers in .claude/rungate.json
|
|
543
|
+
const configPath = join(projectRoot, ".claude", "rungate.json");
|
|
544
|
+
if (!existsSync(configPath)) {
|
|
545
|
+
_registryCache.set(cacheKey, builtIn);
|
|
546
|
+
return builtIn;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
let customMatchers: Array<{ name: string; regex: string }>;
|
|
550
|
+
try {
|
|
551
|
+
const config = JSON.parse(readFileSync(configPath, "utf-8"));
|
|
552
|
+
customMatchers = config.customMatchers ?? [];
|
|
553
|
+
} catch (err) {
|
|
554
|
+
// Falling back to the built-ins is right — one consumer's broken config
|
|
555
|
+
// must not take the engine's own matchers down with it. Doing it in
|
|
556
|
+
// silence was not: every SC written against a custom matcher then reads
|
|
557
|
+
// as unrecognised, with nothing anywhere saying the config failed to
|
|
558
|
+
// parse (#212).
|
|
559
|
+
//
|
|
560
|
+
// The error's TYPE is reported and its message is not. A parse failure's
|
|
561
|
+
// message routinely quotes the bytes around the offending position, and
|
|
562
|
+
// this file is a project's own config — echoing its contents into build
|
|
563
|
+
// logs is how a secret in a misplaced field reaches somewhere it was
|
|
564
|
+
// never meant to go.
|
|
565
|
+
console.warn(
|
|
566
|
+
`WARN: ${configPath} could not be read as JSON (${err instanceof Error ? err.name : "error"}; ` +
|
|
567
|
+
`message withheld — it can quote file contents). ` +
|
|
568
|
+
`Falling back to the built-in matcher registry — any customMatchers in that file are NOT loaded, ` +
|
|
569
|
+
`so SCs written against them will be reported as having no matcher.`,
|
|
570
|
+
);
|
|
571
|
+
_registryCache.set(cacheKey, builtIn);
|
|
572
|
+
return builtIn;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
if (customMatchers.length === 0) {
|
|
576
|
+
_registryCache.set(cacheKey, builtIn);
|
|
577
|
+
return builtIn;
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
const builtInNames = new Set(builtIn.map((e) => e.name));
|
|
581
|
+
const accepted: Array<{ name: string; regex: string }> = [];
|
|
582
|
+
for (const candidate of customMatchers) {
|
|
583
|
+
if (!allowOverride && builtInNames.has(candidate.name)) {
|
|
584
|
+
console.warn(
|
|
585
|
+
`WARN: ${configPath} defines a custom matcher named "${candidate.name}", which is a built-in. ` +
|
|
586
|
+
`A project cannot replace a built-in matcher while being verified by it — the entry is IGNORED. ` +
|
|
587
|
+
`Rename it to extend the vocabulary instead.`,
|
|
588
|
+
);
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
// Compiled here rather than at use: a regex that cannot compile would
|
|
592
|
+
// otherwise throw from inside the matcher loop, taking down a run over
|
|
593
|
+
// one bad consumer entry. The entry is refused; the rest still load.
|
|
594
|
+
try {
|
|
595
|
+
new RegExp(candidate.regex, "i");
|
|
596
|
+
} catch {
|
|
597
|
+
console.warn(
|
|
598
|
+
`WARN: ${configPath} custom matcher "${candidate.name}" has a regex that does not compile — the entry is IGNORED.`,
|
|
599
|
+
);
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
accepted.push(candidate);
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
// Built-ins first. In the permissive view a shadowing entry has already
|
|
606
|
+
// displaced its built-in; in the strict view it was dropped above, so
|
|
607
|
+
// ordering alone cannot let a consumer entry pre-empt a built-in check.
|
|
608
|
+
const merged = allowOverride
|
|
609
|
+
? [...builtIn.filter((e) => !accepted.some((c) => c.name === e.name)), ...accepted]
|
|
610
|
+
: [...builtIn, ...accepted];
|
|
153
611
|
|
|
154
|
-
|
|
155
|
-
|
|
612
|
+
_registryCache.set(cacheKey, merged);
|
|
613
|
+
return merged;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
// ── Compound success criteria (#220) ────────────────────────
|
|
617
|
+
//
|
|
618
|
+
// `content-contains` captures exactly one bracket list and `matchPattern`
|
|
619
|
+
// returns on the first registry hit, so "FILE contains [a] and not contains
|
|
620
|
+
// [b]" bound, asserted [a], and discarded the rest with no signal. 21 ticked
|
|
621
|
+
// SCs were in that state; three of them asserted something false and passed
|
|
622
|
+
// anyway. The sweep is in #220.
|
|
623
|
+
//
|
|
624
|
+
// Two rules the parser follows, both of which a narrower reading gets wrong:
|
|
625
|
+
//
|
|
626
|
+
// - An SC's requirement ends where its explanation begins. Everything from
|
|
627
|
+
// the first " — " is rationale, and rationale saying "the reader and the
|
|
628
|
+
// writer agree" must not become a clause.
|
|
629
|
+
// - The defect is NOT about negation. Two of the 21 carry a POSITIVE second
|
|
630
|
+
// clause naming a different file, and a fix written in terms of
|
|
631
|
+
// "not contains" would leave those just as unasserted.
|
|
632
|
+
|
|
633
|
+
export type SCClause = { file: string; items: string[]; negated: boolean };
|
|
634
|
+
|
|
635
|
+
const CLAUSE_HEAD = /^(\S+)\s+contains?\s+\[([^\]]+)\]/;
|
|
636
|
+
const CLAUSE_TAIL =
|
|
637
|
+
/^\s+and\s+(?:(\S*[./]\S*)\s+)?(not\s+contains?|does\s+not\s+contain|must\s+NOT\s+contain|has\s+no|contains?)\s+\[([^\]]+)\]/i;
|
|
638
|
+
|
|
639
|
+
/** Rationale is commentary, not requirement. */
|
|
640
|
+
const requirementOf = (statement: string) => statement.split(" — ")[0].trimEnd();
|
|
641
|
+
|
|
642
|
+
const splitItems = (raw: string) => raw.split(",").map(i => i.trim()).filter(Boolean);
|
|
643
|
+
|
|
644
|
+
const isCodeFile = (file: string) => /\.(?:ts|tsx|js|jsx|mjs|cjs)$/.test(file);
|
|
156
645
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
646
|
+
/**
|
|
647
|
+
* Parse a contains-family statement into every clause it states.
|
|
648
|
+
* Returns null when the statement is not compound, so callers fall through to
|
|
649
|
+
* the single-clause matcher rather than this becoming a second way to bind.
|
|
650
|
+
*/
|
|
651
|
+
export function parseCompoundClauses(statement: string): SCClause[] | null {
|
|
652
|
+
const body = requirementOf(statement);
|
|
653
|
+
const head = CLAUSE_HEAD.exec(body);
|
|
654
|
+
if (!head) return null;
|
|
655
|
+
|
|
656
|
+
const headFile = head[1].replace(/`/g, "");
|
|
657
|
+
const clauses: SCClause[] = [{ file: headFile, items: splitItems(head[2]), negated: false }];
|
|
658
|
+
|
|
659
|
+
let end = head[0].length;
|
|
660
|
+
for (;;) {
|
|
661
|
+
const tail = CLAUSE_TAIL.exec(body.slice(end));
|
|
662
|
+
if (!tail) break;
|
|
663
|
+
end += tail[0].length;
|
|
664
|
+
const verb = tail[2].toLowerCase();
|
|
665
|
+
clauses.push({
|
|
666
|
+
file: (tail[1] ?? headFile).replace(/`/g, ""),
|
|
667
|
+
items: splitItems(tail[3]),
|
|
668
|
+
negated: /\bnot\b|has\s+no/.test(verb),
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
return clauses.length > 1 ? clauses : null;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* The guard's view of its own input: text a contains-family statement states
|
|
676
|
+
* but no clause consumed. A leftover bracket list is a requirement that would
|
|
677
|
+
* be silently dropped, so it is named rather than discarded.
|
|
678
|
+
*
|
|
679
|
+
* Prose residue is deliberately NOT refused here. Around eight SCs carry real
|
|
680
|
+
* requirements as prose ("and is under 3,000 tokens"); refusing those turns 39
|
|
681
|
+
* SCs malformed at once, and rewriting them into testable form is #211's work.
|
|
682
|
+
* Returns null for statements no contains-family matcher owns — other matchers
|
|
683
|
+
* answer for their own spans, and "AGENTS.md is under [200] lines" is not a
|
|
684
|
+
* discarded assertion.
|
|
685
|
+
*/
|
|
686
|
+
export function unconsumedAssertion(statement: string): string | null {
|
|
687
|
+
const body = requirementOf(statement);
|
|
688
|
+
const head = CLAUSE_HEAD.exec(body);
|
|
689
|
+
if (!head) return null;
|
|
690
|
+
|
|
691
|
+
let end = head[0].length;
|
|
692
|
+
for (;;) {
|
|
693
|
+
const tail = CLAUSE_TAIL.exec(body.slice(end));
|
|
694
|
+
if (!tail) break;
|
|
695
|
+
end += tail[0].length;
|
|
696
|
+
}
|
|
697
|
+
const residue = body.slice(end).trim();
|
|
698
|
+
return /\[[^\]]+\]/.test(residue) ? residue : null;
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
// Handler registry — one function per pattern type
|
|
702
|
+
const matcherHandlers: Record<string, MatcherHandler> = {
|
|
703
|
+
"file-exists": (sc, match) => {
|
|
704
|
+
const target = match[1].replace(/`/g, "");
|
|
160
705
|
if (target.startsWith("~/") || target.startsWith("/")) return null;
|
|
161
|
-
const limitMatch =
|
|
706
|
+
const limitMatch = sc.statement.match(/≤\s*(\d+)\s*lines/);
|
|
162
707
|
return (root) => {
|
|
163
708
|
expect(existsSync(join(root, target))).toBe(true);
|
|
164
709
|
if (limitMatch) {
|
|
165
710
|
const content = readFileSync(join(root, target), "utf-8");
|
|
166
|
-
|
|
711
|
+
const lineCount = content.trimEnd().split("\n").length;
|
|
712
|
+
expect(lineCount).toBeLessThanOrEqual(parseInt(limitMatch[1]));
|
|
167
713
|
}
|
|
168
714
|
};
|
|
169
|
-
}
|
|
715
|
+
},
|
|
170
716
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
const
|
|
174
|
-
const minMatch = s.match(/(?:with\s+)?≥\s*(\d+)\s+(\w+)/);
|
|
717
|
+
"dir-exists": (sc, match) => {
|
|
718
|
+
const dir = match[1].replace(/`/g, "");
|
|
719
|
+
const minMatch = sc.statement.match(/(?:with\s+)?≥\s*(\d+)\s+(\w+)/);
|
|
175
720
|
const aliases: Record<string, string[]> = {
|
|
176
721
|
"tests": ["tests", "test"],
|
|
177
722
|
"test": ["test", "tests"],
|
|
@@ -180,28 +725,55 @@ function matchPattern(sc: ParsedSC): AssertionFn | null {
|
|
|
180
725
|
const candidates = aliases[dir] || [dir];
|
|
181
726
|
return (root) => {
|
|
182
727
|
const found = candidates.find(d => existsSync(join(root, d)));
|
|
728
|
+
if (!found) {
|
|
729
|
+
addFinding({
|
|
730
|
+
ruleId: "SCAFFOLD-DIR-EXISTS",
|
|
731
|
+
severity: "FAIL",
|
|
732
|
+
file: dir,
|
|
733
|
+
message: `Required directory "${dir}" not found`,
|
|
734
|
+
fixCommand: `mkdir -p ${dir}`,
|
|
735
|
+
});
|
|
736
|
+
}
|
|
183
737
|
expect(found).toBeDefined();
|
|
184
738
|
if (minMatch && found) {
|
|
185
|
-
|
|
739
|
+
const count = readdirSync(join(root, found)).length;
|
|
740
|
+
const required = parseInt(minMatch[1]);
|
|
741
|
+
if (count < required) {
|
|
742
|
+
addFinding({
|
|
743
|
+
ruleId: "SCAFFOLD-DIR-COUNT",
|
|
744
|
+
severity: "FAIL",
|
|
745
|
+
file: found,
|
|
746
|
+
message: `${found}/ has ${count} files (requires ≥${required})`,
|
|
747
|
+
fixCommand: `Add files to ${found}/ — needs at least ${required}`,
|
|
748
|
+
});
|
|
749
|
+
}
|
|
750
|
+
expect(count).toBeGreaterThanOrEqual(required);
|
|
186
751
|
}
|
|
187
752
|
};
|
|
188
|
-
}
|
|
753
|
+
},
|
|
189
754
|
|
|
190
|
-
|
|
755
|
+
"all-specs-frontmatter": (sc) => {
|
|
191
756
|
return (root) => {
|
|
192
757
|
const specsDir = join(root, "specs");
|
|
193
758
|
if (!existsSync(specsDir)) return;
|
|
194
|
-
|
|
759
|
+
|
|
760
|
+
// #225 — the hand-rolled one-level descent that used to live here is
|
|
761
|
+
// now the shared collector, so the seven flat callers get the same
|
|
762
|
+
// behaviour instead of this one site being the only correct reader.
|
|
763
|
+
const specFiles = collectSpecFiles(specsDir);
|
|
764
|
+
|
|
765
|
+
const missing = specFiles.filter(f => {
|
|
195
766
|
const fm = parseFrontmatter(readFileSync(join(specsDir, f), "utf-8"));
|
|
767
|
+
// Skip redirect files (status: split)
|
|
768
|
+
if (fm?.status === "split") return false;
|
|
196
769
|
return !fm || !("testable" in fm);
|
|
197
770
|
});
|
|
198
771
|
expect(missing).toEqual([]);
|
|
199
772
|
};
|
|
200
|
-
}
|
|
773
|
+
},
|
|
201
774
|
|
|
202
|
-
|
|
203
|
-
const
|
|
204
|
-
const target = fileMatch?.[1]?.replace(/`/g, "") || "AGENTS.md";
|
|
775
|
+
"paths-resolve": (sc, match) => {
|
|
776
|
+
const target = match[1]?.replace(/`/g, "") || "AGENTS.md";
|
|
205
777
|
return (root) => {
|
|
206
778
|
const p = join(root, target);
|
|
207
779
|
if (!existsSync(p)) return;
|
|
@@ -215,81 +787,586 @@ function matchPattern(sc: ParsedSC): AssertionFn | null {
|
|
|
215
787
|
}
|
|
216
788
|
expect(broken).toEqual([]);
|
|
217
789
|
};
|
|
218
|
-
}
|
|
790
|
+
},
|
|
219
791
|
|
|
220
|
-
|
|
221
|
-
const contentLimit =
|
|
222
|
-
const codeLimit =
|
|
792
|
+
"root-clean": (sc) => {
|
|
793
|
+
const contentLimit = sc.statement.match(/content:\s*≤\s*(\d+)/i);
|
|
794
|
+
const codeLimit = sc.statement.match(/code:\s*≤\s*(\d+)/i);
|
|
223
795
|
return (root) => {
|
|
224
796
|
const items = readdirSync(root).filter(f => !f.startsWith(".") && f !== "node_modules");
|
|
225
|
-
// Detect project type: if src/, lib/, gates/, or package.json with deps → code project
|
|
226
797
|
const isCode = existsSync(join(root, "src")) || existsSync(join(root, "lib")) ||
|
|
227
798
|
existsSync(join(root, "gates")) || existsSync(join(root, "Makefile"));
|
|
228
799
|
const limit = isCode
|
|
229
800
|
? (codeLimit ? parseInt(codeLimit[1]) : 30)
|
|
230
801
|
: (contentLimit ? parseInt(contentLimit[1]) : 10);
|
|
802
|
+
if (items.length > limit) {
|
|
803
|
+
addFinding({
|
|
804
|
+
ruleId: "SCAFFOLD-ROOT-CLEAN",
|
|
805
|
+
severity: "FAIL",
|
|
806
|
+
file: ".",
|
|
807
|
+
message: `Root has ${items.length} items (limit: ${limit}). Move non-essential files to subdirectories.`,
|
|
808
|
+
fixCommand: `Review root items and move docs to docs/, scripts to scripts/, specs to specs/`,
|
|
809
|
+
});
|
|
810
|
+
}
|
|
231
811
|
expect(items.length).toBeLessThanOrEqual(limit);
|
|
232
812
|
};
|
|
233
|
-
}
|
|
813
|
+
},
|
|
234
814
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
const
|
|
238
|
-
const target = pointerMatch[2].replace(/`/g, "");
|
|
815
|
+
"pointer-exists": (sc, match) => {
|
|
816
|
+
const file = match[1].replace(/`/g, "");
|
|
817
|
+
const target = match[2].replace(/`/g, "");
|
|
239
818
|
return (root) => {
|
|
240
|
-
|
|
241
|
-
expect(
|
|
819
|
+
const path = resolveFilePath(root, file);
|
|
820
|
+
expect(existsSync(path)).toBe(true);
|
|
821
|
+
expect(readFileSync(path, "utf-8")).toContain(target);
|
|
242
822
|
};
|
|
243
|
-
}
|
|
823
|
+
},
|
|
244
824
|
|
|
245
|
-
if (/bun\s+test.*passes/i.test(s)) return () => { expect(true).toBe(true); };
|
|
246
825
|
|
|
247
|
-
|
|
826
|
+
"canary": () => {
|
|
248
827
|
return (root) => {
|
|
249
828
|
const testDir = existsSync(join(root, "test")) ? join(root, "test") : join(root, "tests");
|
|
250
829
|
if (!existsSync(testDir)) return;
|
|
251
830
|
const testFiles = readdirSync(testDir).filter(f => f.includes("conformity") || f.includes("canary"));
|
|
252
831
|
expect(testFiles.length).toBeGreaterThanOrEqual(1);
|
|
253
832
|
};
|
|
833
|
+
},
|
|
834
|
+
|
|
835
|
+
"content-contains-compound": (sc) => {
|
|
836
|
+
const clauses = parseCompoundClauses(sc.statement);
|
|
837
|
+
// Not actually compound — fall through so `content-contains` handles it.
|
|
838
|
+
if (!clauses) return null;
|
|
839
|
+
return (root) => {
|
|
840
|
+
for (const clause of clauses) {
|
|
841
|
+
const path = resolveFilePath(root, clause.file);
|
|
842
|
+
if (!existsSync(path)) {
|
|
843
|
+
expect(existsSync(path)).toBe(true);
|
|
844
|
+
return;
|
|
845
|
+
}
|
|
846
|
+
const content = readFileSync(path, "utf-8");
|
|
847
|
+
const view = clause.negated && isCodeFile(clause.file)
|
|
848
|
+
? stripJsComments(content)
|
|
849
|
+
: content;
|
|
850
|
+
for (const item of clause.items) {
|
|
851
|
+
if (clause.negated) {
|
|
852
|
+
expect(view).not.toContain(item);
|
|
853
|
+
} else {
|
|
854
|
+
expect(view).toContain(item);
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
};
|
|
859
|
+
},
|
|
860
|
+
|
|
861
|
+
"content-contains": (sc, match) => {
|
|
862
|
+
const file = match[1].replace(/`/g, "");
|
|
863
|
+
const items = match[2].split(",").map(i => i.trim());
|
|
864
|
+
return (root) => {
|
|
865
|
+
const path = resolveFilePath(root, file);
|
|
866
|
+
if (!existsSync(path)) {
|
|
867
|
+
expect(existsSync(path)).toBe(true);
|
|
868
|
+
return;
|
|
869
|
+
}
|
|
870
|
+
const content = readFileSync(path, "utf-8");
|
|
871
|
+
for (const item of items) {
|
|
872
|
+
expect(content).toContain(item);
|
|
873
|
+
}
|
|
874
|
+
};
|
|
875
|
+
},
|
|
876
|
+
|
|
877
|
+
"content-not-contains": (sc, match) => {
|
|
878
|
+
const file = match[1].replace(/`/g, "");
|
|
879
|
+
const items = match[2].split(",").map(i => i.trim());
|
|
880
|
+
return (root) => {
|
|
881
|
+
const path = resolveFilePath(root, file);
|
|
882
|
+
if (!existsSync(path)) {
|
|
883
|
+
expect(existsSync(path)).toBe(true);
|
|
884
|
+
return;
|
|
885
|
+
}
|
|
886
|
+
const content = readFileSync(path, "utf-8");
|
|
887
|
+
for (const item of items) {
|
|
888
|
+
expect(content).not.toContain(item);
|
|
889
|
+
}
|
|
890
|
+
};
|
|
891
|
+
},
|
|
892
|
+
|
|
893
|
+
"count-threshold": (sc, match) => {
|
|
894
|
+
// #225 — the registry's regex is `^(.+?)\s+(?:is under|at most)…`, which
|
|
895
|
+
// is wider than this handler was. "AGENTS.md Rules section at most [200]
|
|
896
|
+
// words" matched, and the whole phrase "AGENTS.md Rules section" was then
|
|
897
|
+
// treated as a path: the file did not exist, so the check failed an SC
|
|
898
|
+
// that the tree satisfies (97 words against a threshold of 200). The
|
|
899
|
+
// registry's own `syntax` field says `{file} is under [N] lines|words`
|
|
900
|
+
// and never promised sections, so the regex admitted a form the handler
|
|
901
|
+
// could not honour — the shape #212 is about, met from the handler side.
|
|
902
|
+
const subject = match[1].replace(/`/g, "").trim();
|
|
903
|
+
const sectionMatch = subject.match(/^(\S+)\s+(.+?)\s+section$/i);
|
|
904
|
+
const file = sectionMatch ? sectionMatch[1] : subject;
|
|
905
|
+
const section = sectionMatch ? sectionMatch[2] : null;
|
|
906
|
+
const threshold = parseInt(match[2]);
|
|
907
|
+
const unit = match[3].toLowerCase();
|
|
908
|
+
// "(excluding tables/headers)" is part of the criterion, not commentary:
|
|
909
|
+
// a Rules section is mostly a table, and counting the table's words
|
|
910
|
+
// measures the opposite of the leanness the SC asks for.
|
|
911
|
+
const excludeTables = /excluding\s+tables/i.test(sc.statement);
|
|
912
|
+
return (root) => {
|
|
913
|
+
const path = join(root, file);
|
|
914
|
+
if (!existsSync(path)) {
|
|
915
|
+
expect(existsSync(path)).toBe(true);
|
|
916
|
+
return;
|
|
917
|
+
}
|
|
918
|
+
let content = readFileSync(path, "utf-8");
|
|
919
|
+
if (section) {
|
|
920
|
+
const lines = content.split("\n");
|
|
921
|
+
const start = lines.findIndex(l => new RegExp(`^#+\\s+${section}\\s*$`, "i").test(l));
|
|
922
|
+
// A named section that is not there is a failure of the criterion,
|
|
923
|
+
// not a reason to measure the whole file and probably pass.
|
|
924
|
+
expect(start, `${file} has no "${section}" section to measure`).toBeGreaterThan(-1);
|
|
925
|
+
const rest = lines.slice(start + 1);
|
|
926
|
+
const end = rest.findIndex(l => /^#+\s/.test(l));
|
|
927
|
+
content = (end === -1 ? rest : rest.slice(0, end)).join("\n");
|
|
928
|
+
}
|
|
929
|
+
if (excludeTables) {
|
|
930
|
+
content = content
|
|
931
|
+
.split("\n")
|
|
932
|
+
.filter(l => !/^\s*\|/.test(l) && !/^#+\s/.test(l))
|
|
933
|
+
.join("\n");
|
|
934
|
+
}
|
|
935
|
+
if (unit.startsWith("line")) {
|
|
936
|
+
const lineCount = content.trimEnd().split("\n").length;
|
|
937
|
+
expect(lineCount).toBeLessThanOrEqual(threshold);
|
|
938
|
+
} else {
|
|
939
|
+
const wordCount = content.split(/\s+/).filter(w => w.length > 0).length;
|
|
940
|
+
expect(wordCount).toBeLessThanOrEqual(threshold);
|
|
941
|
+
}
|
|
942
|
+
};
|
|
943
|
+
},
|
|
944
|
+
|
|
945
|
+
"json-field-equals": (sc, match) => {
|
|
946
|
+
const file = match[1].replace(/`/g, "");
|
|
947
|
+
const field = match[2];
|
|
948
|
+
const expectedValue = match[3];
|
|
949
|
+
return (root) => {
|
|
950
|
+
const path = join(root, file);
|
|
951
|
+
if (!existsSync(path)) {
|
|
952
|
+
expect(existsSync(path)).toBe(true);
|
|
953
|
+
return;
|
|
954
|
+
}
|
|
955
|
+
const content = readFileSync(path, "utf-8");
|
|
956
|
+
const json = JSON.parse(content);
|
|
957
|
+
// Support nested fields like "config.name"
|
|
958
|
+
const fieldParts = field.split(".");
|
|
959
|
+
let value = json;
|
|
960
|
+
for (const part of fieldParts) {
|
|
961
|
+
value = value?.[part];
|
|
962
|
+
}
|
|
963
|
+
expect(value).toBe(expectedValue);
|
|
964
|
+
};
|
|
965
|
+
},
|
|
966
|
+
|
|
967
|
+
"section-exists": (sc, match) => {
|
|
968
|
+
const file = match[1].replace(/`/g, "");
|
|
969
|
+
const sectionName = match[2];
|
|
970
|
+
return (root) => {
|
|
971
|
+
const path = join(root, file);
|
|
972
|
+
if (!existsSync(path)) {
|
|
973
|
+
expect(existsSync(path)).toBe(true);
|
|
974
|
+
return;
|
|
975
|
+
}
|
|
976
|
+
const content = readFileSync(path, "utf-8");
|
|
977
|
+
// Match both # and ## headings
|
|
978
|
+
const hasSection = new RegExp(`^#+ ${sectionName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}`, "mi").test(content);
|
|
979
|
+
expect(hasSection).toBe(true);
|
|
980
|
+
};
|
|
981
|
+
},
|
|
982
|
+
|
|
983
|
+
"regex-match": (sc, match) => {
|
|
984
|
+
const file = match[1].replace(/`/g, "");
|
|
985
|
+
const pattern = match[2];
|
|
986
|
+
const flags = match[3] || "";
|
|
987
|
+
return (root) => {
|
|
988
|
+
const path = join(root, file);
|
|
989
|
+
if (!existsSync(path)) {
|
|
990
|
+
expect(existsSync(path)).toBe(true);
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
const content = readFileSync(path, "utf-8");
|
|
994
|
+
const regex = new RegExp(pattern, flags);
|
|
995
|
+
expect(regex.test(content)).toBe(true);
|
|
996
|
+
};
|
|
997
|
+
},
|
|
998
|
+
|
|
999
|
+
"source-contains": (sc, match) => {
|
|
1000
|
+
const file = match[1].replace(/`/g, "");
|
|
1001
|
+
// Only match if it's a source file path (lib/, scripts/, hooks/, gates/) or has "harness" prefix
|
|
1002
|
+
const isSourceFile = /^(lib|scripts|hooks|gates)\//.test(file) || sc.statement.toLowerCase().includes("harness");
|
|
1003
|
+
if (isSourceFile) {
|
|
1004
|
+
const items = match[2].split(",").map(i => i.trim());
|
|
1005
|
+
return (root) => {
|
|
1006
|
+
const path = join(root, file);
|
|
1007
|
+
if (!existsSync(path)) {
|
|
1008
|
+
expect(existsSync(path)).toBe(true);
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const content = readFileSync(path, "utf-8");
|
|
1012
|
+
for (const item of items) {
|
|
1013
|
+
expect(content).toContain(item);
|
|
1014
|
+
}
|
|
1015
|
+
};
|
|
1016
|
+
}
|
|
1017
|
+
return null;
|
|
1018
|
+
},
|
|
1019
|
+
|
|
1020
|
+
"json-has-field": (sc, match) => {
|
|
1021
|
+
const file = match[1].replace(/`/g, "");
|
|
1022
|
+
const filePath = file.endsWith(".json") ? file : `${file}.json`;
|
|
1023
|
+
const field = match[2];
|
|
1024
|
+
return (root) => {
|
|
1025
|
+
const path = join(root, filePath);
|
|
1026
|
+
if (!existsSync(path)) {
|
|
1027
|
+
expect(existsSync(path)).toBe(true);
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
1030
|
+
const content = readFileSync(path, "utf-8");
|
|
1031
|
+
const json = JSON.parse(content);
|
|
1032
|
+
// Support nested fields like "config.name"
|
|
1033
|
+
const fieldParts = field.split(".");
|
|
1034
|
+
let value = json;
|
|
1035
|
+
for (const part of fieldParts) {
|
|
1036
|
+
value = value?.[part];
|
|
1037
|
+
}
|
|
1038
|
+
expect(value).toBeDefined();
|
|
1039
|
+
};
|
|
1040
|
+
},
|
|
1041
|
+
|
|
1042
|
+
"scaffold-produces": (sc, match) => {
|
|
1043
|
+
const file = match[1].replace(/`/g, "");
|
|
1044
|
+
return (root) => {
|
|
1045
|
+
// For now, this checks if the file exists in the root
|
|
1046
|
+
// A more complete implementation would actually run scaffold and check output
|
|
1047
|
+
const path = join(root, file);
|
|
1048
|
+
expect(existsSync(path)).toBe(true);
|
|
1049
|
+
};
|
|
1050
|
+
},
|
|
1051
|
+
|
|
1052
|
+
"frontmatter-field": (sc, match) => {
|
|
1053
|
+
const file = match[1].replace(/`/g, "");
|
|
1054
|
+
const field = match[2];
|
|
1055
|
+
const expectedValue = match[3]?.trim();
|
|
1056
|
+
return (root) => {
|
|
1057
|
+
const path = join(root, file);
|
|
1058
|
+
if (!existsSync(path)) {
|
|
1059
|
+
expect(existsSync(path)).toBe(true);
|
|
1060
|
+
return;
|
|
1061
|
+
}
|
|
1062
|
+
const content = readFileSync(path, "utf-8");
|
|
1063
|
+
const fm = parseFrontmatter(content);
|
|
1064
|
+
expect(fm).toBeDefined();
|
|
1065
|
+
expect(fm?.[field]).toBeDefined();
|
|
1066
|
+
if (expectedValue !== undefined) {
|
|
1067
|
+
expect(fm?.[field]).toBe(expectedValue);
|
|
1068
|
+
}
|
|
1069
|
+
};
|
|
1070
|
+
},
|
|
1071
|
+
|
|
1072
|
+
"file-line-range": (sc, match) => {
|
|
1073
|
+
const file = match[1].replace(/`/g, "");
|
|
1074
|
+
const min = parseInt(match[2]);
|
|
1075
|
+
const max = parseInt(match[3]);
|
|
1076
|
+
return (root) => {
|
|
1077
|
+
const path = join(root, file);
|
|
1078
|
+
if (!existsSync(path)) {
|
|
1079
|
+
expect(existsSync(path)).toBe(true);
|
|
1080
|
+
return;
|
|
1081
|
+
}
|
|
1082
|
+
const content = readFileSync(path, "utf-8");
|
|
1083
|
+
const lineCount = content.trimEnd().split("\n").length;
|
|
1084
|
+
expect(lineCount).toBeGreaterThanOrEqual(min);
|
|
1085
|
+
expect(lineCount).toBeLessThanOrEqual(max);
|
|
1086
|
+
};
|
|
1087
|
+
},
|
|
1088
|
+
|
|
1089
|
+
"command-output": (sc, match) => {
|
|
1090
|
+
const command = match[1];
|
|
1091
|
+
const items = match[2].split(",").map(i => i.trim());
|
|
1092
|
+
return (root) => {
|
|
1093
|
+
const result = spawnSync("sh", ["-c", command], {
|
|
1094
|
+
cwd: root,
|
|
1095
|
+
timeout: 10000,
|
|
1096
|
+
encoding: "utf-8",
|
|
1097
|
+
env: { ...process.env },
|
|
1098
|
+
});
|
|
1099
|
+
const exitCode = result.status ?? 1;
|
|
1100
|
+
expect(exitCode).toBe(0);
|
|
1101
|
+
const stdout = result.stdout || "";
|
|
1102
|
+
for (const item of items) {
|
|
1103
|
+
expect(stdout.includes(item)).toBe(true);
|
|
1104
|
+
}
|
|
1105
|
+
};
|
|
1106
|
+
},
|
|
1107
|
+
};
|
|
1108
|
+
|
|
1109
|
+
/**
|
|
1110
|
+
* The names the engine can actually assert. Exported so the registry can be
|
|
1111
|
+
* reconciled against it (#212) instead of the two drifting apart unobserved —
|
|
1112
|
+
* `matchPattern` iterates the REGISTRY, so a handler missing from it is
|
|
1113
|
+
* unreachable however well tested, and an entry missing a handler asserts
|
|
1114
|
+
* nothing while still matching the author's syntax.
|
|
1115
|
+
*/
|
|
1116
|
+
export function matcherHandlerNames(): string[] {
|
|
1117
|
+
return Object.keys(matcherHandlers);
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
/**
|
|
1121
|
+
* Why a statement did or did not become an assertion.
|
|
1122
|
+
*
|
|
1123
|
+
* `matchPattern` answered `null` for two different situations, and collapsing
|
|
1124
|
+
* them is the defect in #212: a statement that matched a registry regex whose
|
|
1125
|
+
* entry has no handler was reported exactly like a statement that matched
|
|
1126
|
+
* nothing at all. The first means "recognised, and nothing implements it";
|
|
1127
|
+
* the second means "unrecognised". SUCCESS.md claim 4 — "could not measure"
|
|
1128
|
+
* must never read as "nothing wrong" — needs them distinct.
|
|
1129
|
+
*/
|
|
1130
|
+
export type MatcherResolution =
|
|
1131
|
+
| { kind: "assert"; matcher: string; assertion: AssertionFn }
|
|
1132
|
+
| { kind: "unimplemented"; matcher: string }
|
|
1133
|
+
| { kind: "unrecognised" };
|
|
1134
|
+
|
|
1135
|
+
export function resolveMatcher(sc: ParsedSC, projectRoot?: string): MatcherResolution {
|
|
1136
|
+
// Strict view: the project being verified may add matchers, never replace a
|
|
1137
|
+
// built-in one. See loadRegistry's docblock — `isMatchablePattern` keeps the
|
|
1138
|
+
// permissive view because it answers an authoring question, not a
|
|
1139
|
+
// verification one.
|
|
1140
|
+
const registry = loadRegistry(projectRoot, { allowBuiltInOverride: false });
|
|
1141
|
+
let recognisedBy: string | null = null;
|
|
1142
|
+
|
|
1143
|
+
for (const entry of registry) {
|
|
1144
|
+
const regex = new RegExp(entry.regex, "i");
|
|
1145
|
+
const match = sc.statement.match(regex);
|
|
1146
|
+
if (!match) continue;
|
|
1147
|
+
|
|
1148
|
+
const handler = matcherHandlers[entry.name];
|
|
1149
|
+
if (!handler) {
|
|
1150
|
+
// Recognised. Remember which matcher claimed it, but keep looking — a
|
|
1151
|
+
// later entry may both match and be implemented, and that is a better
|
|
1152
|
+
// answer than refusing.
|
|
1153
|
+
recognisedBy ??= entry.name;
|
|
1154
|
+
continue;
|
|
1155
|
+
}
|
|
1156
|
+
const result = handler(sc, match);
|
|
1157
|
+
if (result) return { kind: "assert", matcher: entry.name, assertion: result };
|
|
254
1158
|
}
|
|
255
1159
|
|
|
256
|
-
return
|
|
1160
|
+
return recognisedBy ? { kind: "unimplemented", matcher: recognisedBy } : { kind: "unrecognised" };
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
export function matchPattern(sc: ParsedSC, projectRoot?: string): AssertionFn | null {
|
|
1164
|
+
const outcome = resolveMatcher(sc, projectRoot);
|
|
1165
|
+
return outcome.kind === "assert" ? outcome.assertion : null;
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
/**
|
|
1169
|
+
* Check if an SC statement matches any pattern in the registry (built-in + consumer).
|
|
1170
|
+
* Unlike matchPattern, this only checks regex matches — it does not require a handler.
|
|
1171
|
+
* Used by create-spec for SC validation: a consumer custom pattern without a handler
|
|
1172
|
+
* is still a valid SC pattern for specification purposes.
|
|
1173
|
+
*/
|
|
1174
|
+
export function isMatchablePattern(sc: ParsedSC, projectRoot?: string): boolean {
|
|
1175
|
+
const registry = loadRegistry(projectRoot);
|
|
1176
|
+
for (const entry of registry) {
|
|
1177
|
+
const regex = new RegExp(entry.regex, "i");
|
|
1178
|
+
if (regex.test(sc.statement)) {
|
|
1179
|
+
return true;
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
return false;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
// ── SC → test binding ───────────────────────────────────────
|
|
1186
|
+
|
|
1187
|
+
/**
|
|
1188
|
+
* How a single SC becomes (or fails to become) a test.
|
|
1189
|
+
*
|
|
1190
|
+
* Kept as data, separate from `describe`/`test`, so the binding rules can be
|
|
1191
|
+
* asserted directly — see test/conformity-sc-binding.test.ts. The important
|
|
1192
|
+
* property is that an `assert` entry hands back the RAW assertion: nothing
|
|
1193
|
+
* here catches, so a failing checked SC fails the suite.
|
|
1194
|
+
*/
|
|
1195
|
+
export type SCPlanEntry =
|
|
1196
|
+
| { kind: "behavioral"; sc: ParsedSC; routing: string | null }
|
|
1197
|
+
| { kind: "todo"; sc: ParsedSC }
|
|
1198
|
+
| { kind: "unmatched"; sc: ParsedSC }
|
|
1199
|
+
| { kind: "unimplemented"; sc: ParsedSC; matcher: string }
|
|
1200
|
+
| { kind: "assert"; sc: ParsedSC; assertion: AssertionFn };
|
|
1201
|
+
|
|
1202
|
+
/**
|
|
1203
|
+
* Decide what each SC becomes:
|
|
1204
|
+
* behavioral — runtime-only, routed to SESSION-AUDIT-SPEC
|
|
1205
|
+
* todo — unchecked, i.e. work not started; never a silent assertion
|
|
1206
|
+
* assert — checked and matchable; runs for real and may fail
|
|
1207
|
+
* unmatched — checked but the statement binds to no matcher: a coverage
|
|
1208
|
+
* gap, reported and ratcheted rather than silently dropped
|
|
1209
|
+
* unimplemented — checked, and the statement DID match a registry entry
|
|
1210
|
+
* that has no handler. Reported apart from `unmatched`
|
|
1211
|
+
* because "recognised and nothing implements it" is a
|
|
1212
|
+
* different defect from "unrecognised", and the author of
|
|
1213
|
+
* such an SC was told by create-spec that it was valid (#212)
|
|
1214
|
+
*/
|
|
1215
|
+
export function planSCTests(scs: ParsedSC[], projectRoot?: string): SCPlanEntry[] {
|
|
1216
|
+
return scs.map((sc): SCPlanEntry => {
|
|
1217
|
+
if (isBehavioralSC(sc)) return { kind: "behavioral", sc, routing: getBehavioralRouting(sc) };
|
|
1218
|
+
if (!sc.checked) return { kind: "todo", sc };
|
|
1219
|
+
const outcome = resolveMatcher(sc, projectRoot);
|
|
1220
|
+
if (outcome.kind === "assert") return { kind: "assert", sc, assertion: outcome.assertion };
|
|
1221
|
+
if (outcome.kind === "unimplemented") return { kind: "unimplemented", sc, matcher: outcome.matcher };
|
|
1222
|
+
return { kind: "unmatched", sc };
|
|
1223
|
+
});
|
|
257
1224
|
}
|
|
258
1225
|
|
|
259
1226
|
// ── Exported test runners ───────────────────────────────────
|
|
260
1227
|
|
|
261
1228
|
export function runScaffoldConformity(root: string, opts?: { extraSpecDirs?: string[] }) {
|
|
262
|
-
const
|
|
263
|
-
const
|
|
264
|
-
const allSCs = collectTestableSpecs(root, opts?.extraSpecDirs ?? defaultExtraSpecs);
|
|
1229
|
+
const specMap = collectTestableSpecs(root, opts?.extraSpecDirs ?? []);
|
|
1230
|
+
const specsDir = join(root, "specs");
|
|
265
1231
|
|
|
266
|
-
|
|
267
|
-
if (
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
1232
|
+
function resolveSpecPath(specFile: string): string {
|
|
1233
|
+
if (specFile.includes("/")) {
|
|
1234
|
+
const parts = specFile.split("/");
|
|
1235
|
+
for (const dir of opts?.extraSpecDirs ?? []) {
|
|
1236
|
+
if (dir.endsWith(parts[0])) return join(dir, parts.slice(1).join("/"));
|
|
1237
|
+
}
|
|
272
1238
|
}
|
|
1239
|
+
return join(specsDir, specFile);
|
|
1240
|
+
}
|
|
273
1241
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
bySpec.set(sc.specFile, group);
|
|
1242
|
+
describe("Spec-Driven Conformity Tests", () => {
|
|
1243
|
+
if (specMap.size === 0) {
|
|
1244
|
+
test.skip("at least one testable spec with SCs exists (no specs found — consumer project)", () => {});
|
|
1245
|
+
return;
|
|
279
1246
|
}
|
|
280
1247
|
|
|
281
|
-
|
|
1248
|
+
// No afterAll checkbox flip here, deliberately (#86).
|
|
1249
|
+
//
|
|
1250
|
+
// This describe builds its test list at collection time by reading SC
|
|
1251
|
+
// checkboxes off disk. Writing those same files back at the end of the run
|
|
1252
|
+
// meant run N mutated the input run N+1 collected from, so the generated
|
|
1253
|
+
// test list alternated between two shapes — 57 tests, then 99, then 57.
|
|
1254
|
+
// Both parities reported 0 fail, so nothing ever surfaced it: the suite was
|
|
1255
|
+
// not hiding a failure, it was hiding whether 42 checks ran at all.
|
|
1256
|
+
//
|
|
1257
|
+
// Checkbox syncing still happens, deliberately and separately, via the
|
|
1258
|
+
// `posttest` hook in package.json -> scripts/sync-sc-status.ts. That script
|
|
1259
|
+
// re-derives pass state from the specs itself rather than riding on a test
|
|
1260
|
+
// run's in-memory bookkeeping, so it is not load-bearing for this suite and
|
|
1261
|
+
// nothing is lost by removing the hook.
|
|
1262
|
+
|
|
1263
|
+
for (const [specFile, metadata] of specMap) {
|
|
282
1264
|
describe(specFile, () => {
|
|
283
1265
|
const unmatched: string[] = [];
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
1266
|
+
const unimplemented: string[] = [];
|
|
1267
|
+
const behavioral: string[] = [];
|
|
1268
|
+
const specPath = resolveSpecPath(specFile);
|
|
1269
|
+
// `root`, not nothing: without it `loadRegistry` never reads the
|
|
1270
|
+
// consumer's `.claude/rungate.json`, so every custom matcher is
|
|
1271
|
+
// invisible to the engine while `create-spec` — which DOES pass a
|
|
1272
|
+
// project root to `isMatchablePattern` — accepts criteria written
|
|
1273
|
+
// against them. That is the authoring-tool/engine divergence in #212,
|
|
1274
|
+
// and it was not hypothetical: the planted-project test in
|
|
1275
|
+
// test/matcher-registry-parity.test.ts reported its criterion as
|
|
1276
|
+
// `unmatched` until this argument was supplied.
|
|
1277
|
+
for (const entry of planSCTests(metadata.scs, root)) {
|
|
1278
|
+
const { sc } = entry;
|
|
1279
|
+
if (entry.kind === "behavioral") {
|
|
1280
|
+
behavioral.push(`${sc.id}: ${sc.statement} → ${entry.routing}`);
|
|
1281
|
+
} else if (entry.kind === "todo") {
|
|
1282
|
+
// bun-types declares test.todo as requiring a body even though the
|
|
1283
|
+
// runtime accepts a bare label. A no-op body satisfies both: todo
|
|
1284
|
+
// tests are not executed unless `bun test --todo` is passed.
|
|
1285
|
+
test.todo(`${sc.id}: ${sc.statement}`, () => {});
|
|
1286
|
+
} else if (entry.kind === "unmatched") {
|
|
1287
|
+
unmatched.push(`${sc.id}: ${sc.statement}`);
|
|
1288
|
+
} else if (entry.kind === "unimplemented") {
|
|
1289
|
+
// Not folded into `unmatched`: this SC's syntax WAS recognised,
|
|
1290
|
+
// and create-spec would have told its author it was valid. The
|
|
1291
|
+
// gap is a registry entry with no handler, which is a different
|
|
1292
|
+
// thing to fix and a different thing to report (#212).
|
|
1293
|
+
unimplemented.push(`${sc.id}: matched "${entry.matcher}", which has no handler — ${sc.statement}`);
|
|
1294
|
+
} else {
|
|
1295
|
+
// A CHECKED SC claims the work is done. Run its assertion for
|
|
1296
|
+
// real: no try/catch, so it fails the suite when the claim is
|
|
1297
|
+
// false. The old body swallowed the throw, which is why 13 false
|
|
1298
|
+
// claims sat green for months (#209).
|
|
1299
|
+
test(`${sc.id}: ${sc.statement}`, () => {
|
|
1300
|
+
entry.assertion(root);
|
|
1301
|
+
recordPassingSC(sc.id, specPath);
|
|
1302
|
+
});
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
if (behavioral.length > 0) {
|
|
1306
|
+
addBehavioralCount(behavioral.length);
|
|
1307
|
+
// This is a REPORT, not a test, and that is the fix (#328).
|
|
1308
|
+
//
|
|
1309
|
+
// It used to be a `test()` whose body was `expect(true).toBe(true)`
|
|
1310
|
+
// — shape three in .claude/rules/checks-must-be-able-to-fail.md,
|
|
1311
|
+
// written out literally rather than arrived at by accident, and the
|
|
1312
|
+
// only executable check on 40 ticked criteria.
|
|
1313
|
+
//
|
|
1314
|
+
// The first attempt at a repair kept the `test()` and asserted that
|
|
1315
|
+
// every listed SC carries the `(behavioral)` marker. The mutation
|
|
1316
|
+
// harness measured that at ZERO red, and it was right to: this list
|
|
1317
|
+
// is only ever appended to when `entry.kind === "behavioral"`, which
|
|
1318
|
+
// happens only when `isBehavioralSC(sc)` is already true. The
|
|
1319
|
+
// assertion was a tautology — a second unfalsifiable check wearing
|
|
1320
|
+
// the first one's clothes.
|
|
1321
|
+
//
|
|
1322
|
+
// There is nothing locally falsifiable to put here. Whether a
|
|
1323
|
+
// behavioral criterion is SATISFIED is a runtime question this file
|
|
1324
|
+
// cannot answer, and whether it is CLASSIFIED correctly is true by
|
|
1325
|
+
// construction. So the slot stops pretending: the listing stays,
|
|
1326
|
+
// because findability is genuinely useful, and the teeth live in
|
|
1327
|
+
// test/behavioral-sc-ratchet.test.ts, which bounds the population
|
|
1328
|
+
// and which the same harness measures at 1 red when slackened.
|
|
1329
|
+
//
|
|
1330
|
+
// A report that admits it is a report is strictly better than a test
|
|
1331
|
+
// that cannot fail, because only one of them is counted as evidence.
|
|
1332
|
+
console.log(
|
|
1333
|
+
`Behavioral SCs in ${specFile} — NOT VERIFIED HERE, bounded by ` +
|
|
1334
|
+
`BEHAVIORAL_TICKED_RATCHET (#328):\n ${behavioral.join("\n ")}`,
|
|
1335
|
+
);
|
|
288
1336
|
}
|
|
289
1337
|
if (unmatched.length > 0) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
1338
|
+
// SC-287 used to FAIL here in strict mode. That branch had never
|
|
1339
|
+
// fired: the only SCs that reached it were unchecked ones, and
|
|
1340
|
+
// unchecked SCs now become `todo` before the matcher runs. The
|
|
1341
|
+
// population is now CHECKED SCs written in prose no matcher binds
|
|
1342
|
+
// to — 54 of them on the day they first became visible (#209).
|
|
1343
|
+
//
|
|
1344
|
+
// Failing 54 specs at once would get the whole file skipped, so the
|
|
1345
|
+
// teeth live in a single repo-wide ratchet instead:
|
|
1346
|
+
// test/conformity-sc-binding.test.ts asserts the total may fall but
|
|
1347
|
+
// never rise. Here we name them, per spec, so they are findable.
|
|
1348
|
+
const level = metadata.compliance === "permissive" ? "WARN" : "WARN (strict)";
|
|
1349
|
+
test(`${level}: ${unmatched.length} checked SCs have no pattern matcher`, () => {
|
|
1350
|
+
console.warn(`Unmatched checked SCs in ${specFile}:\n ${unmatched.join("\n ")}`);
|
|
1351
|
+
expect(unmatched.length).toBeGreaterThan(0);
|
|
1352
|
+
});
|
|
1353
|
+
}
|
|
1354
|
+
// Deliberately BELOW the unmatched branch, not between it and the
|
|
1355
|
+
// behavioral report. #328's guard slices the source from the
|
|
1356
|
+
// behavioral branch to `if (unmatched.length > 0) {` and asserts that
|
|
1357
|
+
// region contains no `test(` — the behavioral slot must stay a report.
|
|
1358
|
+
// This block legitimately IS a test, so placing it in that window
|
|
1359
|
+
// turns #328's guard red. The guard is right; this belongs here.
|
|
1360
|
+
if (unimplemented.length > 0) {
|
|
1361
|
+
// Unlike the unmatched aggregate above, this one FAILS, and it can
|
|
1362
|
+
// afford to: the population is zero in this repo and the parity
|
|
1363
|
+
// test in test/matcher-registry-parity.test.ts keeps it there, so
|
|
1364
|
+
// it cannot turn specs red en masse. A registry entry with no
|
|
1365
|
+
// handler is a harness defect, not a backlog of prose criteria
|
|
1366
|
+
// waiting to be rewritten — the author was told by create-spec that
|
|
1367
|
+
// their syntax was valid, and the engine then asserted nothing.
|
|
1368
|
+
test(`FAIL: ${unimplemented.length} SCs matched a registry entry that has no handler`, () => {
|
|
1369
|
+
expect(unimplemented).toEqual([]);
|
|
293
1370
|
});
|
|
294
1371
|
}
|
|
295
1372
|
});
|
|
@@ -299,12 +1376,16 @@ export function runScaffoldConformity(root: string, opts?: { extraSpecDirs?: str
|
|
|
299
1376
|
describe("Scaffold: structural checks", () => {
|
|
300
1377
|
test("AGENTS.md has required standard sections", () => {
|
|
301
1378
|
if (!existsSync(join(root, "AGENTS.md"))) {
|
|
1379
|
+
addFinding({ ruleId: "SCAFFOLD-AGENTS-MISSING", severity: "FAIL", file: "AGENTS.md", message: "AGENTS.md not found", fixCommand: "Run scaffold to generate AGENTS.md" });
|
|
302
1380
|
expect(existsSync(join(root, "AGENTS.md"))).toBe(true);
|
|
303
1381
|
return;
|
|
304
1382
|
}
|
|
305
1383
|
const content = readFileSync(join(root, "AGENTS.md"), "utf-8");
|
|
306
|
-
const required = ["Project Identity", "
|
|
1384
|
+
const required = ["Project Identity", "Rules", "Commands", "Workflow"];
|
|
307
1385
|
const missing = required.filter(s => !content.toLowerCase().includes(s.toLowerCase()));
|
|
1386
|
+
if (missing.length > 0) {
|
|
1387
|
+
addFinding({ ruleId: "SCAFFOLD-AGENTS-SECTIONS", severity: "FAIL", file: "AGENTS.md", message: `Missing sections: ${missing.join(", ")}`, fixCommand: "Re-run scaffold to regenerate AGENTS.md" });
|
|
1388
|
+
}
|
|
308
1389
|
expect(missing).toEqual([]);
|
|
309
1390
|
});
|
|
310
1391
|
|
|
@@ -336,25 +1417,56 @@ export function runSpecDiscovery(root: string) {
|
|
|
336
1417
|
|
|
337
1418
|
test("every .md in specs/ has testable field", () => {
|
|
338
1419
|
if (!existsSync(specsDir)) return;
|
|
339
|
-
const missing =
|
|
1420
|
+
const missing = collectSpecFiles(specsDir).filter(f => {
|
|
340
1421
|
const fm = parseFrontmatter(readFileSync(join(specsDir, f), "utf-8"));
|
|
341
1422
|
return !fm || !("testable" in fm);
|
|
342
1423
|
});
|
|
1424
|
+
for (const f of missing) {
|
|
1425
|
+
addFinding({ ruleId: "SPEC-FRONTMATTER", severity: "FAIL", file: `specs/${f}`, message: `Spec "${f}" missing testable field in frontmatter`, fixCommand: `Add "testable: true" or "testable: false" to frontmatter of specs/${f}` });
|
|
1426
|
+
}
|
|
343
1427
|
expect(missing).toEqual([]);
|
|
344
1428
|
});
|
|
345
1429
|
|
|
346
1430
|
test("at least one testable spec exists", () => {
|
|
347
1431
|
if (!existsSync(specsDir)) return;
|
|
348
|
-
const testable =
|
|
1432
|
+
const testable = collectSpecFiles(specsDir).filter(f => {
|
|
349
1433
|
const fm = parseFrontmatter(readFileSync(join(specsDir, f), "utf-8"));
|
|
350
1434
|
return fm?.testable === "true";
|
|
351
1435
|
});
|
|
352
|
-
|
|
1436
|
+
if (testable.length === 0) {
|
|
1437
|
+
addFinding({
|
|
1438
|
+
ruleId: "SPEC-DISCOVERY-TESTABLE",
|
|
1439
|
+
severity: "WARN",
|
|
1440
|
+
file: "specs/",
|
|
1441
|
+
message: "No spec has testable: true. Add testable: true to a spec when ready for conformity testing.",
|
|
1442
|
+
fixCommand: `Add "testable: true" to frontmatter of a spec in specs/`,
|
|
1443
|
+
});
|
|
1444
|
+
console.warn("SPEC-DISCOVERY: No testable specs found — skipping (add testable: true when ready)");
|
|
1445
|
+
} else {
|
|
1446
|
+
expect(testable.length).toBeGreaterThan(0);
|
|
1447
|
+
}
|
|
353
1448
|
});
|
|
354
1449
|
});
|
|
355
1450
|
}
|
|
356
1451
|
|
|
357
|
-
|
|
1452
|
+
/**
|
|
1453
|
+
* @param options.untestedAllowlist Specs permitted to have no referencing test,
|
|
1454
|
+
* owned by the CALLING project. Omitted means the caller has not adopted the
|
|
1455
|
+
* ratchet, and DRIFT-2 reports advisory instead of failing — see the comment
|
|
1456
|
+
* on DRIFT-2 for why a consumer must not inherit rungate's backlog.
|
|
1457
|
+
*
|
|
1458
|
+
* The allowlist is deliberately in `options`, not position 2: the consumer test
|
|
1459
|
+
* that lib/scaffold/steps.ts generates calls `runSpecDrift(ROOT, findings)`, so
|
|
1460
|
+
* a positional second parameter would receive a findings array and exempt
|
|
1461
|
+
* nothing, failing every consumer build.
|
|
1462
|
+
*/
|
|
1463
|
+
export function runSpecDrift(
|
|
1464
|
+
root: string,
|
|
1465
|
+
_findings?: unknown[],
|
|
1466
|
+
options?: { untestedAllowlist?: string[] },
|
|
1467
|
+
) {
|
|
1468
|
+
const untestedAllowlist =
|
|
1469
|
+
options?.untestedAllowlist ?? loadConformityAllowlists(root).untestedSpecs;
|
|
358
1470
|
const specsDir = join(root, "specs");
|
|
359
1471
|
const testDirs = [join(root, "test"), join(root, "tests")].filter(d => existsSync(d));
|
|
360
1472
|
|
|
@@ -387,8 +1499,8 @@ export function runSpecDrift(root: string) {
|
|
|
387
1499
|
});
|
|
388
1500
|
|
|
389
1501
|
test("DRIFT-2: Every testable spec is referenced by at least one test", () => {
|
|
390
|
-
if (!existsSync(specsDir)) return;
|
|
391
|
-
const testable =
|
|
1502
|
+
if (!existsSync(specsDir)) return checkSkipped("DRIFT-2", "no specs/ directory");
|
|
1503
|
+
const testable = collectSpecFiles(specsDir).filter(f => {
|
|
392
1504
|
const fm = parseFrontmatter(readFileSync(join(specsDir, f), "utf-8"));
|
|
393
1505
|
return fm?.testable === "true";
|
|
394
1506
|
});
|
|
@@ -401,15 +1513,170 @@ export function runSpecDrift(root: string) {
|
|
|
401
1513
|
}
|
|
402
1514
|
|
|
403
1515
|
const orphaned = testable.filter(s => !allContent.includes(s));
|
|
404
|
-
|
|
405
|
-
|
|
1516
|
+
|
|
1517
|
+
// Was `expect(true).toBe(true)` after a console.warn, so the mechanical
|
|
1518
|
+
// enforcement of CLAUDE.md's "SCs without tests are wishes" could never
|
|
1519
|
+
// fail (#80). Turning it on revealed 10 testable specs in rungate with no
|
|
1520
|
+
// test referencing them at all — over half the testable surface.
|
|
1521
|
+
//
|
|
1522
|
+
// That backlog cannot be asserted to zero without writing ten tests, so
|
|
1523
|
+
// adopters pass a RATCHET list: anything on it may stay untested, an
|
|
1524
|
+
// addition fails the build, and a listed spec that GAINS a test must be
|
|
1525
|
+
// delisted or the build fails — otherwise the allowlist rots into another
|
|
1526
|
+
// artifact that certifies nothing.
|
|
1527
|
+
//
|
|
1528
|
+
// The list belongs to the CALLING PROJECT, never to this file. An earlier
|
|
1529
|
+
// version hardcoded rungate's ten spec filenames here, and
|
|
1530
|
+
// lib/scaffold/steps.ts generates a consumer conformity test that calls
|
|
1531
|
+
// this function — so every consumer inherited rungate's backlog. A
|
|
1532
|
+
// consumer with an untested spec of a matching name was silently
|
|
1533
|
+
// exempted, and a consumer that HAD such a spec with a test failed its
|
|
1534
|
+
// own build demanding it delist an entry from a file inside rungate that
|
|
1535
|
+
// it cannot edit.
|
|
1536
|
+
if (!untestedAllowlist) {
|
|
1537
|
+
// No ratchet adopted. Report, but do not fail: before #80 this check
|
|
1538
|
+
// could not fail at all, so failing here would break every existing
|
|
1539
|
+
// consumer on upgrade for a backlog they never opted into.
|
|
1540
|
+
if (orphaned.length > 0) {
|
|
1541
|
+
console.warn(
|
|
1542
|
+
`WARN DRIFT-2: ${orphaned.length} testable spec(s) have no referencing test — ` +
|
|
1543
|
+
`${orphaned.join(", ")}. This check is ADVISORY because no untestedAllowlist ` +
|
|
1544
|
+
`was supplied. Pass one to runSpecDrift to enforce it as a ratchet.`,
|
|
1545
|
+
);
|
|
1546
|
+
}
|
|
1547
|
+
return;
|
|
406
1548
|
}
|
|
407
|
-
|
|
1549
|
+
|
|
1550
|
+
const newlyOrphaned = orphaned.filter(s => !untestedAllowlist.includes(s));
|
|
1551
|
+
expect(newlyOrphaned).toEqual([]);
|
|
1552
|
+
|
|
1553
|
+
const staleAllowlist = untestedAllowlist.filter(
|
|
1554
|
+
s => testable.includes(s) && !orphaned.includes(s),
|
|
1555
|
+
);
|
|
1556
|
+
expect(staleAllowlist).toEqual([]);
|
|
1557
|
+
});
|
|
1558
|
+
|
|
1559
|
+
test("DRIFT-3: the exemption lists did not GROW", () => {
|
|
1560
|
+
// #302. The staleness checks above enforce one direction: an entry that
|
|
1561
|
+
// stops being a violation fails. A NEW entry for a genuinely new
|
|
1562
|
+
// violation is not stale, so until now an agent could introduce a
|
|
1563
|
+
// conformity violation and exempt it in the same commit — passing a
|
|
1564
|
+
// gate by editing the gate, which is the one move this harness exists
|
|
1565
|
+
// to refuse.
|
|
1566
|
+
//
|
|
1567
|
+
// It sits here rather than in the collection guard deliberately. The
|
|
1568
|
+
// collection guard sees paths, not contents, and would catch only
|
|
1569
|
+
// growth arriving via agent collection — not a human commit, a heal
|
|
1570
|
+
// round, or a single-worktree run that bypasses collection entirely.
|
|
1571
|
+
const allowlistPath = join(root, ".claude", "conformity-allowlists.json");
|
|
1572
|
+
if (!existsSync(allowlistPath)) return;
|
|
1573
|
+
|
|
1574
|
+
const show = spawnSync("git", ["-C", root, "show", `origin/main:.claude/conformity-allowlists.json`], {
|
|
1575
|
+
encoding: "utf-8",
|
|
1576
|
+
timeout: 20_000,
|
|
1577
|
+
});
|
|
1578
|
+
|
|
1579
|
+
if (show.error || show.status !== 0) {
|
|
1580
|
+
// ADVISORY, not a refusal, and only here. A consumer may legitimately
|
|
1581
|
+
// have no `origin/main` — a fresh scaffold, a local-only repo — and
|
|
1582
|
+
// refusing for want of a comparison base would break every one of
|
|
1583
|
+
// them on upgrade for a rule they never opted into. That is the same
|
|
1584
|
+
// judgement loadConformityAllowlists already makes about a missing
|
|
1585
|
+
// allowlist file. rungate's own suite asserts the base IS readable,
|
|
1586
|
+
// because in this repo an unreadable one is a fault, not a setup.
|
|
1587
|
+
console.warn(
|
|
1588
|
+
`WARN DRIFT-3: could not read origin/main's copy of the conformity allowlist, so ` +
|
|
1589
|
+
`growth was NOT checked. This is UNMEASURED, not clean.`,
|
|
1590
|
+
);
|
|
1591
|
+
return;
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
const added = allowlistAdditions(show.stdout, readFileSync(allowlistPath, "utf-8"));
|
|
1595
|
+
expect(
|
|
1596
|
+
added,
|
|
1597
|
+
`an exemption was ADDED relative to origin/main. Fix the violation, or make the ` +
|
|
1598
|
+
`exemption a deliberate recorded decision rather than a diff.`,
|
|
1599
|
+
).toEqual([]);
|
|
408
1600
|
});
|
|
409
1601
|
});
|
|
410
1602
|
}
|
|
411
1603
|
|
|
412
|
-
|
|
1604
|
+
/**
|
|
1605
|
+
* @param options.orphanAllowlist Files permitted to have zero references, owned
|
|
1606
|
+
* by the CALLING project. Same contract and same positional constraint as
|
|
1607
|
+
* runSpecDrift's untestedAllowlist — the generated consumer test calls
|
|
1608
|
+
* `runDocHygiene(ROOT, findings)`.
|
|
1609
|
+
*/
|
|
1610
|
+
/**
|
|
1611
|
+
* Every path's most recent commit date, from ONE git subprocess (#323).
|
|
1612
|
+
*
|
|
1613
|
+
* Exported so the regression test can run it against the repository and
|
|
1614
|
+
* compare it to the per-file calls it replaced — a single site to mutate, and
|
|
1615
|
+
* the only way to prove the parser still agrees with git rather than merely
|
|
1616
|
+
* being faster. A detector that got quicker by finding less would pass every
|
|
1617
|
+
* timing test and none of the ones that matter.
|
|
1618
|
+
*
|
|
1619
|
+
* Walks newest-first, so the first time a path appears is its latest commit
|
|
1620
|
+
* and later entries are ignored. Paths deleted or renamed earlier in history
|
|
1621
|
+
* also appear; they are never looked up, which is cheaper than filtering them.
|
|
1622
|
+
*
|
|
1623
|
+
* `pathspecs` must be the SET OF PATHS THE CALLER WILL LOOK UP, not the
|
|
1624
|
+
* directories they mostly live in. The first version of this took the six doc
|
|
1625
|
+
* directories, which is where the staleness scan finds most of its files — but
|
|
1626
|
+
* the scan also walks root-level markdown, and `git log -- specs docs` cannot
|
|
1627
|
+
* report a commit for `AGENTS.md`. Six files silently dropped out of the
|
|
1628
|
+
* lookup and `if (!dateStr) continue` read that as "newly added, so not
|
|
1629
|
+
* stale". A detector narrower than what it detects, shipped inside the fix for
|
|
1630
|
+
* a different instance of the same thing. Passing the lookup set makes the
|
|
1631
|
+
* pathspec and the keys the same list by construction, so they cannot drift.
|
|
1632
|
+
*
|
|
1633
|
+
* Fails CLOSED. `spawnSync` does not throw when git exits non-zero, so a
|
|
1634
|
+
* broken invocation would otherwise return an empty map and every file would
|
|
1635
|
+
* read as newly added — total blindness that looks exactly like a clean run.
|
|
1636
|
+
*/
|
|
1637
|
+
export function lastCommitDates(root: string, pathspecs: string[]): Map<string, string> {
|
|
1638
|
+
const lastCommit = new Map<string, string>();
|
|
1639
|
+
// Not a git repository: nothing is tracked, so no path has a commit date.
|
|
1640
|
+
// That is a real answer, not a failure, and scaffolded projects hit it.
|
|
1641
|
+
if (!existsSync(join(root, ".git"))) return lastCommit;
|
|
1642
|
+
if (pathspecs.length === 0) return lastCommit;
|
|
1643
|
+
{
|
|
1644
|
+
const log = spawnSync(
|
|
1645
|
+
"git",
|
|
1646
|
+
["-C", root, "log", "--format=%H %ci", "--name-only", "--", ...pathspecs],
|
|
1647
|
+
{ timeout: 60_000, maxBuffer: 64 * 1024 * 1024 },
|
|
1648
|
+
);
|
|
1649
|
+
if (log.error) {
|
|
1650
|
+
throw new Error(`lastCommitDates: git log could not run in ${root}: ${log.error.message}`);
|
|
1651
|
+
}
|
|
1652
|
+
if (log.status !== 0) {
|
|
1653
|
+
throw new Error(
|
|
1654
|
+
`lastCommitDates: git log exited ${log.status} in ${root}: ` +
|
|
1655
|
+
`${log.stderr?.toString().trim() || "(no stderr)"}`,
|
|
1656
|
+
);
|
|
1657
|
+
}
|
|
1658
|
+
let currentDate = "";
|
|
1659
|
+
for (const line of log.stdout.toString().split("\n")) {
|
|
1660
|
+
const header = /^[0-9a-f]{7,40} (\d{4}-\d{2}-\d{2})/.exec(line);
|
|
1661
|
+
if (header) {
|
|
1662
|
+
currentDate = header[1];
|
|
1663
|
+
continue;
|
|
1664
|
+
}
|
|
1665
|
+
const path = line.trim();
|
|
1666
|
+
if (!path || !currentDate) continue;
|
|
1667
|
+
if (!lastCommit.has(path)) lastCommit.set(path, currentDate);
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
return lastCommit;
|
|
1671
|
+
}
|
|
1672
|
+
|
|
1673
|
+
export function runDocHygiene(
|
|
1674
|
+
root: string,
|
|
1675
|
+
_findings?: unknown[],
|
|
1676
|
+
options?: { orphanAllowlist?: string[] },
|
|
1677
|
+
) {
|
|
1678
|
+
const orphanAllowlist =
|
|
1679
|
+
options?.orphanAllowlist ?? loadConformityAllowlists(root).orphanFiles;
|
|
413
1680
|
const specsDir = join(root, "specs");
|
|
414
1681
|
const refDir = join(root, "reference");
|
|
415
1682
|
// .claude/agents/ excluded — Claude Code reads those by name convention, not import
|
|
@@ -440,7 +1707,24 @@ export function runDocHygiene(root: string) {
|
|
|
440
1707
|
? readdirSync(dir).filter(f => exts.some(e => f.endsWith(e)))
|
|
441
1708
|
: readdirSync(dir, { recursive: true }).map(f => String(f)).filter(f => exts.some(e => f.endsWith(e)));
|
|
442
1709
|
for (const f of files) {
|
|
443
|
-
|
|
1710
|
+
const rel = String(f);
|
|
1711
|
+
|
|
1712
|
+
// skipDirs above only filters TOP-LEVEL entries of root, so the
|
|
1713
|
+
// recursive walk still descended into .claude/worktrees/ — which
|
|
1714
|
+
// holds full copies of the repo, 39 of them on the machine where
|
|
1715
|
+
// this was found. Every file in every worktree counted as a
|
|
1716
|
+
// reference, so nothing was ever an orphan locally while a clean CI
|
|
1717
|
+
// checkout saw three. That divergence is what made HYGIENE-3 look
|
|
1718
|
+
// intermittent and got a real failure dismissed as flake.
|
|
1719
|
+
if (rel.includes("worktrees/") || skipDirs.has(rel.split("/")[0])) continue;
|
|
1720
|
+
|
|
1721
|
+
// The allowlist names the very files it exempts, so counting it as a
|
|
1722
|
+
// reference makes every listed orphan stop being an orphan — the
|
|
1723
|
+
// ratchet then fails demanding they be delisted, and delisting makes
|
|
1724
|
+
// them orphans again. A list that invalidates itself by existing.
|
|
1725
|
+
if (rel.endsWith("conformity-allowlists.json")) continue;
|
|
1726
|
+
|
|
1727
|
+
try { chunks.push(readFileSync(join(dir, rel), "utf-8")); } catch {}
|
|
444
1728
|
}
|
|
445
1729
|
} catch {}
|
|
446
1730
|
}
|
|
@@ -455,27 +1739,33 @@ export function runDocHygiene(root: string) {
|
|
|
455
1739
|
|
|
456
1740
|
describe("Doc Hygiene", () => {
|
|
457
1741
|
test("HYGIENE-1: All specs have non-empty governs field", () => {
|
|
458
|
-
if (!existsSync(specsDir)) return;
|
|
1742
|
+
if (!existsSync(specsDir)) return checkSkipped("HYGIENE-1", "no specs/ directory");
|
|
459
1743
|
const missing: string[] = [];
|
|
460
|
-
for (const f of
|
|
1744
|
+
for (const f of collectSpecFiles(specsDir)) { // #225 — subdirectories too
|
|
461
1745
|
const content = readFileSync(join(specsDir, f), "utf-8");
|
|
462
1746
|
const fm = parseFrontmatter(content);
|
|
463
1747
|
if (!fm) continue;
|
|
464
1748
|
const governs = fm.governs || "";
|
|
465
1749
|
if (!governs || governs.includes("TODO")) missing.push(f);
|
|
466
1750
|
}
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
}
|
|
470
|
-
expect(true).toBe(true);
|
|
1751
|
+
// Was unconditionally passing after a console.warn (#80).
|
|
1752
|
+
expect(missing).toEqual([]);
|
|
471
1753
|
});
|
|
472
1754
|
|
|
473
1755
|
test("HYGIENE-2: All specs have updated field in frontmatter", () => {
|
|
474
1756
|
if (!existsSync(specsDir)) return;
|
|
475
1757
|
const missing: string[] = [];
|
|
476
|
-
|
|
1758
|
+
|
|
1759
|
+
// #225 — the hand-rolled one-level descent that used to live here is
|
|
1760
|
+
// now the shared collector, so the seven flat callers get the same
|
|
1761
|
+
// behaviour instead of this one site being the only correct reader.
|
|
1762
|
+
const specFiles = collectSpecFiles(specsDir);
|
|
1763
|
+
|
|
1764
|
+
for (const f of specFiles) {
|
|
477
1765
|
const content = readFileSync(join(specsDir, f), "utf-8");
|
|
478
1766
|
const fm = parseFrontmatter(content);
|
|
1767
|
+
// Skip redirect files (status: split)
|
|
1768
|
+
if (fm?.status === "split") continue;
|
|
479
1769
|
if (!fm || !fm.updated) missing.push(f);
|
|
480
1770
|
}
|
|
481
1771
|
expect(missing).toEqual([]);
|
|
@@ -490,25 +1780,60 @@ export function runDocHygiene(root: string) {
|
|
|
490
1780
|
if (!index.includes(f)) orphans.push(`${relPath}/${f}`);
|
|
491
1781
|
}
|
|
492
1782
|
}
|
|
493
|
-
|
|
494
|
-
|
|
1783
|
+
// Was unconditionally passing after a console.warn (#80). Two orphans it
|
|
1784
|
+
// had been quietly reporting — specs/bootstrap-data-flow.png and
|
|
1785
|
+
// scripts/self-containment-check.sh — were archived to reference/, the
|
|
1786
|
+
// remedy this check recommends in its own message.
|
|
1787
|
+
//
|
|
1788
|
+
// Converting it to a bare `expect(orphans).toEqual([])` then turned Gates
|
|
1789
|
+
// red on main, because sizing from the warn output was taken as the
|
|
1790
|
+
// post-merge state without re-running the check against a clean tree. The
|
|
1791
|
+
// warn output listed two; a clean checkout has three more that local runs
|
|
1792
|
+
// did not surface. Same instrument as DRIFT-2 applies here and was not
|
|
1793
|
+
// used: list what cannot be fixed now, fail on an addition, fail if a
|
|
1794
|
+
// listed file gains a reference and is not delisted.
|
|
1795
|
+
//
|
|
1796
|
+
// Caller-owned for the same reason as DRIFT-2 — consumers must not
|
|
1797
|
+
// inherit rungate's exemptions.
|
|
1798
|
+
if (!orphanAllowlist) {
|
|
1799
|
+
if (orphans.length > 0) {
|
|
1800
|
+
console.warn(
|
|
1801
|
+
`WARN HYGIENE-3: ${orphans.length} file(s) with zero references — ` +
|
|
1802
|
+
`${orphans.join(", ")}. ADVISORY because no orphanAllowlist was supplied. ` +
|
|
1803
|
+
`Pass one to runDocHygiene to enforce it as a ratchet.`,
|
|
1804
|
+
);
|
|
1805
|
+
}
|
|
1806
|
+
return;
|
|
495
1807
|
}
|
|
496
|
-
|
|
1808
|
+
|
|
1809
|
+
const newOrphans = orphans.filter(o => !orphanAllowlist.includes(o));
|
|
1810
|
+
expect(newOrphans).toEqual([]);
|
|
1811
|
+
|
|
1812
|
+
const staleAllowlist = orphanAllowlist.filter(o => !orphans.includes(o));
|
|
1813
|
+
expect(staleAllowlist).toEqual([]);
|
|
497
1814
|
});
|
|
498
1815
|
|
|
499
|
-
test("HYGIENE-4:
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
1816
|
+
test("HYGIENE-4: Every spec appears in the specs routing table", () => {
|
|
1817
|
+
// The table lives in .claude/rules/specs-routing.md, NOT AGENTS.md. This
|
|
1818
|
+
// check read AGENTS.md, which contains no spec references at all, so it
|
|
1819
|
+
// "found" 22 violations — every spec in the repo — and reported none of
|
|
1820
|
+
// them because it ended in expect(true).toBe(true) (#80). The premise was
|
|
1821
|
+
// stale, which is how a check this wrong stayed invisible.
|
|
1822
|
+
const routingPath = join(root, ".claude", "rules", "specs-routing.md");
|
|
1823
|
+
if (!existsSync(specsDir)) return checkSkipped("HYGIENE-4", "no specs/ directory");
|
|
1824
|
+
if (!existsSync(routingPath)) return checkSkipped("HYGIENE-4", "no .claude/rules/specs-routing.md");
|
|
1825
|
+
const routing = readFileSync(routingPath, "utf-8");
|
|
1826
|
+
const specFiles = readdirSync(specsDir)
|
|
1827
|
+
.filter(f => f.endsWith(".md"))
|
|
1828
|
+
// SPEC-TEMPLATE.md documents how to write a spec; it governs nothing.
|
|
1829
|
+
.filter(f => f !== "SPEC-TEMPLATE.md");
|
|
1830
|
+
const unlisted = specFiles.filter(f => !routing.includes(f));
|
|
1831
|
+
expect(unlisted).toEqual([]);
|
|
508
1832
|
});
|
|
509
1833
|
|
|
510
1834
|
test("HYGIENE-5: reference/ contents not in active routing", () => {
|
|
511
|
-
if (!existsSync(refDir)
|
|
1835
|
+
if (!existsSync(refDir)) return checkSkipped("HYGIENE-5", "no reference/ directory");
|
|
1836
|
+
if (!existsSync(join(root, "AGENTS.md"))) return checkSkipped("HYGIENE-5", "no AGENTS.md");
|
|
512
1837
|
const agentsContent = readFileSync(join(root, "AGENTS.md"), "utf-8");
|
|
513
1838
|
const leaked: string[] = [];
|
|
514
1839
|
for (const sub of readdirSync(refDir)) {
|
|
@@ -523,10 +1848,8 @@ export function runDocHygiene(root: string) {
|
|
|
523
1848
|
}
|
|
524
1849
|
} catch {}
|
|
525
1850
|
}
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
}
|
|
529
|
-
expect(true).toBe(true);
|
|
1851
|
+
// Was unconditionally passing after a console.warn (#80).
|
|
1852
|
+
expect(leaked).toEqual([]);
|
|
530
1853
|
});
|
|
531
1854
|
|
|
532
1855
|
test("HYGIENE-10: All ADRs have doc-type: adr in frontmatter", () => {
|
|
@@ -540,7 +1863,7 @@ export function runDocHygiene(root: string) {
|
|
|
540
1863
|
missing.push(f);
|
|
541
1864
|
addFinding({
|
|
542
1865
|
ruleId: "HYGIENE-10",
|
|
543
|
-
severity: "
|
|
1866
|
+
severity: "WARN",
|
|
544
1867
|
file: `docs/adr/${f}`,
|
|
545
1868
|
message: `ADR "${f}" missing doc-type: adr in frontmatter`,
|
|
546
1869
|
fixCommand: `Add "doc-type: adr" to frontmatter of docs/adr/${f}`,
|
|
@@ -548,9 +1871,8 @@ export function runDocHygiene(root: string) {
|
|
|
548
1871
|
}
|
|
549
1872
|
}
|
|
550
1873
|
if (missing.length > 0) {
|
|
551
|
-
console.
|
|
1874
|
+
console.warn(`HYGIENE-10 WARN: ADRs missing doc-type: ${missing.join(", ")}. Fix: add "doc-type: adr" to frontmatter.`);
|
|
552
1875
|
}
|
|
553
|
-
expect(missing).toEqual([]);
|
|
554
1876
|
});
|
|
555
1877
|
|
|
556
1878
|
test("HYGIENE-7: No spec files at root (must be in specs/)", () => {
|
|
@@ -636,7 +1958,14 @@ export function runDocHygiene(root: string) {
|
|
|
636
1958
|
expect(misplaced).toEqual([]);
|
|
637
1959
|
});
|
|
638
1960
|
|
|
639
|
-
|
|
1961
|
+
// WARN, not a gate, and the name now says so. This reports a review
|
|
1962
|
+
// BACKLOG — 119 pending candidates on rungate as of #80 — so asserting it
|
|
1963
|
+
// to zero would block every push until someone triages all of them. It
|
|
1964
|
+
// previously claimed "No unreviewed constraint candidates" while ending in
|
|
1965
|
+
// expect(true).toBe(true), which is the dishonest combination: a gate-
|
|
1966
|
+
// sounding name over an unconditional pass. Advisory is the honest shape
|
|
1967
|
+
// here; the lie was the name, not the leniency.
|
|
1968
|
+
test("WARN HYGIENE-6: Unreviewed constraint candidates in changed docs", async () => {
|
|
640
1969
|
const { extractConstraints } = await import("../scripts/extract-constraints");
|
|
641
1970
|
const result = await extractConstraints(root, { apply: false });
|
|
642
1971
|
|
|
@@ -663,6 +1992,110 @@ export function runDocHygiene(root: string) {
|
|
|
663
1992
|
expect(true).toBe(true);
|
|
664
1993
|
});
|
|
665
1994
|
|
|
1995
|
+
test("HYGIENE-11: Stale unreferenced files warn with archival suggestion", () => {
|
|
1996
|
+
const index = getRefIndex();
|
|
1997
|
+
// Load stalenessThreshold from rungate.json, default 90 days
|
|
1998
|
+
const configPath = join(root, ".claude", "rungate.json");
|
|
1999
|
+
let threshold = 90;
|
|
2000
|
+
if (existsSync(configPath)) {
|
|
2001
|
+
try {
|
|
2002
|
+
const config = JSON.parse(readFileSync(configPath, "utf-8"));
|
|
2003
|
+
if (typeof config.stalenessThreshold === "number") {
|
|
2004
|
+
threshold = config.stalenessThreshold;
|
|
2005
|
+
}
|
|
2006
|
+
} catch {}
|
|
2007
|
+
}
|
|
2008
|
+
|
|
2009
|
+
// Scan docs directories for markdown files — skip reference/ entirely (AC-2)
|
|
2010
|
+
const scanTargets: string[] = [];
|
|
2011
|
+
const docDirs = ["specs", "docs", "docs/adr", "docs/research", "docs/council", "docs/guides"];
|
|
2012
|
+
for (const rel of docDirs) {
|
|
2013
|
+
const dir = join(root, rel);
|
|
2014
|
+
if (!existsSync(dir)) continue;
|
|
2015
|
+
for (const f of readdirSync(dir).filter(f => f.endsWith(".md"))) {
|
|
2016
|
+
const fullPath = join(dir, f);
|
|
2017
|
+
try {
|
|
2018
|
+
if (statSync(fullPath).isFile()) {
|
|
2019
|
+
scanTargets.push(`${rel}/${f}`);
|
|
2020
|
+
}
|
|
2021
|
+
} catch {}
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
// Also scan root-level markdown (excluding reference/)
|
|
2026
|
+
for (const f of readdirSync(root).filter(f => f.endsWith(".md"))) {
|
|
2027
|
+
scanTargets.push(f);
|
|
2028
|
+
}
|
|
2029
|
+
|
|
2030
|
+
// Filter out reference/ paths (AC-2)
|
|
2031
|
+
const filtered = scanTargets.filter(f => !f.startsWith("reference/") && !f.includes("/reference/"));
|
|
2032
|
+
|
|
2033
|
+
// One git call for every file, not one per file (#323).
|
|
2034
|
+
//
|
|
2035
|
+
// This was `spawnSync("git", ["log", "-1", ..., "--", rel])` inside the
|
|
2036
|
+
// loop: 60 serial subprocesses against the same repository, each with
|
|
2037
|
+
// its own 5-second timeout. Nothing was wrong with any single call — the
|
|
2038
|
+
// test was load-sensitive by construction, and if any one spawn was slow
|
|
2039
|
+
// (another suite holding the index, a busy machine, a cold cache) it hit
|
|
2040
|
+
// the timeout and the test went red. Measured at 1 failure in 3
|
|
2041
|
+
// identical runs, always at ~5007 ms. It also got worse on its own as
|
|
2042
|
+
// the repo grew, because more docs means more dice rolls.
|
|
2043
|
+
//
|
|
2044
|
+
// `--name-only` over the whole history, walked newest-first, gives every
|
|
2045
|
+
// path's most recent commit date from a single subprocess. The first
|
|
2046
|
+
// time a path is seen is its latest commit, so later entries are
|
|
2047
|
+
// ignored.
|
|
2048
|
+
//
|
|
2049
|
+
// `filtered` is passed, NOT `docDirs`. The first cut of this fix passed
|
|
2050
|
+
// the six doc directories, and `git log -- specs docs` has nothing to
|
|
2051
|
+
// say about the six root-level markdown files this scan also walks —
|
|
2052
|
+
// AGENTS.md, CLAUDE.md, SUCCESS.md and friends dropped out of the map,
|
|
2053
|
+
// and the `if (!dateStr) continue` below read that as "newly added, so
|
|
2054
|
+
// not stale". Handing it the exact lookup set keeps the pathspec and the
|
|
2055
|
+
// keys the same list by construction.
|
|
2056
|
+
const lastCommit = lastCommitDates(root, filtered);
|
|
2057
|
+
|
|
2058
|
+
const staleUnreferenced: { file: string; daysSince: number }[] = [];
|
|
2059
|
+
for (const rel of filtered) {
|
|
2060
|
+
const dateStr = lastCommit.get(rel);
|
|
2061
|
+
// No commit for this path means it is untracked or newly added, which
|
|
2062
|
+
// is the opposite of stale.
|
|
2063
|
+
if (!dateStr) continue;
|
|
2064
|
+
const daysSince = (Date.now() - new Date(dateStr).getTime()) / (1000 * 60 * 60 * 24);
|
|
2065
|
+
if (daysSince <= threshold) continue;
|
|
2066
|
+
|
|
2067
|
+
// Check cross-references: filename must appear somewhere in the codebase index
|
|
2068
|
+
const basename = rel.split("/").pop()!;
|
|
2069
|
+
if (index.includes(basename)) continue;
|
|
2070
|
+
|
|
2071
|
+
staleUnreferenced.push({ file: rel, daysSince: Math.floor(daysSince) });
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
// AC-3: Each warning includes git mv archival command
|
|
2075
|
+
if (staleUnreferenced.length > 0) {
|
|
2076
|
+
for (const entry of staleUnreferenced) {
|
|
2077
|
+
const archiveDest = `reference/${entry.file.split("/").pop()}`;
|
|
2078
|
+
addFinding({
|
|
2079
|
+
ruleId: "HYGIENE-11",
|
|
2080
|
+
severity: "WARN",
|
|
2081
|
+
file: entry.file,
|
|
2082
|
+
message: `Stale unreferenced file (${entry.daysSince}d, threshold=${threshold}d)`,
|
|
2083
|
+
fixCommand: `git mv ${entry.file} ${archiveDest}`,
|
|
2084
|
+
});
|
|
2085
|
+
console.warn(`HYGIENE-11: ${entry.file} — stale (${entry.daysSince}d) and unreferenced. Fix: git mv ${entry.file} ${archiveDest}`);
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
|
|
2089
|
+
// AC-1: At least 2 assertions
|
|
2090
|
+
// 1. reference/ files were filtered out
|
|
2091
|
+
const referenceFiles = scanTargets.filter(f => f.startsWith("reference/") || f.includes("/reference/"));
|
|
2092
|
+
expect(filtered.every(f => !f.startsWith("reference/") && !f.includes("/reference/"))).toBe(true);
|
|
2093
|
+
|
|
2094
|
+
// 2. All stale unreferenced entries have git mv fix commands
|
|
2095
|
+
const hygieneFindings = getFindings().filter(f => f.ruleId === "HYGIENE-11");
|
|
2096
|
+
expect(hygieneFindings.every(f => f.fixCommand?.startsWith("git mv "))).toBe(true);
|
|
2097
|
+
});
|
|
2098
|
+
|
|
666
2099
|
test("HYGIENE-REPORT: Write unified findings report", () => {
|
|
667
2100
|
const findings = getFindings();
|
|
668
2101
|
const candidates = getCandidates();
|
|
@@ -756,20 +2189,239 @@ export function runAgentFileValidation(root: string) {
|
|
|
756
2189
|
expect(mismatched).toEqual([]);
|
|
757
2190
|
});
|
|
758
2191
|
|
|
759
|
-
test("AGENT-5:
|
|
2192
|
+
test("AGENT-5: Shared agent principles exist as unconditional rule", () => {
|
|
2193
|
+
const rulesDir = join(root, ".claude", "rules");
|
|
2194
|
+
if (!existsSync(rulesDir)) {
|
|
2195
|
+
addFinding({ ruleId: "AGENT-5", severity: "FAIL", file: ".claude/rules/", message: "No rules directory found", fixCommand: "Re-scaffold to generate rules" });
|
|
2196
|
+
expect(existsSync(rulesDir)).toBe(true);
|
|
2197
|
+
return;
|
|
2198
|
+
}
|
|
2199
|
+
const principlesFile = join(rulesDir, "agent-principles.md");
|
|
2200
|
+
expect(existsSync(principlesFile)).toBe(true);
|
|
2201
|
+
const content = readFileSync(principlesFile, "utf-8");
|
|
2202
|
+
expect(content).toContain("Core Principles");
|
|
2203
|
+
});
|
|
2204
|
+
|
|
2205
|
+
test("AGENT-7: All agent briefs have required sections", () => {
|
|
760
2206
|
if (!existsSync(agentsDir)) return;
|
|
2207
|
+
const sectionChecks: Array<{ name: string; pattern: RegExp }> = [
|
|
2208
|
+
{ name: "frontmatter", pattern: /^---\n[\s\S]*?\n---/ },
|
|
2209
|
+
{ name: "identity", pattern: /You are\s+\w+/i },
|
|
2210
|
+
{ name: "agent-specific-section", pattern: /^##\s+(Workflow|Rules|Testing|Discovery|What you|TDD|CLI|UI|Project Type|Efficiency|Design|Architecture|Context|Report)/im },
|
|
2211
|
+
];
|
|
761
2212
|
const missing: string[] = [];
|
|
762
2213
|
for (const f of readdirSync(agentsDir).filter(f => f.endsWith(".md"))) {
|
|
763
2214
|
const content = readFileSync(join(agentsDir, f), "utf-8");
|
|
764
|
-
|
|
765
|
-
|
|
2215
|
+
for (const check of sectionChecks) {
|
|
2216
|
+
if (!check.pattern.test(content)) {
|
|
2217
|
+
missing.push(`${f}: missing required section '${check.name}'`);
|
|
2218
|
+
addFinding({
|
|
2219
|
+
ruleId: "AGENT-REQUIRED-SECTION",
|
|
2220
|
+
severity: "FAIL",
|
|
2221
|
+
file: `.claude/agents/${f}`,
|
|
2222
|
+
message: `Brief missing required section: ${check.name}`,
|
|
2223
|
+
});
|
|
2224
|
+
}
|
|
766
2225
|
}
|
|
767
2226
|
}
|
|
768
2227
|
if (missing.length > 0) {
|
|
769
|
-
console.
|
|
2228
|
+
console.error(`Agent briefs missing required sections:\n ${missing.join("\n ")}`);
|
|
770
2229
|
}
|
|
771
2230
|
expect(missing).toEqual([]);
|
|
772
2231
|
});
|
|
2232
|
+
|
|
2233
|
+
test("AGENT-8: Agent brief models match the roles config", () => {
|
|
2234
|
+
if (!existsSync(agentsDir)) return;
|
|
2235
|
+
// Was hardcoded to "sonnet". Roles are now routed per-role, so the
|
|
2236
|
+
// meaningful check is that each brief matches its configured model —
|
|
2237
|
+
// briefs are generated from that config, and a mismatch means someone
|
|
2238
|
+
// hand-edited generated output.
|
|
2239
|
+
//
|
|
2240
|
+
// #90: this read `.claude/rungate.json` directly, which is the LEGACY
|
|
2241
|
+
// monolith. Scaffold migrated projects to `.claude/rungate/` and
|
|
2242
|
+
// generates briefs from `roles.json` there, so on any migrated project
|
|
2243
|
+
// the generator and this validator read different files. They disagreed
|
|
2244
|
+
// in this very repo — roles.json said sonnet, the stale monolith said
|
|
2245
|
+
// opus — and AGENT-8 reported a violation against output that exactly
|
|
2246
|
+
// matched its real source.
|
|
2247
|
+
//
|
|
2248
|
+
// The failure mode was worse than a false alarm: the message told the
|
|
2249
|
+
// reader to "re-scaffold instead of editing the brief", and
|
|
2250
|
+
// re-scaffolding reproduced the same briefs, so following the advice
|
|
2251
|
+
// could not clear it.
|
|
2252
|
+
//
|
|
2253
|
+
// Go through the canonical loader, which understands both layouts and
|
|
2254
|
+
// prefers the directory. tryLoadRungateConfig is the non-throwing
|
|
2255
|
+
// variant: absent config is "nothing to check", while a MALFORMED
|
|
2256
|
+
// config still throws, per ADR-001 D2 — a broken config must not read
|
|
2257
|
+
// as an unconfigured one.
|
|
2258
|
+
const VALID_MODELS = ["sonnet", "opus", "haiku", "fable"];
|
|
2259
|
+
const roles: Record<string, { model?: string }> =
|
|
2260
|
+
(tryLoadRungateConfig(root)?.roles as Record<string, { model?: string }>) ?? {};
|
|
2261
|
+
|
|
2262
|
+
const wrong: string[] = [];
|
|
2263
|
+
for (const f of readdirSync(agentsDir).filter(f => f.endsWith(".md"))) {
|
|
2264
|
+
const content = readFileSync(join(agentsDir, f), "utf-8");
|
|
2265
|
+
const fm = parseFrontmatter(content);
|
|
2266
|
+
const role = f.replace(/\.md$/, "");
|
|
2267
|
+
const expected = roles[role]?.model;
|
|
2268
|
+
if (!fm?.model) {
|
|
2269
|
+
wrong.push(`${f}: missing model field`);
|
|
2270
|
+
} else if (!VALID_MODELS.includes(fm.model)) {
|
|
2271
|
+
wrong.push(`${f}: model is "${fm.model}", expected one of ${VALID_MODELS.join("|")}`);
|
|
2272
|
+
} else if (expected && fm.model !== expected) {
|
|
2273
|
+
wrong.push(`${f}: model is "${fm.model}", but the roles config says "${expected}" — re-scaffold instead of editing the brief`);
|
|
2274
|
+
}
|
|
2275
|
+
}
|
|
2276
|
+
if (wrong.length > 0) {
|
|
2277
|
+
wrong.forEach(w => addFinding({
|
|
2278
|
+
ruleId: "AGENT-MODEL",
|
|
2279
|
+
severity: "FAIL",
|
|
2280
|
+
file: `.claude/agents/${w.split(":")[0]}`,
|
|
2281
|
+
message: w,
|
|
2282
|
+
}));
|
|
2283
|
+
console.error(`Agent briefs with wrong model:\n ${wrong.join("\n ")}`);
|
|
2284
|
+
}
|
|
2285
|
+
expect(wrong).toEqual([]);
|
|
2286
|
+
});
|
|
2287
|
+
|
|
2288
|
+
test("AGENT-9: All agent briefs are under 120 lines", () => {
|
|
2289
|
+
if (!existsSync(agentsDir)) return;
|
|
2290
|
+
const over: string[] = [];
|
|
2291
|
+
for (const f of readdirSync(agentsDir).filter(f => f.endsWith(".md"))) {
|
|
2292
|
+
const content = readFileSync(join(agentsDir, f), "utf-8");
|
|
2293
|
+
const lineCount = content.trimEnd().split("\n").length;
|
|
2294
|
+
if (lineCount > 120) {
|
|
2295
|
+
over.push(`${f}: ${lineCount} lines (max 120)`);
|
|
2296
|
+
addFinding({
|
|
2297
|
+
ruleId: "AGENT-LINECOUNT",
|
|
2298
|
+
severity: "FAIL",
|
|
2299
|
+
file: `.claude/agents/${f}`,
|
|
2300
|
+
message: `Brief is ${lineCount} lines, exceeds 120-line cap`,
|
|
2301
|
+
});
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
2304
|
+
if (over.length > 0) {
|
|
2305
|
+
console.error(`Agent briefs over 120 lines:\n ${over.join("\n ")}`);
|
|
2306
|
+
}
|
|
2307
|
+
expect(over).toEqual([]);
|
|
2308
|
+
});
|
|
2309
|
+
|
|
2310
|
+
test("AGENT-11: AGENTS.md is under line cap (sigmoid collapse prevention)", () => {
|
|
2311
|
+
const agentsMdPath = join(root, "AGENTS.md");
|
|
2312
|
+
if (!existsSync(agentsMdPath)) return;
|
|
2313
|
+
const content = readFileSync(agentsMdPath, "utf-8");
|
|
2314
|
+
const lineCount = content.trimEnd().split("\n").length;
|
|
2315
|
+
const consumerLines = (content.match(/^- \w/gm) || []).length;
|
|
2316
|
+
const cap = 100 + Math.max(0, consumerLines - 10);
|
|
2317
|
+
if (lineCount > cap) {
|
|
2318
|
+
addFinding({
|
|
2319
|
+
ruleId: "AGENTS-MD-LINECOUNT",
|
|
2320
|
+
severity: "FAIL",
|
|
2321
|
+
file: "AGENTS.md",
|
|
2322
|
+
message: `AGENTS.md is ${lineCount} lines, exceeds ${cap}-line cap (base 100 + ${Math.max(0, consumerLines - 10)} consumer lines). Move reference tables to .claude/rules/`,
|
|
2323
|
+
});
|
|
2324
|
+
}
|
|
2325
|
+
expect(lineCount).toBeLessThanOrEqual(cap);
|
|
2326
|
+
});
|
|
2327
|
+
|
|
2328
|
+
test("AGENT-10: No unfilled template variables in agent briefs", () => {
|
|
2329
|
+
if (!existsSync(agentsDir)) return;
|
|
2330
|
+
const unfilled: string[] = [];
|
|
2331
|
+
for (const f of readdirSync(agentsDir).filter(f => f.endsWith(".md"))) {
|
|
2332
|
+
const content = readFileSync(join(agentsDir, f), "utf-8");
|
|
2333
|
+
const vars = content.match(/\$\{[A-Z_]+\}/g);
|
|
2334
|
+
if (vars && vars.length > 0) {
|
|
2335
|
+
unfilled.push(`${f}: unfilled template variables: ${vars.join(", ")}`);
|
|
2336
|
+
addFinding({
|
|
2337
|
+
ruleId: "AGENT-TEMPLATE-VAR",
|
|
2338
|
+
severity: "FAIL",
|
|
2339
|
+
file: `.claude/agents/${f}`,
|
|
2340
|
+
message: `Brief has unfilled template variables: ${vars.join(", ")}`,
|
|
2341
|
+
});
|
|
2342
|
+
}
|
|
2343
|
+
}
|
|
2344
|
+
if (unfilled.length > 0) {
|
|
2345
|
+
console.error(`Agent briefs with unfilled template variables:\n ${unfilled.join("\n ")}`);
|
|
2346
|
+
}
|
|
2347
|
+
expect(unfilled).toEqual([]);
|
|
2348
|
+
});
|
|
2349
|
+
|
|
2350
|
+
test("AGENT-6: All file paths referenced in briefs exist", () => {
|
|
2351
|
+
if (!existsSync(agentsDir)) return;
|
|
2352
|
+
const broken: string[] = [];
|
|
2353
|
+
for (const f of readdirSync(agentsDir).filter(f => f.endsWith(".md"))) {
|
|
2354
|
+
const content = readFileSync(join(agentsDir, f), "utf-8");
|
|
2355
|
+
const refs = content.match(/node_modules\/[^\s`*)"']+/g) || [];
|
|
2356
|
+
for (const ref of refs) {
|
|
2357
|
+
const clean = ref.replace(/[`*)"']+$/, "");
|
|
2358
|
+
if (!existsSync(join(root, clean))) {
|
|
2359
|
+
broken.push(`${f} → ${clean}`);
|
|
2360
|
+
addFinding({
|
|
2361
|
+
ruleId: "AGENT-BROKEN-REF",
|
|
2362
|
+
severity: "FAIL",
|
|
2363
|
+
file: `.claude/agents/${f}`,
|
|
2364
|
+
message: `Brief references "${clean}" but file does not exist`,
|
|
2365
|
+
fixCommand: `Check prompts/ directory for correct filename — may be a typo or renamed file`,
|
|
2366
|
+
});
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
if (broken.length > 0) {
|
|
2371
|
+
console.error(`Broken references in agent briefs:\n ${broken.join("\n ")}`);
|
|
2372
|
+
}
|
|
2373
|
+
expect(broken).toEqual([]);
|
|
2374
|
+
});
|
|
2375
|
+
});
|
|
2376
|
+
}
|
|
2377
|
+
|
|
2378
|
+
// ── SC-294: Directory validation ───────────────────────────
|
|
2379
|
+
|
|
2380
|
+
export function runDirectoryValidation(root: string) {
|
|
2381
|
+
const specsDir = join(root, "specs");
|
|
2382
|
+
|
|
2383
|
+
describe("Directory Validation (SC-294)", () => {
|
|
2384
|
+
test("SC-294: Every specs/ subdirectory has a parent spec file", () => {
|
|
2385
|
+
if (!existsSync(specsDir)) return;
|
|
2386
|
+
|
|
2387
|
+
const invalid: string[] = [];
|
|
2388
|
+
|
|
2389
|
+
// Find all subdirectories in specs/
|
|
2390
|
+
const entries = readdirSync(specsDir);
|
|
2391
|
+
const subdirs = entries.filter(entry => {
|
|
2392
|
+
const fullPath = join(specsDir, entry);
|
|
2393
|
+
try {
|
|
2394
|
+
return statSync(fullPath).isDirectory();
|
|
2395
|
+
} catch {
|
|
2396
|
+
return false;
|
|
2397
|
+
}
|
|
2398
|
+
});
|
|
2399
|
+
|
|
2400
|
+
// For each subdirectory, check if there's a corresponding parent spec
|
|
2401
|
+
for (const subdir of subdirs) {
|
|
2402
|
+
// Find all .md files in specs/
|
|
2403
|
+
const specFiles = entries.filter(f => f.endsWith(".md"));
|
|
2404
|
+
|
|
2405
|
+
// Check if any spec file derives to this directory name
|
|
2406
|
+
let found = false;
|
|
2407
|
+
for (const specFile of specFiles) {
|
|
2408
|
+
const derived = deriveDirectoryName(specFile);
|
|
2409
|
+
if (derived === subdir) {
|
|
2410
|
+
found = true;
|
|
2411
|
+
break;
|
|
2412
|
+
}
|
|
2413
|
+
}
|
|
2414
|
+
|
|
2415
|
+
if (!found) {
|
|
2416
|
+
invalid.push(`${subdir}/: no parent spec derives to this directory name`);
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
|
|
2420
|
+
if (invalid.length > 0) {
|
|
2421
|
+
console.error(`Hand-created directories (SC-294 violation):\n ${invalid.join("\n ")}\n\nDirectories MUST be created by split-spec.ts, not manually.`);
|
|
2422
|
+
}
|
|
2423
|
+
expect(invalid).toEqual([]);
|
|
2424
|
+
});
|
|
773
2425
|
});
|
|
774
2426
|
}
|
|
775
2427
|
|
|
@@ -786,7 +2438,38 @@ interface FallowResult {
|
|
|
786
2438
|
workspace_diagnostics: Array<{ kind: string; message: string }>;
|
|
787
2439
|
}
|
|
788
2440
|
|
|
2441
|
+
// `fallow` is an optional external tool and is NOT a declared dependency. When
|
|
2442
|
+
// it is absent, `npx fallow` does a registry fetch on every invocation — four
|
|
2443
|
+
// tests x one fetch each, each outliving bun's per-test timeout. The tests then
|
|
2444
|
+
// fail as timeouts instead of skipping, which blocks the pre-push conformity
|
|
2445
|
+
// gate on a tool nobody installed.
|
|
2446
|
+
//
|
|
2447
|
+
// Resolve it locally instead, with no network path. Memoised PER ROOT —
|
|
2448
|
+
// runFallowCheck is exported and consumer projects call it with their own
|
|
2449
|
+
// root, so a single process can ask about several. A root-independent cache
|
|
2450
|
+
// would answer for the wrong directory.
|
|
2451
|
+
const FALLOW_AVAILABLE = new Map<string, boolean>();
|
|
2452
|
+
|
|
2453
|
+
function fallowAvailable(root: string): boolean {
|
|
2454
|
+
const cached = FALLOW_AVAILABLE.get(root);
|
|
2455
|
+
if (cached !== undefined) return cached;
|
|
2456
|
+
const available =
|
|
2457
|
+
existsSync(join(root, "node_modules", ".bin", "fallow")) ||
|
|
2458
|
+
Bun.which("fallow") !== null;
|
|
2459
|
+
FALLOW_AVAILABLE.set(root, available);
|
|
2460
|
+
return available;
|
|
2461
|
+
}
|
|
2462
|
+
|
|
2463
|
+
/** Shared skip notice — a skipped check must say so, not read as a pass. */
|
|
2464
|
+
function fallowSkipped(check: string): void {
|
|
2465
|
+
console.warn(
|
|
2466
|
+
`SKIP ${check}: fallow is not installed (not a declared dependency) — ` +
|
|
2467
|
+
`this check did NOT run. Install it to enable static analysis.`,
|
|
2468
|
+
);
|
|
2469
|
+
}
|
|
2470
|
+
|
|
789
2471
|
function runFallowCommand(root: string, args: string[]): FallowResult | null {
|
|
2472
|
+
if (!fallowAvailable(root)) return null;
|
|
790
2473
|
const result = Bun.spawnSync(["npx", "fallow", ...args, "--format", "json", "--quiet"], {
|
|
791
2474
|
cwd: root,
|
|
792
2475
|
env: { ...process.env, NODE_NO_WARNINGS: "1" },
|
|
@@ -806,10 +2489,7 @@ export function runFallowCheck(root: string, opts?: { skipUnusedExports?: boolea
|
|
|
806
2489
|
describe("Fallow: static analysis", () => {
|
|
807
2490
|
test("FALLOW-1: No unused files", () => {
|
|
808
2491
|
const result = runFallowCommand(root, ["dead-code", "--unused-files"]);
|
|
809
|
-
if (!result) {
|
|
810
|
-
console.warn("Fallow not available or failed to run — skipping");
|
|
811
|
-
return;
|
|
812
|
-
}
|
|
2492
|
+
if (!result) { fallowSkipped("FALLOW-1 (unused files)"); return; }
|
|
813
2493
|
const unused = result.unused_files.map(f => f.path).filter(p => !p.includes("test/fixtures/") && !p.includes("reference/"));
|
|
814
2494
|
if (unused.length > 0) {
|
|
815
2495
|
console.warn(`Unused files (${unused.length}):\n ${unused.slice(0, 10).join("\n ")}${unused.length > 10 ? `\n ... and ${unused.length - 10} more` : ""}`);
|
|
@@ -820,7 +2500,7 @@ export function runFallowCheck(root: string, opts?: { skipUnusedExports?: boolea
|
|
|
820
2500
|
if (!opts?.skipUnusedExports) {
|
|
821
2501
|
test("FALLOW-2: No unused exports", () => {
|
|
822
2502
|
const result = runFallowCommand(root, ["dead-code", "--unused-exports"]);
|
|
823
|
-
if (!result) return;
|
|
2503
|
+
if (!result) { fallowSkipped("FALLOW-2 (unused exports)"); return; }
|
|
824
2504
|
const unused = (result.unused_exports || [])
|
|
825
2505
|
.filter(e => !e.path.includes("test/fixtures/") && !e.path.includes("reference/"))
|
|
826
2506
|
.map(e => `${e.path}:${e.line ?? "?"} ${e.export_name}`);
|
|
@@ -833,7 +2513,7 @@ export function runFallowCheck(root: string, opts?: { skipUnusedExports?: boolea
|
|
|
833
2513
|
|
|
834
2514
|
test("FALLOW-3: No unused dependencies", () => {
|
|
835
2515
|
const result = runFallowCommand(root, ["dead-code", "--unused-deps"]);
|
|
836
|
-
if (!result) return;
|
|
2516
|
+
if (!result) { fallowSkipped("FALLOW-3 (unused dependencies)"); return; }
|
|
837
2517
|
const unused = result.unused_dependencies?.map(d => d.name) || [];
|
|
838
2518
|
if (unused.length > 0) {
|
|
839
2519
|
console.warn(`Unused dependencies (${unused.length}):\n ${unused.join("\n ")}`);
|
|
@@ -843,10 +2523,10 @@ export function runFallowCheck(root: string, opts?: { skipUnusedExports?: boolea
|
|
|
843
2523
|
|
|
844
2524
|
test("FALLOW-4: No circular dependencies", () => {
|
|
845
2525
|
const result = runFallowCommand(root, ["dead-code"]);
|
|
846
|
-
if (!result) return;
|
|
2526
|
+
if (!result) { fallowSkipped("FALLOW-4 (circular dependencies)"); return; }
|
|
847
2527
|
const circles = result.circular_dependencies || [];
|
|
848
2528
|
if (circles.length > 0) {
|
|
849
|
-
console.warn(`Circular dependencies (${circles.length}):\n ${circles.slice(0, 5).map(c => c.path || JSON.stringify(c.
|
|
2529
|
+
console.warn(`Circular dependencies (${circles.length}):\n ${circles.slice(0, 5).map(c => c.path || JSON.stringify(c.chain ?? c)).join("\n ")}${circles.length > 5 ? `\n ... and ${circles.length - 5} more` : ""}`);
|
|
850
2530
|
}
|
|
851
2531
|
if (fail) expect(circles).toEqual([]);
|
|
852
2532
|
});
|
|
@@ -872,6 +2552,13 @@ export function runPackageValidation(root: string) {
|
|
|
872
2552
|
const pkgPath = join(root, "package.json");
|
|
873
2553
|
if (!existsSync(pkgPath)) return;
|
|
874
2554
|
const pkg = JSON.parse(readFileSync(pkgPath, "utf-8"));
|
|
2555
|
+
// The harness is not a consumer of itself. npm has no way to express
|
|
2556
|
+
// a self-dependency and would be wrong to: `rungate` IS the thing
|
|
2557
|
+
// this check asks every other project to depend on. Surfaced by #208
|
|
2558
|
+
// — once rungate's own conformity test stopped being frozen by
|
|
2559
|
+
// `safeWrite`, this check ran here for the first time and failed on a
|
|
2560
|
+
// requirement it is not possible to satisfy.
|
|
2561
|
+
if (isHarnessItself(root)) return;
|
|
875
2562
|
expect(pkg.devDependencies?.rungate).toBeDefined();
|
|
876
2563
|
});
|
|
877
2564
|
});
|
|
@@ -945,7 +2632,7 @@ export function runAbsenceValidation(root: string) {
|
|
|
945
2632
|
test("ABSENCE-1: anti-criteria use absence verification", () => {
|
|
946
2633
|
const specsDir = join(root, "specs");
|
|
947
2634
|
if (!existsSync(specsDir)) return;
|
|
948
|
-
for (const f of
|
|
2635
|
+
for (const f of collectSpecFiles(specsDir)) { // #225 — subdirectories too
|
|
949
2636
|
const content = readFileSync(join(specsDir, f), "utf-8");
|
|
950
2637
|
const antiCriteria = content.match(/^- \[ \] SC-A\d+:.+$/gm) || [];
|
|
951
2638
|
for (const ac of antiCriteria) {
|