rungate 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ac-quality.ts +119 -0
- package/lib/config-loader.ts +129 -5
- package/lib/conformity.ts +157 -1
- package/lib/context-docs.ts +182 -0
- package/lib/create-brief.ts +22 -17
- package/lib/create-sc.ts +35 -8
- package/lib/doc-scan-targets.ts +435 -0
- package/lib/gate-check-audit.ts +9 -9
- package/lib/organize.ts +129 -40
- package/lib/package-closure.ts +111 -0
- package/lib/project-aliases.ts +100 -0
- package/lib/release-preflight.ts +156 -48
- package/lib/rungate-schema.ts +23 -1
- package/lib/sc-cover.ts +92 -1
- package/lib/sc-ids.ts +151 -0
- package/lib/scaffold/rehearsal.ts +330 -0
- package/lib/scaffold/steps.ts +133 -21
- package/lib/scaffold/write-steps.ts +167 -0
- package/lib/scratch-cleanup.ts +198 -0
- package/lib/scratch-roots.ts +94 -0
- package/lib/validators/spec-validators.ts +23 -34
- package/lib/workflow-security.ts +51 -7
- package/lib/worktree-cleanup.ts +123 -7
- package/lib/worktree-repo.ts +165 -0
- package/package.json +5 -1
- package/scripts/audit-specs.ts +134 -37
- package/scripts/audit-transcript.ts +259 -0
- package/scripts/create-sc.ts +15 -1
- package/scripts/detect-sc-drift.ts +200 -0
- package/scripts/extract-constraints.ts +16 -36
- package/scripts/scaffold-project.ts +26 -136
- package/scripts/split-spec.ts +315 -0
- package/scripts/sync-sc-status.ts +751 -0
|
@@ -0,0 +1,751 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Sync SC checkbox status: test passes → spec checkbox flips from [ ] to [x].
|
|
4
|
+
*
|
|
5
|
+
* 1. Scans spec files for unchecked SCs: - [ ] SC-NNN: ...
|
|
6
|
+
* 2. Scans test files for matching SC-NNN references
|
|
7
|
+
* 3. Runs only the test files that contain unchecked SCs
|
|
8
|
+
* 4. For SCs without hand-written tests, runs matchPattern() conformity assertions
|
|
9
|
+
* 5. If assertions pass, flips matching checkboxes to [x]
|
|
10
|
+
*
|
|
11
|
+
* Usage:
|
|
12
|
+
* bun scripts/sync-sc-status.ts # Run targeted tests and flip checkboxes
|
|
13
|
+
* bun scripts/sync-sc-status.ts --dry-run # Show what would flip without changing files
|
|
14
|
+
* bun scripts/sync-sc-status.ts --report # Output per-spec coverage report
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { existsSync, readFileSync, writeFileSync, readdirSync } from "fs";
|
|
18
|
+
import { commentMask, enclosingComment, stripJsComments } from "../lib/strip-js-comments";
|
|
19
|
+
import { commentClaimForm, type ClaimForm } from "../lib/sc-claim-forms";
|
|
20
|
+
import { join } from "path";
|
|
21
|
+
import { execSync } from "child_process";
|
|
22
|
+
import { matchPattern, isBehavioralSC, type ParsedSC } from "../lib/conformity";
|
|
23
|
+
import { readFreshCache } from "../lib/behavioral-cache";
|
|
24
|
+
import { detectDrift } from "./detect-sc-drift";
|
|
25
|
+
import {
|
|
26
|
+
ambiguityOf,
|
|
27
|
+
buildScRegistry,
|
|
28
|
+
isQualifiedBy,
|
|
29
|
+
isTemplateSpec,
|
|
30
|
+
oneScMentionPattern,
|
|
31
|
+
parseScDeclarations,
|
|
32
|
+
requiresQualification,
|
|
33
|
+
scDeclarationPattern,
|
|
34
|
+
scMentionPattern as scIdMentionPattern,
|
|
35
|
+
type ScRegistry,
|
|
36
|
+
} from "../lib/sc-ids";
|
|
37
|
+
|
|
38
|
+
export { isTemplateSpec };
|
|
39
|
+
|
|
40
|
+
const ROOT = join(import.meta.dir, "..");
|
|
41
|
+
const SPECS_DIR = join(ROOT, "specs");
|
|
42
|
+
const TEST_DIR = join(ROOT, "test");
|
|
43
|
+
const dryRun = process.argv.includes("--dry-run");
|
|
44
|
+
const reportMode = process.argv.includes("--report");
|
|
45
|
+
|
|
46
|
+
export interface UncheckedSC {
|
|
47
|
+
id: string;
|
|
48
|
+
statement?: string;
|
|
49
|
+
specFile: string;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The registry every credit decision is made against (#330).
|
|
54
|
+
*
|
|
55
|
+
* Built from the same spec scan `findUncheckedSCs` walks, so the ids a test
|
|
56
|
+
* can be credited for and the ids a spec declares are the same set by
|
|
57
|
+
* construction rather than by two regexes agreeing.
|
|
58
|
+
*/
|
|
59
|
+
export function readScRegistry(specsDir?: string): ScRegistry {
|
|
60
|
+
const dir = specsDir || SPECS_DIR;
|
|
61
|
+
if (!existsSync(dir)) return buildScRegistry([]);
|
|
62
|
+
const specs: Array<{ file: string; content: string }> = [];
|
|
63
|
+
for (const file of new Bun.Glob("**/*.md").scanSync({ cwd: dir, absolute: false })) {
|
|
64
|
+
specs.push({ file, content: readFileSync(join(dir, file), "utf-8") });
|
|
65
|
+
}
|
|
66
|
+
return buildScRegistry(specs);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function findUncheckedSCs(specsDir?: string): UncheckedSC[] {
|
|
70
|
+
const dir = specsDir || SPECS_DIR;
|
|
71
|
+
const unchecked: UncheckedSC[] = [];
|
|
72
|
+
if (!existsSync(dir)) return unchecked;
|
|
73
|
+
|
|
74
|
+
for (const file of new Bun.Glob("**/*.md").scanSync({ cwd: dir, absolute: false })) {
|
|
75
|
+
if (isTemplateSpec(file)) continue;
|
|
76
|
+
const content = readFileSync(join(dir, file), "utf-8");
|
|
77
|
+
// #330: was `/^- \[ \] (SC-\d+):/`, the fourth copy of this pattern in the
|
|
78
|
+
// tree and the second that could not see an anti-criterion. One owner now.
|
|
79
|
+
for (const match of content.matchAll(scDeclarationPattern())) {
|
|
80
|
+
if (match[1].toLowerCase() === "x") continue;
|
|
81
|
+
unchecked.push({ id: match[2], statement: match[3].trim(), specFile: file });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return unchecked;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Does this text name this SC, as opposed to a longer one starting with it?
|
|
89
|
+
*
|
|
90
|
+
* `content.includes("SC-1")` is true of any file mentioning `SC-144`, so
|
|
91
|
+
* eighteen test files were credited as covering `SC-1` and none of them
|
|
92
|
+
* mentioned it (#148). The trailing boundary is "not another digit" rather
|
|
93
|
+
* than `\b`, because `SC-1` is followed by `:`, `,`, `)` or end-of-line in
|
|
94
|
+
* every real reference and `\b` would accept `SC-1` inside `SC-1A` — but a
|
|
95
|
+
* digit is the only character that makes it a different ID.
|
|
96
|
+
*
|
|
97
|
+
* #330 moved the definition to `lib/sc-ids.ts` and widened the boundary to
|
|
98
|
+
* "not a word character and not a hyphen", so `SC-1` also stops matching
|
|
99
|
+
* inside `SC-1-draft`. Re-exported here because callers import it from this
|
|
100
|
+
* module; the body lives in one place.
|
|
101
|
+
*/
|
|
102
|
+
export const scMentionPattern = oneScMentionPattern;
|
|
103
|
+
|
|
104
|
+
/** Opens a `describe`/`test`/`it` title, allowing `.each(...)`, `.only`, `.skip`. */
|
|
105
|
+
const TITLE_CALL = /\b(?:describe|test|it)\s*(?:\.\w+\s*(?:\([^)]*\))?\s*)*\(\s*$/;
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The success criteria a source file CLAIMS to cover (#149).
|
|
109
|
+
*
|
|
110
|
+
* The old rule was "the file mentions the id". Tests of the spec tooling embed
|
|
111
|
+
* spec lines and SC records as INPUT — they assert how an SC is parsed,
|
|
112
|
+
* flipped or scanned, and nothing about the criterion the id names — so a
|
|
113
|
+
* mention was never evidence of coverage. #148 removed the two worst shapes by
|
|
114
|
+
* name with a pair of regexes; this replaces the convention, which is what
|
|
115
|
+
* that fix deferred here.
|
|
116
|
+
*
|
|
117
|
+
* ## The rule
|
|
118
|
+
*
|
|
119
|
+
* One question, not a list of shapes: **is the id the whole of what was
|
|
120
|
+
* written, or is it embedded in something else?**
|
|
121
|
+
*
|
|
122
|
+
* claim a mention in a comment, or in a describe/test/it title
|
|
123
|
+
* data everything else — any id inside a string literal
|
|
124
|
+
*
|
|
125
|
+
* A list of forbidden shapes is a second place to forget, and forgetting there
|
|
126
|
+
* reads as the mention being a claim, which is the failure direction that
|
|
127
|
+
* flatters us. An allowlist of two declaring positions cannot fail that way:
|
|
128
|
+
* a shape nobody thought of is not credited.
|
|
129
|
+
*
|
|
130
|
+
* ## Why not "a complete string literal also counts"
|
|
131
|
+
*
|
|
132
|
+
* Because it cannot be made safe, and the attempt is worth recording. A
|
|
133
|
+
* criterion registry is real coverage:
|
|
134
|
+
*
|
|
135
|
+
* const EXPECTED_REREVIEW_SCS = ["SC-621", "SC-622", …]; // coverage
|
|
136
|
+
*
|
|
137
|
+
* but so is this, and it is a test PARAMETER naming the criterion a bug was
|
|
138
|
+
* about:
|
|
139
|
+
*
|
|
140
|
+
* const SHORTEST_ID = "SC-1"; // not coverage
|
|
141
|
+
* const scs = [{ id: "SC-1", done: true }]; // not coverage
|
|
142
|
+
* findTestFilesForSCs(new Set(["SC-9", "SC-92"]), dir); // not coverage
|
|
143
|
+
*
|
|
144
|
+
* All four are bare ids in quotes. Nothing syntactic separates them — not
|
|
145
|
+
* `id:`, not array-vs-scalar, not the variable's name. A rule that credited
|
|
146
|
+
* them would certify criteria nobody tested, and that is the direction this
|
|
147
|
+
* project's own `.claude/rules/checks-must-be-able-to-fail.md` says never to
|
|
148
|
+
* fail in.
|
|
149
|
+
*
|
|
150
|
+
* So a registry declares itself with a comment. That costs one line, it is
|
|
151
|
+
* already the convention most files follow, and unlike an inference it cannot
|
|
152
|
+
* be wrong in the flattering direction.
|
|
153
|
+
*
|
|
154
|
+
* ## Why a bare literal counts
|
|
155
|
+
*
|
|
156
|
+
* #149's body proposed titles and comments only. Measured against the tree,
|
|
157
|
+
* that demotes `const EXPECTED_REREVIEW_SCS = ["SC-621", …]` in
|
|
158
|
+
* test/harness-standard-security.test.ts — a criterion registry driving a
|
|
159
|
+
* per-criterion loop, which .claude/rules/checks-must-be-able-to-fail.md holds
|
|
160
|
+
* up as the worked example for that spec. It is a BETTER coverage mechanism
|
|
161
|
+
* than a hand-written title because the ids cannot drift from the assertions,
|
|
162
|
+
* and a rule that dropped it would push authors toward the weaker pattern.
|
|
163
|
+
*
|
|
164
|
+
* ## Why it is built on stripJsComments rather than on lines
|
|
165
|
+
*
|
|
166
|
+
* The three categories have to be told apart at the right level, and a
|
|
167
|
+
* line-oriented detector cannot do it. `fixtureTestDir("x", "// covers SC-92")`
|
|
168
|
+
* contains a comment that is STRING DATA: a rule asking "does this line start
|
|
169
|
+
* with //" is fooled by its own test fixture. stripJsComments is an
|
|
170
|
+
* offset-preserving scanner over exactly these contexts, so two views of the
|
|
171
|
+
* same source — comments blanked, and comments plus literal CONTENTS blanked —
|
|
172
|
+
* classify every character by subtraction, with no second lexer to drift.
|
|
173
|
+
*/
|
|
174
|
+
export function coverageClaimIds(source: string): Set<string> {
|
|
175
|
+
return new Set(coverageClaimSites(source).map(c => c.id));
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* The same scan, keeping WHERE each claim was written (#330).
|
|
180
|
+
*
|
|
181
|
+
* An ambiguous id — an anti-criterion, a collision, or a number an issue's
|
|
182
|
+
* own criteria also use — is creditable only when the surrounding text names
|
|
183
|
+
* the spec, and "surrounding" needs the offset.
|
|
184
|
+
*/
|
|
185
|
+
export interface ClaimSite {
|
|
186
|
+
id: string;
|
|
187
|
+
at: number;
|
|
188
|
+
/**
|
|
189
|
+
* Where it was written. A comment is a claim for an unambiguous id and is
|
|
190
|
+
* NOT enough for an ambiguous one — see `findTestFilesForSCs`.
|
|
191
|
+
*/
|
|
192
|
+
kind: "comment" | "title" | "bare";
|
|
193
|
+
/**
|
|
194
|
+
* For a comment, WHICH structural declaring form it occupies, or `null` for
|
|
195
|
+
* a bare mention inside running prose (#360). Always `null` for the other
|
|
196
|
+
* two kinds, which are declaring positions by construction.
|
|
197
|
+
*
|
|
198
|
+
* Reported rather than filtered: the credit rule reads this, and
|
|
199
|
+
* `unqualifiedClaims` reports what the credit rule refused.
|
|
200
|
+
*/
|
|
201
|
+
form: ClaimForm | null;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export function coverageClaimSites(source: string): ClaimSite[] {
|
|
205
|
+
const noComments = stripJsComments(source);
|
|
206
|
+
const codeOnly = stripJsComments(source, { blankLiterals: true });
|
|
207
|
+
const mask = commentMask(source);
|
|
208
|
+
const claims: ClaimSite[] = [];
|
|
209
|
+
const add = (id: string, at: number, kind: ClaimSite["kind"], form: ClaimForm | null = null) =>
|
|
210
|
+
claims.push({ id, at, kind, form });
|
|
211
|
+
|
|
212
|
+
for (const m of source.matchAll(scIdMentionPattern())) {
|
|
213
|
+
const id = m[0];
|
|
214
|
+
const at = m.index!;
|
|
215
|
+
|
|
216
|
+
// Blanked by the first pass => the id sits in a comment. Offsets are
|
|
217
|
+
// preserved, so this is an exact answer rather than a heuristic.
|
|
218
|
+
if (noComments[at] !== source[at]) {
|
|
219
|
+
// #360 records the FORM here and credits on it downstream, rather than
|
|
220
|
+
// dropping the site. `unqualifiedClaims` exists so an ambiguous mention
|
|
221
|
+
// that failed to earn a credit is reported instead of vanishing (#358);
|
|
222
|
+
// filtering here would have re-created that silent drop one layer down,
|
|
223
|
+
// and did — it turned "an uncredited ambiguous mention is reported" red.
|
|
224
|
+
const { text, offset } = enclosingComment(source, mask, at);
|
|
225
|
+
add(id, at, "comment", commentClaimForm(text, offset, id));
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
// Survives the first pass but not the second => inside a literal. Code
|
|
230
|
+
// context is neither (and `SC-7` is not a legal identifier anyway), so it
|
|
231
|
+
// can only be something like a stray token; count it rather than guess.
|
|
232
|
+
if (codeOnly[at] === source[at]) {
|
|
233
|
+
add(id, at, "bare");
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
// Inside a literal. The only literal that declares coverage is a test
|
|
238
|
+
// title — the project's oldest convention, and prose by definition, so no
|
|
239
|
+
// rule about the literal's contents can apply to it. Walk out to the
|
|
240
|
+
// opening delimiter, which survives `blankLiterals`; that is the property
|
|
241
|
+
// that makes the boundary findable without a second parser.
|
|
242
|
+
let open = at;
|
|
243
|
+
while (open > 0 && !/['"`]/.test(codeOnly[open])) open--;
|
|
244
|
+
if (!/['"`]/.test(codeOnly[open] ?? "")) continue;
|
|
245
|
+
|
|
246
|
+
if (TITLE_CALL.test(codeOnly.slice(Math.max(0, open - 120), open))) add(id, at, "title");
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
return claims;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** A credit, scoped to the criterion it actually names. */
|
|
253
|
+
export interface ScCredit {
|
|
254
|
+
id: string;
|
|
255
|
+
specFile: string;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Which test file covers which criterion — spec-scoped (#330, #352, #358).
|
|
260
|
+
*
|
|
261
|
+
* This used to return bare ids, and the flipper then ticked that id in EVERY
|
|
262
|
+
* spec declaring it (`unchecked.filter(u => u.id === scId)`). With one id per
|
|
263
|
+
* spec that was harmless. With `SC-A1` declared by three specs it is not: one
|
|
264
|
+
* passing test about doc hygiene would tick an unrelated anti-criterion in
|
|
265
|
+
* the GitHub API migration spec. The narrow `SC-\d+` pattern was the only
|
|
266
|
+
* thing hiding it, which is why #330 said widening the regex without fixing
|
|
267
|
+
* scoping makes the measurement worse.
|
|
268
|
+
*
|
|
269
|
+
* An id whose bare mention does not identify a criterion is credited only
|
|
270
|
+
* when the text around it names the spec. That is the whole of the fix for
|
|
271
|
+
* three issues:
|
|
272
|
+
*
|
|
273
|
+
* - `SC-A1` in `ceremony-overrides.test.ts` names no spec, and no spec
|
|
274
|
+
* declares those criteria — they are an issue's own. Not credited.
|
|
275
|
+
* - `SC-6` in `context-preload-parity.test.ts` is #71's sixth criterion, not
|
|
276
|
+
* `bootstrap-data-flow`'s. Not credited.
|
|
277
|
+
* - `SC-640` in `harness-standard-security.test.ts` is unique in the
|
|
278
|
+
* registry, so it needs no qualifier and keeps its credit.
|
|
279
|
+
*
|
|
280
|
+
* ## Why a qualified COMMENT is still not enough
|
|
281
|
+
*
|
|
282
|
+
* The first version of this accepted any mention with the spec's name
|
|
283
|
+
* nearby, and gained two credits it should not have. `behavioral-sc-ratchet
|
|
284
|
+
* .test.ts` opens with prose about a past bug:
|
|
285
|
+
*
|
|
286
|
+
* used `SC-[0-9]+` and SC ids are not always numeric — `SC-A1` and `SC-A2`
|
|
287
|
+
* in GITHUB-API-MIGRATION-SPEC.md were missed.
|
|
288
|
+
*
|
|
289
|
+
* Every ingredient of a qualified claim, and it is a citation. A window says
|
|
290
|
+
* WHICH criterion a mention means; it cannot say WHETHER the mention is a
|
|
291
|
+
* claim, and #149 already decided that a bare comment counts as one — which
|
|
292
|
+
* is fine for an id only this file would write and wrong for an id prose
|
|
293
|
+
* cites constantly.
|
|
294
|
+
*
|
|
295
|
+
* So an ambiguous id must be claimed from a `describe`/`test`/`it` title AND
|
|
296
|
+
* name its spec. That is the strongest position the convention has, it is
|
|
297
|
+
* where a real anti-criterion test would put it anyway, and prose cannot
|
|
298
|
+
* reach it by accident. The cost is that six anti-criteria stay uncredited
|
|
299
|
+
* until someone writes such a title — under-crediting, which is the
|
|
300
|
+
* direction this is allowed to fail in.
|
|
301
|
+
*/
|
|
302
|
+
export function findTestFilesForSCs(
|
|
303
|
+
scs: Array<{ id: string; specFile: string }>,
|
|
304
|
+
reg: ScRegistry,
|
|
305
|
+
testDir?: string,
|
|
306
|
+
): Map<string, ScCredit[]> {
|
|
307
|
+
const dir = testDir || TEST_DIR;
|
|
308
|
+
const out = new Map<string, ScCredit[]>();
|
|
309
|
+
if (!existsSync(dir)) return out;
|
|
310
|
+
|
|
311
|
+
const wanted = new Map<string, string[]>();
|
|
312
|
+
for (const sc of scs) {
|
|
313
|
+
const specs = wanted.get(sc.id) ?? [];
|
|
314
|
+
if (!specs.includes(sc.specFile)) specs.push(sc.specFile);
|
|
315
|
+
wanted.set(sc.id, specs);
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
for (const file of readdirSync(dir).filter(f => f.endsWith(".test.ts"))) {
|
|
319
|
+
const source = readFileSync(join(dir, file), "utf-8");
|
|
320
|
+
const credits: ScCredit[] = [];
|
|
321
|
+
for (const { id, at, kind, form } of coverageClaimSites(source)) {
|
|
322
|
+
// #360. A comment earns a credit only in a structural declaring form.
|
|
323
|
+
// The position was never the question — a comment is equally where this
|
|
324
|
+
// project declares coverage and where it explains itself, and a bare id
|
|
325
|
+
// inside a running sentence is a citation.
|
|
326
|
+
if (kind === "comment" && !form) continue;
|
|
327
|
+
for (const specFile of wanted.get(id) ?? []) {
|
|
328
|
+
if (requiresQualification(id, reg) && !(kind === "title" && isQualifiedBy(source, at, specFile))) continue;
|
|
329
|
+
if (!credits.some(c => c.id === id && c.specFile === specFile)) {
|
|
330
|
+
credits.push({ id, specFile });
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
if (credits.length > 0) out.set(file, credits);
|
|
335
|
+
}
|
|
336
|
+
return out;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Mentions that look like claims and are not credited, with why (#330).
|
|
341
|
+
*
|
|
342
|
+
* Reported rather than silently dropped. Under-crediting is the safe
|
|
343
|
+
* direction and it is still a measurement error, so the number has to be
|
|
344
|
+
* visible or nobody will ever qualify a mention.
|
|
345
|
+
*/
|
|
346
|
+
export function unqualifiedClaims(
|
|
347
|
+
source: string,
|
|
348
|
+
reg: ScRegistry,
|
|
349
|
+
): Array<{ id: string; reason: string }> {
|
|
350
|
+
const out: Array<{ id: string; reason: string }> = [];
|
|
351
|
+
const sites = coverageClaimSites(source);
|
|
352
|
+
|
|
353
|
+
// Which ids this file DOES claim properly, settled before anything is
|
|
354
|
+
// reported (#392). `findTestFilesForSCs` has always credited a file that
|
|
355
|
+
// cites an id in one place and declares it in another — "one file doing
|
|
356
|
+
// both is credited once, by the claim" — and this reporter did not: the
|
|
357
|
+
// first non-title mention won, so a correctly qualified `describe` title
|
|
358
|
+
// was reported as unqualified because a comment two lines above it named
|
|
359
|
+
// the same id. Site order decided the answer, which means a comment ABOVE
|
|
360
|
+
// the title reported and the same comment BELOW it did not.
|
|
361
|
+
const qualified = new Set(
|
|
362
|
+
sites
|
|
363
|
+
.filter(s => s.kind === "title")
|
|
364
|
+
.filter(s => (reg.declaredById.get(s.id) ?? []).some(f => isQualifiedBy(source, s.at, f)))
|
|
365
|
+
.map(s => s.id),
|
|
366
|
+
);
|
|
367
|
+
|
|
368
|
+
for (const { id, at, kind } of sites) {
|
|
369
|
+
if (!requiresQualification(id, reg)) continue;
|
|
370
|
+
if (qualified.has(id)) continue;
|
|
371
|
+
// Every declarer, so the reason names the spec whose HEADING made the id
|
|
372
|
+
// ambiguous (#392). Listing only the checkbox holders would report
|
|
373
|
+
// "declared by SCAFFOLD-DECOMPOSITION-SPEC.md" as the reason a mention
|
|
374
|
+
// needs qualifying against it, which reads as nonsense.
|
|
375
|
+
const specs = reg.declaredById.get(id) ?? [];
|
|
376
|
+
if (kind === "title" && specs.some(s => isQualifiedBy(source, at, s))) continue;
|
|
377
|
+
if (out.some(o => o.id === id)) continue;
|
|
378
|
+
out.push({
|
|
379
|
+
id,
|
|
380
|
+
reason:
|
|
381
|
+
`${ambiguityOf(id, reg)} — declared by ${specs.join(", ")}. To claim it, name the ` +
|
|
382
|
+
`spec in a describe/test/it title beside the id. If this is an issue's own ` +
|
|
383
|
+
`criterion, write it as "criterion N of #NNN" — SC- is the spec registry's.`,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
return out;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function runTestFile(file: string): boolean {
|
|
390
|
+
try {
|
|
391
|
+
execSync(`bun test test/${file}`, {
|
|
392
|
+
cwd: ROOT, timeout: 120000, stdio: ["ignore", "pipe", "pipe"]
|
|
393
|
+
});
|
|
394
|
+
return true;
|
|
395
|
+
} catch (e: any) {
|
|
396
|
+
const out = (e.stdout || "") + (e.stderr || "");
|
|
397
|
+
const failMatch = out.match(/(\d+) fail/);
|
|
398
|
+
return failMatch ? parseInt(failMatch[1]) === 0 : false;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
export interface ConformityResult {
|
|
403
|
+
passing: Set<string>;
|
|
404
|
+
failing: Set<string>;
|
|
405
|
+
unmatchable: Set<string>;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Composite key scoped to spec file — prevents cross-spec ID collisions.
|
|
410
|
+
*/
|
|
411
|
+
export function scopedKey(specFile: string, id: string): string {
|
|
412
|
+
return `${specFile}::${id}`;
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Every `specFile::id` a test file declares, per
|
|
417
|
+
* `.claude/rules/claiming-sc-coverage.md` (#441).
|
|
418
|
+
*
|
|
419
|
+
* ONE owner. This was written out longhand in `test/ticked-sc-cover.test.ts`
|
|
420
|
+
* and #441 needed the same set in a second place; copying it would have put
|
|
421
|
+
* two implementations of the credit rule in the tree, which is the way this
|
|
422
|
+
* class of bug arrives and the reason `uncoveredTickedScs` takes `claimed`
|
|
423
|
+
* as a parameter rather than computing it.
|
|
424
|
+
*
|
|
425
|
+
* It lives here rather than in `lib/` because the credit rule lives here —
|
|
426
|
+
* `findTestFilesForSCs` and `commentClaimForm` are what decide, and a `lib/`
|
|
427
|
+
* module reaching back into `scripts/` would invert the dependency.
|
|
428
|
+
*/
|
|
429
|
+
export function claimedScKeys(specsDir: string, testDir?: string): Set<string> {
|
|
430
|
+
const specs = readdirSync(specsDir, { recursive: true, encoding: "utf-8" })
|
|
431
|
+
.filter((f): f is string => typeof f === "string" && f.endsWith(".md"))
|
|
432
|
+
.map(f => ({ file: f, content: readFileSync(join(specsDir, f), "utf-8") }));
|
|
433
|
+
const declarations = specs
|
|
434
|
+
.filter(s => !isTemplateSpec(s.file))
|
|
435
|
+
.flatMap(s => parseScDeclarations(s.content, s.file));
|
|
436
|
+
const credits = findTestFilesForSCs(declarations, buildScRegistry(specs), testDir);
|
|
437
|
+
return new Set([...credits.values()].flat().map(c => scopedKey(c.specFile, c.id)));
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Detect duplicate SC IDs across specs.
|
|
442
|
+
*/
|
|
443
|
+
export function findDuplicateSCIds(scs: UncheckedSC[]): Map<string, string[]> {
|
|
444
|
+
const idToSpecs = new Map<string, string[]>();
|
|
445
|
+
for (const sc of scs) {
|
|
446
|
+
if (isTemplateSpec(sc.specFile)) continue;
|
|
447
|
+
const specs = idToSpecs.get(sc.id) || [];
|
|
448
|
+
if (!specs.includes(sc.specFile)) specs.push(sc.specFile);
|
|
449
|
+
idToSpecs.set(sc.id, specs);
|
|
450
|
+
}
|
|
451
|
+
const duplicates = new Map<string, string[]>();
|
|
452
|
+
for (const [id, specs] of idToSpecs) {
|
|
453
|
+
if (specs.length > 1) duplicates.set(id, specs);
|
|
454
|
+
}
|
|
455
|
+
return duplicates;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* AC-2: For each unchecked SC without a hand-written test, runs matchPattern()
|
|
460
|
+
* and executes the returned assertion against the project root.
|
|
461
|
+
* Results are scoped by spec file to prevent cross-spec ID collisions.
|
|
462
|
+
*
|
|
463
|
+
* Behavioral SCs are checked against the behavioral-results cache first.
|
|
464
|
+
* If a fresh cached result exists, it's used instead of matchPattern.
|
|
465
|
+
*/
|
|
466
|
+
export function checkConformitySCs(uncheckedSCs: UncheckedSC[], root: string): ConformityResult {
|
|
467
|
+
const passing = new Set<string>();
|
|
468
|
+
const failing = new Set<string>();
|
|
469
|
+
const unmatchable = new Set<string>();
|
|
470
|
+
|
|
471
|
+
// Read behavioral cache for behavioral SCs
|
|
472
|
+
const cachePath = join(root, ".rungate", "behavioral-results.json");
|
|
473
|
+
const behavioralCache = readFreshCache(cachePath);
|
|
474
|
+
|
|
475
|
+
for (const sc of uncheckedSCs) {
|
|
476
|
+
const statement = sc.statement || "";
|
|
477
|
+
const key = scopedKey(sc.specFile, sc.id);
|
|
478
|
+
const parsed: ParsedSC = { id: sc.id, statement, specFile: sc.specFile };
|
|
479
|
+
|
|
480
|
+
// Check behavioral cache first for behavioral SCs
|
|
481
|
+
if (isBehavioralSC(parsed) && behavioralCache[sc.id]) {
|
|
482
|
+
if (behavioralCache[sc.id].passed) {
|
|
483
|
+
passing.add(key);
|
|
484
|
+
} else {
|
|
485
|
+
failing.add(key);
|
|
486
|
+
}
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
const assertion = matchPattern(parsed);
|
|
491
|
+
|
|
492
|
+
if (!assertion) {
|
|
493
|
+
unmatchable.add(key);
|
|
494
|
+
continue;
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
try {
|
|
498
|
+
assertion(root);
|
|
499
|
+
passing.add(key);
|
|
500
|
+
} catch {
|
|
501
|
+
failing.add(key);
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
return { passing, failing, unmatchable };
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
export function flipCheckboxes(specFile: string, scIds: string[], specsDir?: string, isDryRun?: boolean): number {
|
|
509
|
+
const dir = specsDir || SPECS_DIR;
|
|
510
|
+
const filePath = join(dir, specFile);
|
|
511
|
+
let content = readFileSync(filePath, "utf-8");
|
|
512
|
+
let flipped = 0;
|
|
513
|
+
|
|
514
|
+
for (const id of scIds) {
|
|
515
|
+
const pattern = `- [ ] ${id}:`;
|
|
516
|
+
if (content.includes(pattern)) {
|
|
517
|
+
content = content.replace(pattern, `- [x] ${id}:`);
|
|
518
|
+
flipped++;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// Write unless dry-run: use explicit isDryRun param if provided, else module-level flag
|
|
523
|
+
const shouldSkipWrite = isDryRun !== undefined ? isDryRun : dryRun;
|
|
524
|
+
if (flipped > 0 && !shouldSkipWrite) {
|
|
525
|
+
writeFileSync(filePath, content);
|
|
526
|
+
}
|
|
527
|
+
return flipped;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
export interface SpecCoverage {
|
|
531
|
+
testedPassing: string[];
|
|
532
|
+
testedFailing: string[];
|
|
533
|
+
conformityPassing: string[];
|
|
534
|
+
conformityFailing: string[];
|
|
535
|
+
unmatchable: string[];
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
/**
|
|
539
|
+
* AC-4: --report flag outputs per-spec coverage showing
|
|
540
|
+
* tested-passing, tested-failing, and unmatchable SCs.
|
|
541
|
+
*/
|
|
542
|
+
export function generateReport(specCoverage: Map<string, SpecCoverage>): string {
|
|
543
|
+
const lines: string[] = [];
|
|
544
|
+
lines.push("=== SC Coverage Report ===\n");
|
|
545
|
+
|
|
546
|
+
for (const [specFile, coverage] of specCoverage) {
|
|
547
|
+
lines.push(`${specFile}:`);
|
|
548
|
+
if (coverage.testedPassing.length > 0)
|
|
549
|
+
lines.push(` tested-passing (${coverage.testedPassing.length}): ${coverage.testedPassing.join(", ")}`);
|
|
550
|
+
if (coverage.testedFailing.length > 0)
|
|
551
|
+
lines.push(` tested-failing (${coverage.testedFailing.length}): ${coverage.testedFailing.join(", ")}`);
|
|
552
|
+
if (coverage.conformityPassing.length > 0)
|
|
553
|
+
lines.push(` conformity-passing (${coverage.conformityPassing.length}): ${coverage.conformityPassing.join(", ")}`);
|
|
554
|
+
if (coverage.conformityFailing.length > 0)
|
|
555
|
+
lines.push(` conformity-failing (${coverage.conformityFailing.length}): ${coverage.conformityFailing.join(", ")}`);
|
|
556
|
+
if (coverage.unmatchable.length > 0)
|
|
557
|
+
lines.push(` unmatchable (${coverage.unmatchable.length}): ${coverage.unmatchable.join(", ")}`);
|
|
558
|
+
lines.push("");
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
return lines.join("\n");
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// --- Main (only runs when executed directly) ---
|
|
565
|
+
const isDirectExecution = import.meta.main;
|
|
566
|
+
|
|
567
|
+
if (isDirectExecution) {
|
|
568
|
+
const unchecked = findUncheckedSCs();
|
|
569
|
+
if (unchecked.length === 0) {
|
|
570
|
+
console.log("All SCs are already checked off");
|
|
571
|
+
process.exit(0);
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
console.log(`Found ${unchecked.length} unchecked SCs across specs`);
|
|
575
|
+
|
|
576
|
+
// Step 0: Check for drift and staleness
|
|
577
|
+
const driftResult = detectDrift(SPECS_DIR, ROOT);
|
|
578
|
+
const driftedSCs = new Set<string>();
|
|
579
|
+
const staleSCs = new Set<string>();
|
|
580
|
+
|
|
581
|
+
if (driftResult.drifted.size > 0 || driftResult.stale.size > 0) {
|
|
582
|
+
console.log(`\n⚠️ SC drift/staleness detected:`);
|
|
583
|
+
|
|
584
|
+
for (const [specFile, details] of driftResult.drifted) {
|
|
585
|
+
for (const detail of details) {
|
|
586
|
+
console.log(` DRIFT: ${detail.scId} in ${specFile} references ${detail.path} but file does not exist`);
|
|
587
|
+
driftedSCs.add(scopedKey(specFile, detail.scId));
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
for (const [specFile, details] of driftResult.stale) {
|
|
592
|
+
for (const detail of details) {
|
|
593
|
+
console.log(` STALE: ${detail.scId} in ${specFile} expects [${detail.keyword}] in ${detail.path} but it's missing`);
|
|
594
|
+
staleSCs.add(scopedKey(specFile, detail.scId));
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
console.log(` Fix: update SC descriptions to match current file structure.\n`);
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
// Step 1: Detect duplicate SC IDs across specs
|
|
602
|
+
const duplicates = findDuplicateSCIds(unchecked);
|
|
603
|
+
if (duplicates.size > 0) {
|
|
604
|
+
console.log(`\n⚠️ Duplicate SC IDs found across specs:`);
|
|
605
|
+
for (const [id, specs] of duplicates) {
|
|
606
|
+
console.log(` ${id}: ${specs.join(", ")}`);
|
|
607
|
+
}
|
|
608
|
+
console.log(` Fix: renumber duplicates so each SC ID is unique across all specs.\n`);
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
// Step 2: Find SCs covered by hand-written test files.
|
|
612
|
+
// Scoped to (spec, id) since #330 — see findTestFilesForSCs. The old bare-id
|
|
613
|
+
// credit ticked the id in every spec declaring it.
|
|
614
|
+
const registry = readScRegistry();
|
|
615
|
+
const testFileMap = findTestFilesForSCs(unchecked, registry);
|
|
616
|
+
const testedKeys = new Set(
|
|
617
|
+
[...testFileMap.values()].flat().map(c => scopedKey(c.specFile, c.id)),
|
|
618
|
+
);
|
|
619
|
+
const untestedSCs = unchecked.filter(s => !testedKeys.has(scopedKey(s.specFile, s.id)));
|
|
620
|
+
|
|
621
|
+
// Mentions that look like claims and are not credited. Visible on purpose:
|
|
622
|
+
// under-crediting is the safe direction and still a measurement error.
|
|
623
|
+
const unqualified = new Map<string, string>();
|
|
624
|
+
for (const file of readdirSync(TEST_DIR).filter(f => f.endsWith(".test.ts"))) {
|
|
625
|
+
for (const u of unqualifiedClaims(readFileSync(join(TEST_DIR, file), "utf-8"), registry)) {
|
|
626
|
+
unqualified.set(`${file} ${u.id}`, ` ${file}: ${u.id} — ${u.reason}`);
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
if (unqualified.size > 0) {
|
|
630
|
+
console.log(`\n⚠️ ${unqualified.size} ambiguous SC mention(s) credited to nothing:`);
|
|
631
|
+
for (const line of [...unqualified.values()].sort()) console.log(line);
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
// Step 3: Run hand-written tests — scope by spec file
|
|
635
|
+
const passingSCs = new Set<string>();
|
|
636
|
+
const failingSCs = new Set<string>();
|
|
637
|
+
|
|
638
|
+
for (const [testFile, credits] of testFileMap) {
|
|
639
|
+
process.stdout.write(` Running ${testFile}...`);
|
|
640
|
+
const passed = runTestFile(testFile);
|
|
641
|
+
const target = passed ? passingSCs : failingSCs;
|
|
642
|
+
console.log(
|
|
643
|
+
passed
|
|
644
|
+
? ` (${credits.length} SCs)`
|
|
645
|
+
: ` (${credits.map(c => `${c.id} in ${c.specFile}`).join(", ")})`,
|
|
646
|
+
);
|
|
647
|
+
for (const c of credits) target.add(scopedKey(c.specFile, c.id));
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
// Remove any SCs that failed from passing set
|
|
651
|
+
for (const key of failingSCs) passingSCs.delete(key);
|
|
652
|
+
|
|
653
|
+
// Step 4: AC-2 — Run conformity assertions on untested SCs
|
|
654
|
+
if (untestedSCs.length > 0) {
|
|
655
|
+
console.log(`\nChecking ${untestedSCs.length} untested SCs via conformity engine...`);
|
|
656
|
+
const conformityResult = checkConformitySCs(untestedSCs, ROOT);
|
|
657
|
+
|
|
658
|
+
for (const key of conformityResult.passing) {
|
|
659
|
+
passingSCs.add(key);
|
|
660
|
+
const scId = key.split("::")[1];
|
|
661
|
+
console.log(` ${scId}: conformity PASS`);
|
|
662
|
+
}
|
|
663
|
+
for (const key of conformityResult.failing) {
|
|
664
|
+
failingSCs.add(key);
|
|
665
|
+
const scId = key.split("::")[1];
|
|
666
|
+
console.log(` ${scId}: conformity FAIL`);
|
|
667
|
+
}
|
|
668
|
+
if (conformityResult.unmatchable.size > 0) {
|
|
669
|
+
const ids = [...conformityResult.unmatchable].map(k => k.split("::")[1]);
|
|
670
|
+
console.log(` ${conformityResult.unmatchable.size} SCs unmatchable: ${ids.join(", ")}`);
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// AC-4: --report flag
|
|
674
|
+
if (reportMode) {
|
|
675
|
+
const specCoverage = new Map<string, SpecCoverage>();
|
|
676
|
+
|
|
677
|
+
for (const sc of unchecked) {
|
|
678
|
+
if (!specCoverage.has(sc.specFile)) {
|
|
679
|
+
specCoverage.set(sc.specFile, {
|
|
680
|
+
testedPassing: [],
|
|
681
|
+
testedFailing: [],
|
|
682
|
+
conformityPassing: [],
|
|
683
|
+
conformityFailing: [],
|
|
684
|
+
unmatchable: [],
|
|
685
|
+
});
|
|
686
|
+
}
|
|
687
|
+
const coverage = specCoverage.get(sc.specFile)!;
|
|
688
|
+
const key = scopedKey(sc.specFile, sc.id);
|
|
689
|
+
|
|
690
|
+
// Scoped since #330: a bare id matched a criterion in every spec
|
|
691
|
+
// declaring it, so one spec's report borrowed another's test result.
|
|
692
|
+
if (testedKeys.has(key)) {
|
|
693
|
+
if (passingSCs.has(key)) coverage.testedPassing.push(sc.id);
|
|
694
|
+
else if (failingSCs.has(key)) coverage.testedFailing.push(sc.id);
|
|
695
|
+
} else if (conformityResult.passing.has(key)) {
|
|
696
|
+
coverage.conformityPassing.push(sc.id);
|
|
697
|
+
} else if (conformityResult.failing.has(key)) {
|
|
698
|
+
coverage.conformityFailing.push(sc.id);
|
|
699
|
+
} else if (conformityResult.unmatchable.has(key)) {
|
|
700
|
+
coverage.unmatchable.push(sc.id);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
console.log("\n" + generateReport(specCoverage));
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
// Step 4: Filter out drifted/stale SCs before flipping
|
|
709
|
+
for (const key of driftedSCs) {
|
|
710
|
+
if (passingSCs.has(key)) {
|
|
711
|
+
passingSCs.delete(key);
|
|
712
|
+
const scId = key.split("::")[1];
|
|
713
|
+
console.log(`\n⚠️ Not flipping ${scId}: drifted (references nonexistent file)`);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
for (const key of staleSCs) {
|
|
717
|
+
if (passingSCs.has(key)) {
|
|
718
|
+
passingSCs.delete(key);
|
|
719
|
+
const scId = key.split("::")[1];
|
|
720
|
+
console.log(`\n⚠️ Not flipping ${scId}: stale (keyword missing from file)`);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
// Step 5: Flip passing SCs
|
|
725
|
+
if (passingSCs.size === 0) {
|
|
726
|
+
console.log("No SCs to flip");
|
|
727
|
+
process.exit(0);
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
// Group passing SCs by spec file and flip — scoped keys prevent cross-spec bleeding
|
|
731
|
+
const bySpec = new Map<string, string[]>();
|
|
732
|
+
for (const sc of unchecked) {
|
|
733
|
+
const key = scopedKey(sc.specFile, sc.id);
|
|
734
|
+
if (passingSCs.has(key)) {
|
|
735
|
+
const list = bySpec.get(sc.specFile) || [];
|
|
736
|
+
list.push(sc.id);
|
|
737
|
+
bySpec.set(sc.specFile, list);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
let totalFlipped = 0;
|
|
742
|
+
for (const [specFile, scs] of bySpec) {
|
|
743
|
+
const flipped = flipCheckboxes(specFile, scs);
|
|
744
|
+
if (flipped > 0) {
|
|
745
|
+
console.log(`${dryRun ? "Would flip" : "Flipped"} ${flipped} SCs in ${specFile}: ${scs.join(", ")}`);
|
|
746
|
+
totalFlipped += flipped;
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
console.log(`\n${dryRun ? "Would flip" : "Flipped"} ${totalFlipped} SC checkboxes`);
|
|
751
|
+
}
|