rungate 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/rungate.ts +70 -0
- package/lib/aes-calculator.ts +66 -0
- package/lib/agent-audit.ts +139 -0
- package/lib/agent-brief-validation.ts +359 -0
- package/lib/agents-md-refs.ts +90 -0
- package/lib/bash-file-read.ts +197 -0
- package/lib/behavioral-cache.ts +78 -0
- package/lib/branch-cleanup.ts +234 -0
- package/lib/brief-context-parser.ts +35 -0
- package/lib/canary.ts +117 -0
- package/lib/check-registry.ts +226 -0
- package/lib/ci-checkout.ts +400 -0
- package/lib/cli-commands.ts +137 -0
- package/lib/compliance-backfill.ts +220 -0
- package/lib/compliance-report.ts +504 -0
- package/lib/compliance.ts +361 -0
- package/lib/config-loader.ts +255 -0
- package/lib/conformity.ts +1818 -131
- package/lib/container-provenance.ts +230 -0
- package/lib/create-brief.ts +150 -0
- package/lib/create-sc.ts +272 -0
- package/lib/deep-merge.ts +69 -0
- package/lib/dev-liveness.ts +214 -0
- package/lib/directive-extractor.ts +210 -0
- package/lib/distribution-standing.ts +113 -0
- package/lib/doc-hygiene.ts +238 -0
- package/lib/eval-criteria.ts +6 -0
- package/lib/evidence-portability.ts +123 -0
- package/lib/evidence-prevalidator.ts +390 -0
- package/lib/evidence-required.ts +150 -0
- package/lib/evidence-satisfiability.ts +288 -0
- package/lib/frontmatter.ts +163 -0
- package/lib/gap-scanner.ts +348 -0
- package/lib/gate-enforcement.ts +366 -0
- package/lib/generators/agent-briefs.ts +136 -0
- package/lib/generators/agents-md.ts +506 -0
- package/lib/generators/code-map.ts +60 -0
- package/lib/generators/types.ts +184 -0
- package/lib/git-ref-normalizer.ts +107 -0
- package/lib/github.ts +539 -0
- package/lib/harness-identity.ts +63 -0
- package/lib/hill-climb.ts +164 -0
- package/lib/instruction-effectiveness.ts +131 -0
- package/lib/issue-closure-audit.ts +225 -0
- package/lib/issue-create-override.ts +343 -0
- package/lib/js-top-level-code.ts +250 -0
- package/lib/liveness-contradiction.ts +149 -0
- package/lib/organize.ts +505 -0
- package/lib/paths.ts +197 -6
- package/lib/post-fix-verify.ts +157 -0
- package/lib/prior-branch.ts +489 -0
- package/lib/project-state-shape.ts +166 -0
- package/lib/promote-outputs.ts +156 -0
- package/lib/prove-label-gate.ts +35 -0
- package/lib/reachability.ts +302 -0
- package/lib/release-preflight.ts +186 -0
- package/lib/rule-health-pipeline.ts +264 -0
- package/lib/rule-health-trend.ts +161 -0
- package/lib/rule-provenance.ts +149 -0
- package/lib/rule-registry.ts +128 -0
- package/lib/rungate-schema.ts +114 -4
- package/lib/sc-claim-forms.ts +179 -0
- package/lib/sc-cover.ts +150 -0
- package/lib/sc-guard.ts +157 -0
- package/lib/sc-ids.ts +344 -0
- package/lib/scaffold/defaults.ts +165 -0
- package/lib/scaffold/managed-workflow.ts +370 -0
- package/lib/scaffold/owned-files.ts +265 -0
- package/lib/scaffold/steps.ts +2680 -0
- package/lib/scanner.ts +701 -0
- package/lib/screenshot-upload.ts +73 -0
- package/lib/secret-patterns.ts +112 -0
- package/lib/security-verdict.ts +616 -0
- package/lib/signal-phrases.ts +1 -0
- package/lib/spec-change-conformity.ts +80 -0
- package/lib/spec-files.ts +62 -0
- package/lib/spec-registry.ts +133 -0
- package/lib/spec-updater.ts +96 -0
- package/lib/stale-cleanup.ts +224 -0
- package/lib/stale-issue-scanner.ts +36 -0
- package/lib/strip-js-comments.ts +269 -0
- package/lib/suite-measurement.ts +228 -0
- package/lib/task-completion-checks.ts +177 -0
- package/lib/test-suite-lock.ts +1136 -0
- package/lib/transcript-checker.ts +1140 -0
- package/lib/validators/spec-validators.ts +171 -0
- package/lib/verdict-capture.ts +73 -0
- package/lib/verified-push.ts +358 -0
- package/lib/workflow-security.ts +174 -0
- package/lib/worktree-cleanup.ts +250 -0
- package/lib/worktree-collect.ts +283 -0
- package/lib/worktree-isolation.ts +82 -0
- package/package.json +24 -2
- package/prompts/ac-adversary.md +14 -2
- package/prompts/aditi.md +5 -0
- package/prompts/blast-radius.md +5 -0
- package/prompts/container-verify.md +5 -0
- package/prompts/discovery.md +4 -0
- package/prompts/environment.md +5 -0
- package/prompts/evidence-hierarchy.md +5 -0
- package/prompts/evidence-validator.md +11 -0
- package/prompts/marcus.md +5 -0
- package/prompts/prevention.md +5 -0
- package/prompts/prove-reproducer.md +12 -0
- package/prompts/quinn-decision-tree.md +5 -0
- package/prompts/quinn-ui-brief.md +5 -0
- package/prompts/quinn.md +5 -0
- package/prompts/rca.md +5 -0
- package/prompts/read-before-write.md +4 -0
- package/prompts/regression.md +5 -0
- package/prompts/serena.md +5 -0
- package/scripts/audit-specs.ts +464 -0
- package/scripts/convert-spec.ts +293 -0
- package/scripts/create-sc.ts +92 -0
- package/scripts/create-spec.ts +166 -0
- package/scripts/extract-constraints.ts +4 -3
- package/scripts/generate-code-map.ts +1 -1
- package/scripts/scaffold-project.ts +138 -1426
- package/scripts/sync-spec-tests.ts +211 -126
package/lib/rungate-schema.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
|
-
const DevSchema = z.object({
|
|
3
|
+
export const DevSchema = z.object({
|
|
4
4
|
start: z.string(),
|
|
5
5
|
apiBase: z.string().url(),
|
|
6
6
|
uiBase: z.string().url(),
|
|
@@ -8,19 +8,91 @@ const DevSchema = z.object({
|
|
|
8
8
|
testCmd: z.string().optional(),
|
|
9
9
|
typeCheck: z.string().optional(),
|
|
10
10
|
note: z.string().optional(),
|
|
11
|
+
/**
|
|
12
|
+
* The path the dev-server liveness probe requests, relative to `apiBase`.
|
|
13
|
+
*
|
|
14
|
+
* #304. `checkDevServerLiveness` used to curl a literal
|
|
15
|
+
* `${apiBase}/api/admin/health` — one consumer's route written into
|
|
16
|
+
* the harness. It worked for exactly one consumer by coincidence, and the
|
|
17
|
+
* first consumer with a different health route would be told its running
|
|
18
|
+
* server was DOWN and to start it. `container.healthPath` was already a
|
|
19
|
+
* declared concept; `dev` had no equivalent, so this is it.
|
|
20
|
+
*
|
|
21
|
+
* No default. A probe that guesses a path reports DOWN for a server that is
|
|
22
|
+
* up, and "could not check" must never become "it is down".
|
|
23
|
+
*
|
|
24
|
+
* **Nullable as well as optional (#346.)** The scaffold detects a health
|
|
25
|
+
* route rather than defaulting one, and writes `null` when it looked and
|
|
26
|
+
* found nothing. Absent and null are different facts — "nobody configured
|
|
27
|
+
* this" versus "we scanned the conventional entry points and there is no
|
|
28
|
+
* health endpoint there" — and collapsing them loses exactly the
|
|
29
|
+
* distinction the liveness record exists to carry. `probeTarget` treats
|
|
30
|
+
* both as UNMEASURED, which is correct for both.
|
|
31
|
+
*/
|
|
32
|
+
healthPath: z.string().nullable().optional(),
|
|
11
33
|
});
|
|
12
34
|
|
|
13
|
-
|
|
35
|
+
/**
|
|
36
|
+
* `command`/`timeout` were read by gate-executor.ts, ship.js and prove.js and
|
|
37
|
+
* declared by none of them in zod, so validation deleted the two keys the
|
|
38
|
+
* readers depend on. ship.js's own inline JSON schema (`:1651`) has always
|
|
39
|
+
* declared both — this brings zod up to what the harness already assumed.
|
|
40
|
+
*/
|
|
41
|
+
export const TestSchema = z.object({
|
|
14
42
|
start: z.string().optional(),
|
|
15
43
|
apiBase: z.string().url().optional(),
|
|
44
|
+
command: z.string().optional(),
|
|
45
|
+
timeout: z.number().optional(),
|
|
16
46
|
note: z.string().optional(),
|
|
17
47
|
});
|
|
18
48
|
|
|
19
|
-
|
|
49
|
+
/**
|
|
50
|
+
* The test container. Shape taken verbatim from ship.js's setup-agent schema
|
|
51
|
+
* (`:1651`) and read by its container-verify block (`:3754`). Undeclared here
|
|
52
|
+
* until now, so a validated config lost it and the container path rested on a
|
|
53
|
+
* key the validator removed.
|
|
54
|
+
*
|
|
55
|
+
* NOTE: `ProdSchema.rebuild` and `container.rebuildCommand` now coexist. They
|
|
56
|
+
* are different things — `prod.rebuild` is the production rebuild a human
|
|
57
|
+
* runs, `container.rebuildCommand` is interpolated into an AGENT PROMPT at
|
|
58
|
+
* ship.js:3766. Declaring the key does not constrain its value: a consumer
|
|
59
|
+
* must never put a command its own CLAUDE.md forbids (DDB: `make rebuild`)
|
|
60
|
+
* behind this key.
|
|
61
|
+
*/
|
|
62
|
+
export const ContainerSchema = z.object({
|
|
63
|
+
port: z.number().optional(),
|
|
64
|
+
rebuildCommand: z.string().optional(),
|
|
65
|
+
healthPath: z.string().optional(),
|
|
66
|
+
hosts: z.array(z.string()).optional(),
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
export type ContainerConfig = z.infer<typeof ContainerSchema>;
|
|
70
|
+
|
|
71
|
+
export const ProdSchema = z.object({
|
|
20
72
|
rebuild: z.string().optional(),
|
|
21
73
|
apiBase: z.string().url().optional(),
|
|
22
74
|
uiBase: z.string().url().optional(),
|
|
23
75
|
smokeTest: z.string().optional(),
|
|
76
|
+
/** See `DevSchema.healthPath` — the prove gate probes this tier too (#304). */
|
|
77
|
+
healthPath: z.string().nullable().optional(),
|
|
78
|
+
/**
|
|
79
|
+
* The promoted container `container-has-fix` inspects, and which runtime to
|
|
80
|
+
* ask (#341).
|
|
81
|
+
*
|
|
82
|
+
* All three were read by `gates/workflow.test.ts` and declared by none of
|
|
83
|
+
* them. That check loads `rungate.json` with a bare `JSON.parse`, so they
|
|
84
|
+
* reached it — but a key no schema mentions is a key nothing validates, and
|
|
85
|
+
* `ProdSchema` is not `passthrough()`, so any caller that *does* go through
|
|
86
|
+
* zod silently drops them.
|
|
87
|
+
*
|
|
88
|
+
* `containerRuntime` is new. The old code chose by probe —
|
|
89
|
+
* `podman` if present, otherwise `docker` whether docker existed or not —
|
|
90
|
+
* and with neither installed the inspect failed and the check passed. Config
|
|
91
|
+
* first, probe as a fallback, and "neither" is now its own outcome.
|
|
92
|
+
*/
|
|
93
|
+
containerName: z.string().optional(),
|
|
94
|
+
imageName: z.string().optional(),
|
|
95
|
+
containerRuntime: z.string().optional(),
|
|
24
96
|
});
|
|
25
97
|
|
|
26
98
|
const McpServerSchema = z.object({
|
|
@@ -33,19 +105,57 @@ const ResearchToolSchema = z.object({
|
|
|
33
105
|
type: z.enum(["mcp", "cli", "api"]).optional(),
|
|
34
106
|
});
|
|
35
107
|
|
|
36
|
-
const
|
|
108
|
+
const RoleSchema = z.object({
|
|
109
|
+
brief: z.string(),
|
|
110
|
+
isolation: z.string().optional(),
|
|
111
|
+
standardTask: z.string(),
|
|
112
|
+
description: z.string().optional(),
|
|
113
|
+
tools: z.string().optional(),
|
|
114
|
+
model: z.string().optional(),
|
|
115
|
+
tiers: z.record(z.array(z.string())).optional(),
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
export type RoleConfig = z.infer<typeof RoleSchema>;
|
|
119
|
+
|
|
120
|
+
const CeremonyTierOverrideSchema = z.object({
|
|
121
|
+
maxIterations: z.number().optional(),
|
|
122
|
+
checks: z.record(z.array(z.string())).optional(),
|
|
123
|
+
skipScopeGate: z.boolean().optional(),
|
|
124
|
+
githubComments: z.number().optional(),
|
|
125
|
+
stopMarkers: z.number().optional(),
|
|
126
|
+
agents: z.record(z.union([z.boolean(), z.string()])).optional(),
|
|
127
|
+
}).passthrough();
|
|
128
|
+
|
|
129
|
+
export const ProjectHarnessSchema = z.object({
|
|
37
130
|
project: z.string(),
|
|
38
131
|
repo: z.string(),
|
|
39
132
|
issueRepo: z.string(),
|
|
40
133
|
dev: DevSchema.optional(),
|
|
41
134
|
test: TestSchema.optional(),
|
|
42
135
|
prod: ProdSchema.optional(),
|
|
136
|
+
container: ContainerSchema.optional(),
|
|
43
137
|
pages: z.record(z.string()).optional(),
|
|
44
138
|
codeCommittedPaths: z.array(z.string()).optional(),
|
|
139
|
+
/**
|
|
140
|
+
* Commands an agent must never run in this project (#343).
|
|
141
|
+
*
|
|
142
|
+
* `lib/transcript-checker.ts` used to hardcode `make rebuild` — one
|
|
143
|
+
* consumer's rule — so for every other consumer the forbidden-command
|
|
144
|
+
* directive fell through to N/A, which `computeScore` drops from its
|
|
145
|
+
* denominator. The rule went unenforced and the score did not move.
|
|
146
|
+
*
|
|
147
|
+
* No default. A guessed prohibition is worse than none: agent briefs quote
|
|
148
|
+
* it, agents obey it, and nobody finds out it was invented.
|
|
149
|
+
*/
|
|
150
|
+
forbiddenCommands: z.array(z.string()).optional(),
|
|
45
151
|
consumers: z.array(z.string()).optional(),
|
|
152
|
+
promptKeywords: z.record(z.array(z.string())).optional(),
|
|
46
153
|
contextDocs: z.record(z.string().nullable()).optional(),
|
|
47
154
|
mcp: z.array(McpServerSchema).optional(),
|
|
48
155
|
research: z.array(ResearchToolSchema).optional(),
|
|
156
|
+
roles: z.record(RoleSchema).optional(),
|
|
157
|
+
stalenessThreshold: z.number().optional(),
|
|
158
|
+
ceremonyOverrides: z.record(CeremonyTierOverrideSchema).optional(),
|
|
49
159
|
schemaVersion: z.number().default(1),
|
|
50
160
|
});
|
|
51
161
|
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Telling a coverage CLAIM from a CITATION inside a comment (#360).
|
|
3
|
+
*
|
|
4
|
+
* `coverageClaimSites` credits a test file with covering `SC-N` when the file
|
|
5
|
+
* mentions the id in a comment. #149 chose that position deliberately and is
|
|
6
|
+
* right about it — a comment is prose, and prose is where a project declares
|
|
7
|
+
* intent. What the position cannot do is tell these apart:
|
|
8
|
+
*
|
|
9
|
+
* // covers SC-640 ← a claim
|
|
10
|
+
* // SC-640 was the criterion the collision broke ← a citation
|
|
11
|
+
*
|
|
12
|
+
* Two of those were caught in the wild while shipping #330, both of them a
|
|
13
|
+
* file that discusses coverage attribution certifying the criterion it
|
|
14
|
+
* discusses. That is the self-reference this area keeps producing.
|
|
15
|
+
*
|
|
16
|
+
* ## Why these five forms and not a new marker
|
|
17
|
+
*
|
|
18
|
+
* Measured first, per the project's own rule, and the measurement chose the
|
|
19
|
+
* design. Of 220 comment-position credits on the tree at the time:
|
|
20
|
+
*
|
|
21
|
+
* 67 a coverage verb within 60 chars before the id
|
|
22
|
+
* 34 the id ends a leading id-list terminated by `:`
|
|
23
|
+
* 40 the id is the first token on its line
|
|
24
|
+
* 79 none of those
|
|
25
|
+
*
|
|
26
|
+
* A systematic sample of 16 of the 79, read in full, split 11 claims to 5
|
|
27
|
+
* citations — and the split was not subtle. Every one of the five citations
|
|
28
|
+
* sat MID-SENTENCE inside running prose; every one of the eleven claims sat
|
|
29
|
+
* in a structural position, in two forms the first three rules missed.
|
|
30
|
+
*
|
|
31
|
+
* So the rule is not "adopt a `covers:` marker". The population already
|
|
32
|
+
* separates itself by structural position, and a new marker would mean
|
|
33
|
+
* migrating ~70 files to re-state something they already say. The rule is:
|
|
34
|
+
* **a declaration occupies a structural position; a bare id inside a running
|
|
35
|
+
* sentence is a citation.**
|
|
36
|
+
*
|
|
37
|
+
* Projected loss when this landed: ~25 of 220, not 206 — which is what makes
|
|
38
|
+
* the staging requirement satisfiable by the design instead of by a
|
|
39
|
+
* migration.
|
|
40
|
+
*
|
|
41
|
+
* ## The failure direction this is allowed to have
|
|
42
|
+
*
|
|
43
|
+
* Toward NOT crediting. A form nobody thought of is a lost credit, which
|
|
44
|
+
* shows up as an unticked checkbox someone fixes by adding one line. The
|
|
45
|
+
* other direction certifies a criterion nothing tests, and
|
|
46
|
+
* `.claude/rules/checks-must-be-able-to-fail.md` says never to fail that way.
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
/** A structural position that could only have been written to declare coverage. */
|
|
50
|
+
export type ClaimForm =
|
|
51
|
+
/** `covers`/`covering`/`coverage` within 60 chars before the id. */
|
|
52
|
+
| "covers"
|
|
53
|
+
/** The id ends a leading id-list terminated by `:` — `AC-2 / SC-516 / SC-517:`. */
|
|
54
|
+
| "label"
|
|
55
|
+
/** The id is the first token on its line, after comment delimiters. */
|
|
56
|
+
| "line"
|
|
57
|
+
/** A `SPEC-REF:` line, whatever follows it on that line. */
|
|
58
|
+
| "spec-ref"
|
|
59
|
+
/** The id is inside a parenthetical holding only ids — `… turns it red (SC-4).` */
|
|
60
|
+
| "attribution";
|
|
61
|
+
|
|
62
|
+
const COVERS_NEAR = /\b(?:covers?|covered|covering|coverage)\b[^\n]{0,60}$/i;
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* A verb directly after the id, making the line a SENTENCE about the
|
|
66
|
+
* criterion rather than a heading naming it.
|
|
67
|
+
*
|
|
68
|
+
* Opening a line is not quite enough on its own:
|
|
69
|
+
* `// SC-621 was the criterion the collision broke` is a citation that
|
|
70
|
+
* happens to come first. But the first fix for that was worse than the bug.
|
|
71
|
+
* An allowlist of what may follow a heading — end of line, a parenthetical,
|
|
72
|
+
* a dash, a spec filename — rejected ` * SC-722 of GATE-CONTRACTS-SPEC.md.`
|
|
73
|
+
* and ` * #173 — SC-587, SC-588, SC-589, SC-590.`, and took 30 criteria to
|
|
74
|
+
* zero coverage in one step. Measured, not argued: the loose rule costs 13
|
|
75
|
+
* credits across the tree, the allowlist costs 44.
|
|
76
|
+
*
|
|
77
|
+
* So this is a small closed stop-list instead, which is the one place a
|
|
78
|
+
* stop-list beats an allowlist. The set of English words that turn a heading
|
|
79
|
+
* into a sentence is small and closed; the set of ways to write a heading is
|
|
80
|
+
* neither. A word missing from this list costs ONE false credit on a line
|
|
81
|
+
* that still had to begin with the id — bounded. A heading form missing from
|
|
82
|
+
* an allowlist costs every credit written that way.
|
|
83
|
+
*
|
|
84
|
+
* Every entry was taken from a real citation in this tree.
|
|
85
|
+
*/
|
|
86
|
+
const SENTENCE_TAIL =
|
|
87
|
+
/^(?:'s|’s)?\s+(?:was|were|is|are|says?|said|below|above|asked|exists?|existed|had|has|became|names?|named|declares?|declared|turns?|would|will|means?|refers?)\b/i;
|
|
88
|
+
const SPEC_REF_LINE = /^[\s*\/|─—-]*SPEC-REF\b/i;
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* An id-list opening a line: `SC-561..SC-564 (SPEC.md)`, `SC-699, SC-700, …`.
|
|
92
|
+
*
|
|
93
|
+
* Ranges are why `line` alone is not enough — on `SC-561..SC-564 (SPEC.md)`
|
|
94
|
+
* only `SC-561` is the line's first token, and the credit being asked about
|
|
95
|
+
* is usually one of the others.
|
|
96
|
+
*
|
|
97
|
+
* The optional `#\d+` prefix is this repo's most common header form and was
|
|
98
|
+
* the single largest group in the unmarked population: `#178 / SC-413:`,
|
|
99
|
+
* `#173 — SC-587, SC-588, SC-589, SC-590.`, `#323 SC-6 —`, `#190 / SC-624:`.
|
|
100
|
+
* An issue number before the ids does not make the line less of a heading.
|
|
101
|
+
*/
|
|
102
|
+
const LEADING_ID_RUN = /^[\s*\/|─—-]*(?:#\d+\s*[\/|—–:-]*\s*)?(?:(?:SC|AC)-[A-Z]?\d+(?:\s*(?:\.\.+|,|\/|\||and|&|–|—|-)\s*)?)+/;
|
|
103
|
+
|
|
104
|
+
/** The same run, but required to terminate in a colon — a label, not a list. */
|
|
105
|
+
const LEADING_ID_LABEL = /^[\s*\/|─—-]*(?:#\d+\s*[\/|—–-]*\s*)?(?:(?:SC|AC)-[A-Z]?\d+[\s,\/|]*)+:/;
|
|
106
|
+
|
|
107
|
+
const LINE_START = /(?:^|\n)[\s*\/|─—-]*$/;
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Which structural form the mention at `at` occupies, or `null` for prose.
|
|
111
|
+
*
|
|
112
|
+
* `comment` is the full enclosing comment and `at` is the id's offset WITHIN
|
|
113
|
+
* it. The caller owns recovering both; see `enclosingComment`.
|
|
114
|
+
*
|
|
115
|
+
* THE single exported predicate for this question, so there is one binding to
|
|
116
|
+
* mutate. A second place to decide what counts as a claim would survive any
|
|
117
|
+
* mutation of this one and the guard would go red for a reason that has
|
|
118
|
+
* nothing to do with the mutation working.
|
|
119
|
+
*/
|
|
120
|
+
export function commentClaimForm(comment: string, at: number, id: string): ClaimForm | null {
|
|
121
|
+
if (at < 0 || at >= comment.length) return null;
|
|
122
|
+
const before = comment.slice(0, at);
|
|
123
|
+
const lineStart = before.lastIndexOf("\n") + 1;
|
|
124
|
+
const line = comment.slice(lineStart);
|
|
125
|
+
const inLine = at - lineStart;
|
|
126
|
+
|
|
127
|
+
if (COVERS_NEAR.test(before)) return "covers";
|
|
128
|
+
if (SPEC_REF_LINE.test(line)) return "spec-ref";
|
|
129
|
+
|
|
130
|
+
const label = LEADING_ID_LABEL.exec(line);
|
|
131
|
+
if (label && label[0].length > inLine) return "label";
|
|
132
|
+
|
|
133
|
+
const run = LEADING_ID_RUN.exec(line);
|
|
134
|
+
const leading = (run && run[0].length > inLine) || LINE_START.test(before);
|
|
135
|
+
if (leading && !SENTENCE_TAIL.test(line.slice(inLine + id.length))) return "line";
|
|
136
|
+
|
|
137
|
+
if (inIdOnlyParenthetical(line, inLine)) return "attribution";
|
|
138
|
+
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* A parenthetical holding NOTHING BUT ids — `(SC-4)`, `(SC-358, SC-359)`.
|
|
144
|
+
*
|
|
145
|
+
* Line-scoped on purpose. A multi-line scan would let an unclosed paren three
|
|
146
|
+
* paragraphs up swallow a whole docblock, and every prose mention in it would
|
|
147
|
+
* read as an attribution — the fail-open direction this module is not allowed
|
|
148
|
+
* to have.
|
|
149
|
+
*
|
|
150
|
+
* Ids-only is the second half, and it was not the first version. Allowing any
|
|
151
|
+
* parenthetical credited `("see SC-621")` in test/spec-sc-ids-unique.test.ts —
|
|
152
|
+
* inside a comment whose next clause is "Counting citations would make". A
|
|
153
|
+
* rule that credits a sentence explaining why citations must not be credited
|
|
154
|
+
* is the self-reference this whole area keeps producing, and it was caught by
|
|
155
|
+
* reading the diff of lost credits rather than by the rule itself.
|
|
156
|
+
*/
|
|
157
|
+
const IDS_ONLY =
|
|
158
|
+
/^[\s,\/|&+-]*(?:(?:#\d+|(?:SC|AC)-[A-Z]?\d+)(?:\s*(?:\.\.+|,|\/|\||&|and|–|—|-)\s*)?)+[\s.]*$/;
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* A parenthetical that LABELS: opens with an id and a colon, then prose.
|
|
162
|
+
*
|
|
163
|
+
* `── Extraction floor (SC-343: this file is the engine's unit-test home) ──`
|
|
164
|
+
* is a heading, and the colon is the same declaring signal as `SC-640:` at
|
|
165
|
+
* the start of a line. Restricted to parentheses on purpose: the bare
|
|
166
|
+
* "an id followed by a colon anywhere" rule would credit
|
|
167
|
+
* `// The SC statement in a spec file: "SC-318: Something (behavioral)"`,
|
|
168
|
+
* which is a QUOTED EXAMPLE of a spec line, not a claim about one.
|
|
169
|
+
*/
|
|
170
|
+
const PAREN_LABEL = /^\s*(?:SC|AC)-[A-Z]?\d+\s*:/;
|
|
171
|
+
|
|
172
|
+
function inIdOnlyParenthetical(line: string, at: number): boolean {
|
|
173
|
+
const open = line.lastIndexOf("(", at);
|
|
174
|
+
if (open < 0) return false;
|
|
175
|
+
const close = line.indexOf(")", open);
|
|
176
|
+
if (close < 0 || close < at) return false;
|
|
177
|
+
const inner = line.slice(open + 1, close);
|
|
178
|
+
return IDS_ONLY.test(inner) || PAREN_LABEL.test(inner);
|
|
179
|
+
}
|
package/lib/sc-cover.ts
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What counts as COVER for a ticked success criterion (#327).
|
|
3
|
+
*
|
|
4
|
+
* A tick is a claim that something was proven. #149 made it visible that the
|
|
5
|
+
* claim and the proof had come apart: it replaced "a test file mentions the
|
|
6
|
+
* id" with "a test file DECLARES the id", and eight criteria lost their only
|
|
7
|
+
* coverage claim without losing their tick. Three spot-checked then were
|
|
8
|
+
* certified by, in order, a line of unrelated fixture text, a key-formatting
|
|
9
|
+
* test, and a duplicate-detection record. None asserted anything about the
|
|
10
|
+
* criterion it certified.
|
|
11
|
+
*
|
|
12
|
+
* The eight were the visible edge. Measured over the whole tree at the time
|
|
13
|
+
* of this issue: **748 ticked criteria, 161 of them with no cover of any
|
|
14
|
+
* kind** — no declaring test claim, no passing conformity assertion, no
|
|
15
|
+
* behavioural exemption, no supersession. That is the number this module
|
|
16
|
+
* exists to stop growing.
|
|
17
|
+
*
|
|
18
|
+
* ## Four ways a tick can be honest, and they are not interchangeable
|
|
19
|
+
*
|
|
20
|
+
* claim a test file declares the id in a comment or a title
|
|
21
|
+
* (`.claude/rules/claiming-sc-coverage.md` owns the forms)
|
|
22
|
+
* conformity the statement matches a registry pattern AND the assertion
|
|
23
|
+
* passes right now — not merely that it matched
|
|
24
|
+
* behavioural marked `(behavioral)`, bounded elsewhere by #328's ratchet
|
|
25
|
+
* superseded the statement opens `SUPERSEDED`, so the tick is a pointer
|
|
26
|
+
* to the criteria that replaced it rather than a proof
|
|
27
|
+
*
|
|
28
|
+
* Supersession is in the list because GITHUB-API-MIGRATION-SPEC.md uses it
|
|
29
|
+
* three times in a row (SC-499, SC-500, SC-501) and the convention is
|
|
30
|
+
* deliberate: the line records what the criterion USED to say and why it was
|
|
31
|
+
* wrong. Demanding a test for a criterion that explicitly no longer applies
|
|
32
|
+
* would push the spec to delete its own history.
|
|
33
|
+
*
|
|
34
|
+
* ## One binding
|
|
35
|
+
*
|
|
36
|
+
* `uncoveredTickedScs` is the only place that decides, so
|
|
37
|
+
* `test/ticked-sc-cover.test.ts` can short-circuit it and watch the ratchet
|
|
38
|
+
* stop refusing. Two functions answering this would leave a mutation of
|
|
39
|
+
* either one looking like the guard working — which is the defect in
|
|
40
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`, and the defect this whole
|
|
41
|
+
* issue is an instance of.
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import { readdirSync, readFileSync } from "fs";
|
|
45
|
+
import { join } from "path";
|
|
46
|
+
|
|
47
|
+
import { isBehavioralSC, matchPattern } from "./conformity";
|
|
48
|
+
import { buildScRegistry, isTemplateSpec, parseScDeclarations, type ScDeclaration } from "./sc-ids";
|
|
49
|
+
|
|
50
|
+
export interface UncoveredSc {
|
|
51
|
+
id: string;
|
|
52
|
+
specFile: string;
|
|
53
|
+
statement: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** A tick that is a pointer rather than a proof. */
|
|
57
|
+
export function isSuperseded(statement: string): boolean {
|
|
58
|
+
return /^SUPERSEDED\b/.test(statement.trim());
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function readSpecs(specsDir: string): Array<{ file: string; content: string }> {
|
|
62
|
+
return readdirSync(specsDir, { recursive: true, encoding: "utf-8" })
|
|
63
|
+
.filter(f => typeof f === "string" && f.endsWith(".md"))
|
|
64
|
+
.map(f => ({ file: f, content: readFileSync(join(specsDir, f), "utf-8") }));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Every ticked criterion with no cover of any kind.
|
|
69
|
+
*
|
|
70
|
+
* `claimed` is passed in rather than computed here, because the credit rule
|
|
71
|
+
* lives in `scripts/sync-sc-status.ts` and having two implementations of it
|
|
72
|
+
* is how this class of bug arrives. Keys are `scopedKey(specFile, id)`.
|
|
73
|
+
*/
|
|
74
|
+
export function uncoveredTickedScs(opts: {
|
|
75
|
+
specsDir: string;
|
|
76
|
+
projectRoot: string;
|
|
77
|
+
claimed: Set<string>;
|
|
78
|
+
scopedKey: (specFile: string, id: string) => string;
|
|
79
|
+
}): UncoveredSc[] {
|
|
80
|
+
const { specsDir, projectRoot, claimed, scopedKey } = opts;
|
|
81
|
+
const out: UncoveredSc[] = [];
|
|
82
|
+
|
|
83
|
+
for (const { file, content } of readSpecs(specsDir)) {
|
|
84
|
+
if (isTemplateSpec(file)) continue;
|
|
85
|
+
for (const d of parseScDeclarations(content, file) as ScDeclaration[]) {
|
|
86
|
+
if (!d.checked) continue;
|
|
87
|
+
if (claimed.has(scopedKey(d.specFile, d.id))) continue;
|
|
88
|
+
if (isSuperseded(d.statement)) continue;
|
|
89
|
+
if (isBehavioralSC(d as never)) continue;
|
|
90
|
+
|
|
91
|
+
// A matcher that MATCHES is not cover; a matcher that PASSES is. The
|
|
92
|
+
// distinction is the whole point — `sync-sc-status` reports matched,
|
|
93
|
+
// failing and unmatchable as three separate states precisely because
|
|
94
|
+
// conflating the first two certifies a broken assertion.
|
|
95
|
+
const assertion = matchPattern(d as never);
|
|
96
|
+
if (assertion) {
|
|
97
|
+
try {
|
|
98
|
+
assertion(projectRoot);
|
|
99
|
+
continue;
|
|
100
|
+
} catch {
|
|
101
|
+
// Matched and failed. Not cover.
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
out.push({ id: d.id, specFile: d.specFile, statement: d.statement });
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The same list, grouped by the spec that declares it. */
|
|
111
|
+
export function uncoveredBySpec(uncovered: UncoveredSc[]): Map<string, string[]> {
|
|
112
|
+
const bySpec = new Map<string, string[]>();
|
|
113
|
+
for (const u of uncovered) {
|
|
114
|
+
bySpec.set(u.specFile, [...(bySpec.get(u.specFile) ?? []), u.id]);
|
|
115
|
+
}
|
|
116
|
+
return bySpec;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Which specs exceed what they are allowed, given a banked per-spec ratchet.
|
|
121
|
+
*
|
|
122
|
+
* Ratchets BOTH ways, like every other allowlist in this repo: a spec over
|
|
123
|
+
* its number fails, and so does a spec under it, because an entry that has
|
|
124
|
+
* stopped being a violation is an entry nobody will ever remove.
|
|
125
|
+
*/
|
|
126
|
+
export function ratchetViolations(
|
|
127
|
+
bySpec: Map<string, string[]>,
|
|
128
|
+
banked: Record<string, number>,
|
|
129
|
+
): string[] {
|
|
130
|
+
const out: string[] = [];
|
|
131
|
+
for (const [spec, ids] of bySpec) {
|
|
132
|
+
const allowed = banked[spec] ?? 0;
|
|
133
|
+
if (ids.length > allowed) {
|
|
134
|
+
out.push(
|
|
135
|
+
`${spec}: ${ids.length} ticked criteria have no cover of any kind, ${allowed} allowed. ` +
|
|
136
|
+
`New: ${ids.slice(allowed).join(", ")}. Write the test, or untick it.`,
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
for (const [spec, allowed] of Object.entries(banked)) {
|
|
141
|
+
const found = bySpec.get(spec)?.length ?? 0;
|
|
142
|
+
if (found < allowed) {
|
|
143
|
+
out.push(
|
|
144
|
+
`${spec}: banked at ${allowed} but only ${found} remain. Lower the number — ` +
|
|
145
|
+
`a ratchet nobody lowers is a number nobody reads.`,
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return out.sort();
|
|
150
|
+
}
|
package/lib/sc-guard.ts
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SC Guard — Core validation logic for spec SC lines.
|
|
3
|
+
*
|
|
4
|
+
* Detects new SC lines in spec edits and validates they are either:
|
|
5
|
+
* 1. Matchable by the matcher registry (testable)
|
|
6
|
+
* 2. Explicitly marked (behavioral) with justification
|
|
7
|
+
*
|
|
8
|
+
* Deep module: hook file is thin, all logic lives here.
|
|
9
|
+
*
|
|
10
|
+
* Issue: #591
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { readFileSync, existsSync } from "fs";
|
|
14
|
+
import { join, dirname } from "path";
|
|
15
|
+
import { fileURLToPath } from "url";
|
|
16
|
+
import { spawnSync } from "child_process";
|
|
17
|
+
import { parseFrontmatter, isMatchablePattern, isBehavioralSC } from "./conformity";
|
|
18
|
+
import type { ParsedSC } from "./conformity";
|
|
19
|
+
|
|
20
|
+
// ── Types ──────────────────────────────────────────────────
|
|
21
|
+
|
|
22
|
+
export interface NewSC {
|
|
23
|
+
id: string;
|
|
24
|
+
statement: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface SCValidationResult {
|
|
28
|
+
scId: string;
|
|
29
|
+
statement: string;
|
|
30
|
+
blocked: boolean;
|
|
31
|
+
reason: string;
|
|
32
|
+
message: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// ── SC Detection ───────────────────────────────────────────
|
|
36
|
+
|
|
37
|
+
const SC_LINE_PATTERN = /^- \[ \] (SC-\w+):\s*(.+)$/gm;
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Extract all unchecked SC lines from content.
|
|
41
|
+
*/
|
|
42
|
+
function extractSCLines(content: string): NewSC[] {
|
|
43
|
+
const scs: NewSC[] = [];
|
|
44
|
+
const pattern = new RegExp(SC_LINE_PATTERN.source, "gm");
|
|
45
|
+
let match;
|
|
46
|
+
while ((match = pattern.exec(content)) !== null) {
|
|
47
|
+
scs.push({ id: match[1], statement: match[2].trim() });
|
|
48
|
+
}
|
|
49
|
+
return scs;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Detect new SC lines added in newContent that were not in oldContent.
|
|
54
|
+
* Compares by SC id — if an SC id exists in old content, it's not new.
|
|
55
|
+
*/
|
|
56
|
+
export function detectNewSCs(oldContent: string, newContent: string): NewSC[] {
|
|
57
|
+
const oldSCs = extractSCLines(oldContent);
|
|
58
|
+
const newSCs = extractSCLines(newContent);
|
|
59
|
+
const oldIds = new Set(oldSCs.map(sc => sc.id));
|
|
60
|
+
return newSCs.filter(sc => !oldIds.has(sc.id));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// ── SC Validation ──────────────────────────────────────────
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Validate new SCs against the matcher registry and behavioral rules.
|
|
67
|
+
*
|
|
68
|
+
* Returns only SCs that have issues (unmatchable without behavioral suffix).
|
|
69
|
+
* Empty array = all SCs are valid.
|
|
70
|
+
*
|
|
71
|
+
* - strict mode: blocked = true for unmatchable SCs
|
|
72
|
+
* - permissive mode: blocked = false (warn only)
|
|
73
|
+
*/
|
|
74
|
+
export function validateNewSCs(
|
|
75
|
+
newSCs: NewSC[],
|
|
76
|
+
compliance: "strict" | "permissive",
|
|
77
|
+
projectRoot?: string,
|
|
78
|
+
): SCValidationResult[] {
|
|
79
|
+
const results: SCValidationResult[] = [];
|
|
80
|
+
|
|
81
|
+
for (const sc of newSCs) {
|
|
82
|
+
const parsed: ParsedSC = {
|
|
83
|
+
id: sc.id,
|
|
84
|
+
statement: sc.statement,
|
|
85
|
+
specFile: "",
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
// Check if SC has (behavioral) suffix
|
|
89
|
+
if (isBehavioralSC(parsed)) {
|
|
90
|
+
continue; // Explicitly behavioral = allowed
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Check if SC matches any pattern in the registry
|
|
94
|
+
if (isMatchablePattern(parsed, projectRoot)) {
|
|
95
|
+
continue; // Matchable = allowed
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Unmatchable and not behavioral — generate validation result
|
|
99
|
+
const blocked = compliance === "strict";
|
|
100
|
+
const reason = blocked
|
|
101
|
+
? `No matching pattern found in matcher-registry.json`
|
|
102
|
+
: `No matching pattern found (permissive mode — warn only)`;
|
|
103
|
+
|
|
104
|
+
const message = blocked
|
|
105
|
+
? `${sc.id}: Blocked — no matching pattern found. ` +
|
|
106
|
+
`Either rewrite the SC to match a pattern from config/matcher-registry.json ` +
|
|
107
|
+
`(use \`bun scripts/create-sc.ts --list\` to see available patterns), ` +
|
|
108
|
+
`or add the (behavioral) suffix if this SC requires runtime verification.`
|
|
109
|
+
: `${sc.id}: Warning (permissive) — no matching pattern found. ` +
|
|
110
|
+
`Consider rewriting to match a pattern from config/matcher-registry.json ` +
|
|
111
|
+
`or adding the (behavioral) suffix.`;
|
|
112
|
+
|
|
113
|
+
results.push({
|
|
114
|
+
scId: sc.id,
|
|
115
|
+
statement: sc.statement,
|
|
116
|
+
blocked,
|
|
117
|
+
reason,
|
|
118
|
+
message,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return results;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ── Git diff helper ────────────────────────────────────────
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Get the last committed version of a file from git.
|
|
129
|
+
* Returns empty string if file is not tracked.
|
|
130
|
+
*/
|
|
131
|
+
export function getLastCommittedContent(filePath: string, cwd?: string): string {
|
|
132
|
+
try {
|
|
133
|
+
const result = spawnSync("git", ["show", `HEAD:${filePath}`], {
|
|
134
|
+
cwd: cwd ?? process.cwd(),
|
|
135
|
+
encoding: "utf-8",
|
|
136
|
+
timeout: 5000,
|
|
137
|
+
});
|
|
138
|
+
if (result.status === 0) {
|
|
139
|
+
return result.stdout;
|
|
140
|
+
}
|
|
141
|
+
return "";
|
|
142
|
+
} catch {
|
|
143
|
+
return "";
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// ── Frontmatter helpers ────────────────────────────────────
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Extract compliance mode from spec frontmatter.
|
|
151
|
+
* Defaults to "strict" if not specified.
|
|
152
|
+
*/
|
|
153
|
+
export function getComplianceMode(content: string): "strict" | "permissive" {
|
|
154
|
+
const fm = parseFrontmatter(content);
|
|
155
|
+
if (fm?.compliance === "permissive") return "permissive";
|
|
156
|
+
return "strict";
|
|
157
|
+
}
|