rungate 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -0
- package/gates/SCHEMA-GUIDE.md +485 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +283 -0
- package/gates/brief-assembler.test.ts +487 -0
- package/gates/brief-assembler.ts +448 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +250 -0
- package/gates/error-classifier.ts +99 -0
- package/gates/gate-executor.ts +3071 -0
- package/gates/orchestrator.test.ts +968 -0
- package/gates/orchestrator.ts +827 -0
- package/gates/preload.ts +89 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prompts.sha256 +3 -0
- package/gates/prove.test.ts +164 -0
- package/gates/rook-state.test.ts +108 -0
- package/gates/run-attribution.ts +290 -0
- package/gates/run-gate.ts +163 -0
- package/gates/schema-parity.test.ts +378 -0
- package/gates/schema.ts +1209 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +125 -0
- package/gates/ship-orchestrator.test.ts +399 -0
- package/gates/ship-orchestrator.ts +384 -0
- package/gates/spec-compliance-probe.ts +159 -0
- package/gates/test-fixtures/broken-no-environments.json +110 -0
- package/gates/test-fixtures/consumer-directory-layout.json +90 -0
- package/gates/test-fixtures/valid-standard.json +91 -0
- package/gates/test-utils.ts +43 -0
- package/gates/typecheck-gate.ts +111 -0
- package/gates/witness.ts +349 -0
- package/gates/workflow.test.ts +2018 -0
- package/hooks/BashWriteGuard.hook.ts +19 -0
- package/hooks/GateResultProtection.hook.ts +27 -1
- package/hooks/VerifyPhaseLock.hook.ts +51 -11
- package/hooks/WorkflowStateGuard.hook.ts +24 -0
- package/lib/ai-review-rounds.ts +462 -0
- package/lib/bash-file-read.ts +17 -3
- package/lib/bash-write-guard.ts +1285 -50
- package/lib/distribution-standing.ts +75 -4
- package/lib/failure-intake.ts +12 -4
- package/lib/file-extensions.ts +81 -0
- package/lib/finding-arbiter.ts +1015 -0
- package/lib/finding-jury.ts +537 -0
- package/lib/gate-enforcement.ts +171 -43
- package/lib/gate-result-protection.ts +12 -2
- package/lib/generators/code-map.ts +55 -4
- package/lib/generators/types.ts +27 -1
- package/lib/github.ts +55 -0
- package/lib/interpreters.ts +64 -0
- package/lib/issue-create-override.ts +69 -8
- package/lib/issue-filing-gate.ts +605 -0
- package/lib/js-top-level-code.ts +20 -2
- package/lib/junit-checks.ts +239 -1
- package/lib/package-closure.ts +114 -0
- package/lib/publish-window.ts +287 -0
- package/lib/scaffold/ai-review-findings.ts +142 -3
- package/lib/scaffold/ai-review-step.ts +1012 -18
- package/lib/scaffold/managed-workflow.ts +163 -4
- package/lib/scaffold/steps.ts +498 -22
- package/lib/scanner.ts +13 -4
- package/lib/ticked-sc-symbols.ts +361 -0
- package/lib/transcript-checker.ts +6 -0
- package/lib/workflow-actions.ts +18 -0
- package/package.json +3 -1
- package/scripts/arbitrate-finding.ts +224 -0
- package/scripts/generate-code-map.ts +18 -10
- package/scripts/sync-sc-status.ts +82 -8
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A ticked criterion that names a function nobody wrote (#1161).
|
|
3
|
+
*
|
|
4
|
+
* `specs/INSTRUCTION-COMPLIANCE-SPEC.md` said both of these, 167 lines apart:
|
|
5
|
+
*
|
|
6
|
+
* ```
|
|
7
|
+
* :86 **Implementation:** `lib/compliance.ts → runBehavioralCompliance(root)` (to be built)
|
|
8
|
+
* :253 - [x] SC-453: runBehavioralCompliance spawns fresh agent + auditor
|
|
9
|
+
* ```
|
|
10
|
+
*
|
|
11
|
+
* No declaration, no export, no caller, no test. The only occurrence of the
|
|
12
|
+
* name in executable source was a docblock in `lib/compliance.ts` describing
|
|
13
|
+
* it as future work. A tick is read as evidence — by agents and by people,
|
|
14
|
+
* and specifically by a session checking whether something is already done —
|
|
15
|
+
* so one tick with nothing behind it means the list cannot be used without
|
|
16
|
+
* re-verifying every entry, which is the opposite of what it is for.
|
|
17
|
+
*
|
|
18
|
+
* ## What this binding decides
|
|
19
|
+
*
|
|
20
|
+
* One question, in one place: **does a ticked criterion name a camelCase
|
|
21
|
+
* identifier that occurs nowhere in this tree's executable source or
|
|
22
|
+
* structured data?** That is the narrowest mechanically-checkable reading of
|
|
23
|
+
* the defect, and narrow is deliberate — the limits below are the honest
|
|
24
|
+
* statement of what a green run here does NOT mean.
|
|
25
|
+
*
|
|
26
|
+
* ## The limits, stated rather than implied
|
|
27
|
+
*
|
|
28
|
+
* - **A criterion naming a BEHAVIOUR is not covered, and most do.** "Auditor
|
|
29
|
+
* after every agent" names no symbol, so nothing here can tell whether it
|
|
30
|
+
* is true. Measured on the tree at the time of writing: 1,450 ticked
|
|
31
|
+
* criteria, 398 of them naming any symbol at all. This check is a floor on
|
|
32
|
+
* honesty, not a measure of it, and the 1,052 it cannot see are the reason
|
|
33
|
+
* it must not be cited as "the SC list is verified".
|
|
34
|
+
* - **camelCase only.** Every symbol this repository writes has an interior
|
|
35
|
+
* capital, and demanding one is what keeps the scan off English prose. A
|
|
36
|
+
* lowercase-only or snake_case name would be missed.
|
|
37
|
+
* - **An identifier is "present" if it occurs anywhere in comment-stripped
|
|
38
|
+
* source or in a JSON/YAML file**, including inside a string literal. That
|
|
39
|
+
* is wider than "declared", and wider in the safe direction: a criterion is
|
|
40
|
+
* flagged only when the name appears nowhere at all. Markdown is excluded
|
|
41
|
+
* everywhere, because markdown is where claims are made, not where they are
|
|
42
|
+
* satisfied — indexing `specs/` would let every criterion clear itself.
|
|
43
|
+
* - **Comments are stripped first**, and that is the whole reason the
|
|
44
|
+
* original instance is detectable. `runBehavioralCompliance` DID occur in
|
|
45
|
+
* `lib/compliance.ts`; a detector reading raw text would have cleared it.
|
|
46
|
+
* `test/ticked-sc-absent-symbol.test.ts` plants that exact case in both
|
|
47
|
+
* directions rather than trusting this paragraph.
|
|
48
|
+
*/
|
|
49
|
+
import { existsSync, readFileSync, readdirSync } from "fs";
|
|
50
|
+
import { basename, join } from "path";
|
|
51
|
+
|
|
52
|
+
import { isSuperseded } from "./sc-cover";
|
|
53
|
+
import { parseScDeclarations, isTemplateSpec } from "./sc-ids";
|
|
54
|
+
import { collectSpecFiles } from "./spec-files";
|
|
55
|
+
import { stripJsComments } from "./strip-js-comments";
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Where a symbol can be said to exist.
|
|
59
|
+
*
|
|
60
|
+
* `specs/`, `docs/` and `reference/` are deliberately absent: a name occurring
|
|
61
|
+
* only in prose is the condition being detected, not a refutation of it.
|
|
62
|
+
*/
|
|
63
|
+
export const SYMBOL_SEARCH_DIRS = [
|
|
64
|
+
".claude",
|
|
65
|
+
".github",
|
|
66
|
+
"bin",
|
|
67
|
+
"config",
|
|
68
|
+
"gates",
|
|
69
|
+
"hooks",
|
|
70
|
+
"lib",
|
|
71
|
+
"prompts",
|
|
72
|
+
"scripts",
|
|
73
|
+
"templates",
|
|
74
|
+
"test",
|
|
75
|
+
"workflows",
|
|
76
|
+
] as const;
|
|
77
|
+
|
|
78
|
+
/** The bank of known rows, relative to the project root. */
|
|
79
|
+
export const ABSENT_SYMBOL_RATCHET = ".claude/ticked-sc-absent-symbol-ratchet.json";
|
|
80
|
+
|
|
81
|
+
/** The suite that drives this binding, relative to the project root. */
|
|
82
|
+
export const ABSENT_SYMBOL_TEST = "test/ticked-sc-absent-symbol.test.ts";
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* The two files that would answer their own question, excluded from the index.
|
|
86
|
+
*
|
|
87
|
+
* Both are inside `SYMBOL_SEARCH_DIRS` and both name, in indexable positions,
|
|
88
|
+
* the identifiers this scan is looking for:
|
|
89
|
+
*
|
|
90
|
+
* - the ratchet is a `.json` under `.claude/` listing every banked symbol,
|
|
91
|
+
* so indexing it makes every row it holds "present" and the scan returns
|
|
92
|
+
* zero. An allowlist naming the exempt things becomes evidence they exist.
|
|
93
|
+
* - the test spells `runBehavioralCompliance` in a string literal, which is
|
|
94
|
+
* code and survives comment-stripping. Measured: with the test indexed,
|
|
95
|
+
* restoring the original ticked criterion to
|
|
96
|
+
* `specs/INSTRUCTION-COMPLIANCE-SPEC.md` scored ZERO red — the suite
|
|
97
|
+
* certified a tree carrying the exact defect it was written for.
|
|
98
|
+
*
|
|
99
|
+
* It is a constant here rather than an argument because a caller who forgets
|
|
100
|
+
* it gets a clean report, which is the failure mode that is hardest to see.
|
|
101
|
+
*/
|
|
102
|
+
export const SELF_REFERENTIAL_FILES: readonly string[] = [
|
|
103
|
+
ABSENT_SYMBOL_RATCHET,
|
|
104
|
+
ABSENT_SYMBOL_TEST,
|
|
105
|
+
];
|
|
106
|
+
|
|
107
|
+
/** Executable source: comments are stripped before indexing. */
|
|
108
|
+
const CODE_EXTENSION = /\.(ts|tsx|js|mjs|cjs)$/;
|
|
109
|
+
|
|
110
|
+
/** Structured data: indexed verbatim — a JSON key is a thing that exists. */
|
|
111
|
+
const DATA_EXTENSION = /\.(json|ya?ml)$/;
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* A camelCase identifier: lowercase start, at least one interior capital.
|
|
115
|
+
*
|
|
116
|
+
* The interior capital is the whole of the prose filter. Without it the scan
|
|
117
|
+
* reports every English word in 1,450 criteria; with it, the only false
|
|
118
|
+
* positives measured on this tree were metavariables inside a quoted code
|
|
119
|
+
* sample, and those are banked by name in the ratchet rather than excused by
|
|
120
|
+
* widening this pattern.
|
|
121
|
+
*/
|
|
122
|
+
const CAMEL_CASE = /\b[a-z][A-Za-z0-9_$]*[A-Z][A-Za-z0-9_$]*\b/g;
|
|
123
|
+
|
|
124
|
+
/** The camelCase symbols a criterion names. */
|
|
125
|
+
export function symbolsNamedBy(statement: string): string[] {
|
|
126
|
+
return [...new Set([...statement.matchAll(CAMEL_CASE)].map(m => m[0]))].sort();
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* A file artifact: a basename and a known extension (#1217).
|
|
131
|
+
*
|
|
132
|
+
* `CAMEL_CASE` cannot see one. Every symbol this repo writes has an interior
|
|
133
|
+
* capital and every file it writes is kebab-case, so the two populations do
|
|
134
|
+
* not overlap at all — which is why this is a second pattern and not a wider
|
|
135
|
+
* first one. Measured: SC-135 and SC-108 are the same defect, one orphaned
|
|
136
|
+
* `workflow-state.json` field each, and the camelCase sweep saw only SC-135
|
|
137
|
+
* because SC-108 spells its artifact `quinn-journey.yaml` and names no field.
|
|
138
|
+
*
|
|
139
|
+
* The extension is the whole of the prose filter, and it has to be: a bare
|
|
140
|
+
* hyphenated token matches "non-coding", "intent-based" and "near-enough".
|
|
141
|
+
* Measured on this tree, hyphen-only scored 247 rows of which essentially all
|
|
142
|
+
* were English; requiring an extension scores 1.
|
|
143
|
+
*/
|
|
144
|
+
const FILE_ARTIFACT = /\b[A-Za-z0-9_][A-Za-z0-9_.-]*\.(?:json|ts|tsx|js|mjs|cjs|ya?ml|sh)\b/g;
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* The file artifacts a criterion names, as basenames.
|
|
148
|
+
*
|
|
149
|
+
* Basenames on both sides, because a criterion writes the path it cares about
|
|
150
|
+
* — `~/.rungate/{slug}/quinn-journey.yaml`, `.rungate/navigability-score.json`
|
|
151
|
+
* — and the tree writes a different one for the same file. Comparing paths
|
|
152
|
+
* would report every artifact in the spec list.
|
|
153
|
+
*/
|
|
154
|
+
export function artifactsNamedBy(statement: string): string[] {
|
|
155
|
+
return [
|
|
156
|
+
...new Set([...withoutGlobs(statement).matchAll(FILE_ARTIFACT)].map(m => basename(m[0]))),
|
|
157
|
+
].sort();
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Drop whitespace-delimited tokens carrying a wildcard — they are patterns.
|
|
162
|
+
*
|
|
163
|
+
* SC-1690 of `CONFIG-DIRECTORY-STRUCTURE-SPEC.md` names `src/**` + `/*.spec.tsx`
|
|
164
|
+
* as a glob the census counts. Nothing is being claimed to exist, and
|
|
165
|
+
* `basename` on it yields `spec.tsx`, a file that cannot exist under that
|
|
166
|
+
* name. Unlike SC-678's metavariables — where narrowing the pattern would
|
|
167
|
+
* also have stopped catching a real symbol written inside a code span — a
|
|
168
|
+
* glob never claims a specific file, so there is no defect this hides.
|
|
169
|
+
*/
|
|
170
|
+
function withoutGlobs(statement: string): string {
|
|
171
|
+
return statement
|
|
172
|
+
.split(/\s+/)
|
|
173
|
+
.filter(tok => !tok.includes("*") && !tok.includes("?"))
|
|
174
|
+
.join(" ");
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* `test.todo("SC-237: …")` is a declaration that something is NOT built.
|
|
179
|
+
*
|
|
180
|
+
* This is the comment case again in a position comment-stripping cannot
|
|
181
|
+
* reach. `test/phase-5.test.ts` holds `test.todo("SC-237: navigability-score
|
|
182
|
+
* .json has required fields")`, so a detector indexing string literals reads
|
|
183
|
+
* the repository's own statement that the file is unwritten as evidence that
|
|
184
|
+
* it exists. Measured, one input both ways: SC-237 scores a row with todo
|
|
185
|
+
* titles dropped and scores none with them indexed.
|
|
186
|
+
*
|
|
187
|
+
* Only the TITLE is dropped, not the call — a `todo` is still a token, and
|
|
188
|
+
* widening this to the whole statement would hide real references in the
|
|
189
|
+
* same file.
|
|
190
|
+
*/
|
|
191
|
+
const TODO_TITLE = /\.todo\s*\(\s*(["'`])(?:\\.|(?!\1)[^\\])*\1/g;
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Every identifier occurring in executable source or structured data.
|
|
195
|
+
*
|
|
196
|
+
* Returned as a bare set rather than a per-file index on purpose: the question
|
|
197
|
+
* is existence, and a caller handed file positions would be tempted to report
|
|
198
|
+
* "declared in X", which an identifier scan cannot honestly say.
|
|
199
|
+
*/
|
|
200
|
+
export function codeIdentifiers(
|
|
201
|
+
root: string,
|
|
202
|
+
dirs: readonly string[] = SYMBOL_SEARCH_DIRS,
|
|
203
|
+
exclude: readonly string[] = SELF_REFERENTIAL_FILES,
|
|
204
|
+
): Set<string> {
|
|
205
|
+
return indexTree(root, dirs, exclude).identifiers;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* One walk, both indexes.
|
|
210
|
+
*
|
|
211
|
+
* The two halves were separate functions when the artifact index landed, and
|
|
212
|
+
* the real tree's self-reference case then read the whole tree FOUR times
|
|
213
|
+
* instead of twice and timed out at 5,995 ms against bun's 5,000 ms default.
|
|
214
|
+
* Reading every file twice to answer two questions about the same bytes is
|
|
215
|
+
* the kind of cost that gets paid for by raising a budget, which `bunfig
|
|
216
|
+
* .toml` says this suite will not do — so it is folded here instead. With one
|
|
217
|
+
* walk the same case runs in about half that.
|
|
218
|
+
*
|
|
219
|
+
* **Artifact presence is two readings, unioned, because neither alone is
|
|
220
|
+
* honest:**
|
|
221
|
+
*
|
|
222
|
+
* - **on disk.** `lib/rule-registry.ts` exists and its name occurs inside no
|
|
223
|
+
* other file, because an import spells it `./rule-registry`. A text-only
|
|
224
|
+
* reading scores 252 rows on this tree and 251 of them are files you can
|
|
225
|
+
* `ls`.
|
|
226
|
+
* - **named in indexed text.** `.rungate/navigability-score.json` is a
|
|
227
|
+
* runtime output; a clean checkout never contains one, so a
|
|
228
|
+
* filesystem-only reading would demand that generated artifacts be
|
|
229
|
+
* committed.
|
|
230
|
+
*/
|
|
231
|
+
function indexTree(
|
|
232
|
+
root: string,
|
|
233
|
+
dirs: readonly string[] = SYMBOL_SEARCH_DIRS,
|
|
234
|
+
exclude: readonly string[] = SELF_REFERENTIAL_FILES,
|
|
235
|
+
): { identifiers: Set<string>; artifacts: Set<string> } {
|
|
236
|
+
const skip = new Set(exclude);
|
|
237
|
+
const identifiers = new Set<string>();
|
|
238
|
+
const artifacts = new Set<string>();
|
|
239
|
+
for (const d of dirs) {
|
|
240
|
+
const dir = join(root, d);
|
|
241
|
+
if (!existsSync(dir)) continue;
|
|
242
|
+
for (const entry of readdirSync(dir, { recursive: true, encoding: "utf-8" })) {
|
|
243
|
+
if (typeof entry !== "string") continue;
|
|
244
|
+
// A file that exists is present, whatever its extension and whether or
|
|
245
|
+
// not anything names it.
|
|
246
|
+
artifacts.add(basename(entry));
|
|
247
|
+
if (skip.has(`${d}/${entry}`)) continue;
|
|
248
|
+
const isCode = CODE_EXTENSION.test(entry);
|
|
249
|
+
if (!isCode && !DATA_EXTENSION.test(entry)) continue;
|
|
250
|
+
const raw = readFileSync(join(dir, entry), "utf-8");
|
|
251
|
+
const text = isCode ? stripJsComments(raw) : raw;
|
|
252
|
+
for (const m of text.matchAll(/[A-Za-z_$][A-Za-z0-9_$]*/g)) identifiers.add(m[0]);
|
|
253
|
+
const named = isCode ? text.replace(TODO_TITLE, ".todo(") : text;
|
|
254
|
+
for (const m of named.matchAll(FILE_ARTIFACT)) artifacts.add(basename(m[0]));
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return { identifiers, artifacts };
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export interface AbsentSymbol {
|
|
261
|
+
id: string;
|
|
262
|
+
specFile: string;
|
|
263
|
+
statement: string;
|
|
264
|
+
/** The identifier the criterion names and the tree does not contain. */
|
|
265
|
+
symbol: string;
|
|
266
|
+
/** `<spec>::<id>::<symbol>` — the row the ratchet banks. */
|
|
267
|
+
key: string;
|
|
268
|
+
/** Which pattern named it. The bank's key shape is the same for both. */
|
|
269
|
+
kind: "symbol" | "artifact";
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
export interface AbsentSymbolScan {
|
|
273
|
+
/** Ticked criteria naming a symbol the tree does not have, one row per symbol. */
|
|
274
|
+
violations: AbsentSymbol[];
|
|
275
|
+
/** Ticked criteria read, so a clean result can be told from a dead scan. */
|
|
276
|
+
tickedScanned: number;
|
|
277
|
+
/** Ticked criteria naming any symbol at all — the population actually at risk. */
|
|
278
|
+
tickedNamingSymbols: number;
|
|
279
|
+
/** Ticked criteria naming any file artifact — the second population (#1217). */
|
|
280
|
+
tickedNamingArtifacts: number;
|
|
281
|
+
/** Identifiers indexed out of source and data. */
|
|
282
|
+
identifiersIndexed: number;
|
|
283
|
+
/** File artifacts found on disk or named in indexed text (#1217). */
|
|
284
|
+
artifactsIndexed: number;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export function absentSymbolKey(specFile: string, id: string, symbol: string): string {
|
|
288
|
+
return `${specFile}::${id}::${symbol}`;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Scan a tree for ticked criteria naming an absent symbol.
|
|
293
|
+
*
|
|
294
|
+
* The three counts come back whether or not anything was found, because an
|
|
295
|
+
* empty violation list and a scan that read nothing produce identical output
|
|
296
|
+
* otherwise — the third shape in
|
|
297
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`.
|
|
298
|
+
*/
|
|
299
|
+
export function scanTickedScSymbols(opts: {
|
|
300
|
+
specsDir: string;
|
|
301
|
+
projectRoot: string;
|
|
302
|
+
dirs?: readonly string[];
|
|
303
|
+
exclude?: readonly string[];
|
|
304
|
+
}): AbsentSymbolScan {
|
|
305
|
+
const { identifiers: ids, artifacts } = indexTree(opts.projectRoot, opts.dirs, opts.exclude);
|
|
306
|
+
const violations: AbsentSymbol[] = [];
|
|
307
|
+
let tickedScanned = 0;
|
|
308
|
+
let tickedNamingSymbols = 0;
|
|
309
|
+
let tickedNamingArtifacts = 0;
|
|
310
|
+
|
|
311
|
+
for (const rel of collectSpecFiles(opts.specsDir)) {
|
|
312
|
+
if (isTemplateSpec(rel)) continue;
|
|
313
|
+
const content = readFileSync(join(opts.specsDir, rel), "utf-8");
|
|
314
|
+
for (const sc of parseScDeclarations(content, rel)) {
|
|
315
|
+
if (!sc.checked) continue;
|
|
316
|
+
// A SUPERSEDED line records what a criterion USED to claim, so its
|
|
317
|
+
// symbol is expected to be gone. Demanding the dead name still resolve
|
|
318
|
+
// would push specs to delete their own history —
|
|
319
|
+
// `test/ticked-sc-cover.test.ts` makes the same exemption for the same
|
|
320
|
+
// reason.
|
|
321
|
+
if (isSuperseded(sc.statement)) continue;
|
|
322
|
+
tickedScanned++;
|
|
323
|
+
const named = symbolsNamedBy(sc.statement);
|
|
324
|
+
if (named.length > 0) tickedNamingSymbols++;
|
|
325
|
+
for (const symbol of named) {
|
|
326
|
+
if (ids.has(symbol)) continue;
|
|
327
|
+
violations.push({
|
|
328
|
+
id: sc.id,
|
|
329
|
+
specFile: rel,
|
|
330
|
+
statement: sc.statement,
|
|
331
|
+
symbol,
|
|
332
|
+
key: absentSymbolKey(rel, sc.id, symbol),
|
|
333
|
+
kind: "symbol",
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
const files = artifactsNamedBy(sc.statement);
|
|
337
|
+
if (files.length > 0) tickedNamingArtifacts++;
|
|
338
|
+
for (const symbol of files) {
|
|
339
|
+
if (artifacts.has(symbol)) continue;
|
|
340
|
+
violations.push({
|
|
341
|
+
id: sc.id,
|
|
342
|
+
specFile: rel,
|
|
343
|
+
statement: sc.statement,
|
|
344
|
+
symbol,
|
|
345
|
+
key: absentSymbolKey(rel, sc.id, symbol),
|
|
346
|
+
kind: "artifact",
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
violations.sort((a, b) => a.key.localeCompare(b.key));
|
|
353
|
+
return {
|
|
354
|
+
violations,
|
|
355
|
+
tickedScanned,
|
|
356
|
+
tickedNamingSymbols,
|
|
357
|
+
tickedNamingArtifacts,
|
|
358
|
+
identifiersIndexed: ids.size,
|
|
359
|
+
artifactsIndexed: artifacts.size,
|
|
360
|
+
};
|
|
361
|
+
}
|
|
@@ -464,6 +464,12 @@ export const SOURCE_DIRS = [
|
|
|
464
464
|
// unlisted directory is not a test failure, it is TDD going ungraded for
|
|
465
465
|
// everything written there.
|
|
466
466
|
"bin",
|
|
467
|
+
// #1169 added the first `.ts` under evals/ — the reviewer negative control
|
|
468
|
+
// and its subjects — and the derived test caught it on the first CI run,
|
|
469
|
+
// exactly as it caught bin/. Listing it is the fix the comment above
|
|
470
|
+
// prescribes: an eval that executes is implementation, and leaving it out
|
|
471
|
+
// would grade every future write there as NO_SOURCE.
|
|
472
|
+
"evals",
|
|
467
473
|
] as const;
|
|
468
474
|
|
|
469
475
|
/** Whether a written path is implementation rather than test or docs. */
|
package/lib/workflow-actions.ts
CHANGED
|
@@ -145,6 +145,24 @@ export const PINNING: readonly ActionPin[] = [
|
|
|
145
145
|
version: "v4",
|
|
146
146
|
why: "GitHub's own namespace on GitHub's own runner; the tag is how a security patch reaches a consumer who is not watching",
|
|
147
147
|
},
|
|
148
|
+
// #1190. The shard aggregate's evidence channel. These are the two actions
|
|
149
|
+
// that let a shard prove it EXECUTED without the aggregate asking the
|
|
150
|
+
// Actions API: `gh` is only on GitHub's own runner images and `actions:
|
|
151
|
+
// read` is a scope a job has to be granted, whereas the artifact service
|
|
152
|
+
// is reached by the runner itself for the run it is already in. Same
|
|
153
|
+
// first-party reasoning as `actions/checkout` above.
|
|
154
|
+
{
|
|
155
|
+
action: "actions/upload-artifact",
|
|
156
|
+
style: "tag",
|
|
157
|
+
version: "v4",
|
|
158
|
+
why: "GitHub's own namespace on GitHub's own runner; the tag is how a security patch reaches a consumer who is not watching",
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
action: "actions/download-artifact",
|
|
162
|
+
style: "tag",
|
|
163
|
+
version: "v4",
|
|
164
|
+
why: "GitHub's own namespace on GitHub's own runner; the tag is how a security patch reaches a consumer who is not watching",
|
|
165
|
+
},
|
|
148
166
|
{
|
|
149
167
|
action: "oven-sh/setup-bun",
|
|
150
168
|
style: "sha",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "rungate",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.2",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Ship harness — conformity tests, scaffold, and agent briefs for AI-first development",
|
|
6
6
|
"repository": {
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
"files": [
|
|
15
15
|
"bin/",
|
|
16
16
|
"lib/",
|
|
17
|
+
"gates/",
|
|
17
18
|
"hooks/",
|
|
18
19
|
"prompts/",
|
|
19
20
|
"config/",
|
|
@@ -29,6 +30,7 @@
|
|
|
29
30
|
"scripts/create-success.ts",
|
|
30
31
|
"scripts/create-sc.ts",
|
|
31
32
|
"scripts/ai-review-findings.ts",
|
|
33
|
+
"scripts/arbitrate-finding.ts",
|
|
32
34
|
"scripts/split-spec.ts",
|
|
33
35
|
"scripts/sync-sc-status.ts",
|
|
34
36
|
"scripts/audit-transcript.ts",
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Decide what happens to a finding: post it, file it, or drop it (#1187).
|
|
4
|
+
*
|
|
5
|
+
* ## The default is a PR comment
|
|
6
|
+
*
|
|
7
|
+
* Exit **0** — post it as an inline comment that must be answered.
|
|
8
|
+
* Exit **3** — file it as an issue. Only reachable with `--out-of-scope`.
|
|
9
|
+
* Exit **1** — drop it; the refusal and its reason are on stdout.
|
|
10
|
+
* Exit **2** — usage.
|
|
11
|
+
*
|
|
12
|
+
* **Exit 1 is also what UNMEASURED produces.** A finding the arbiter could
|
|
13
|
+
* not evaluate is not a finding that passes; that is #1187's doctrine and the
|
|
14
|
+
* one behaviour here that must not be softened into a warning.
|
|
15
|
+
*
|
|
16
|
+
* ## Why this is a CLI and not a hook
|
|
17
|
+
*
|
|
18
|
+
* `hooks/VerifyPhaseLock.hook.ts` is already the place a `gh issue create`
|
|
19
|
+
* passes through, and it is the wrong place for this. The checks need the
|
|
20
|
+
* tracker and the repository: the dedupe reads every open issue, and branch
|
|
21
|
+
* scope runs `git grep` against `origin/main`. Putting a network round-trip
|
|
22
|
+
* and a tree-wide grep inside a PreToolUse hook buys a multi-second pause on
|
|
23
|
+
* an unrelated Bash call and a check that fails open the moment the network
|
|
24
|
+
* does — which is the failure `.claude/rules/config-access.md` names.
|
|
25
|
+
*
|
|
26
|
+
* #1187 records that the filing boundary has no sandbox around it: it runs in
|
|
27
|
+
* the orchestrating session, not inside `ai-review.yml`'s `--disallowedTools`.
|
|
28
|
+
* So this is a command the caller runs, and its exit status is the answer.
|
|
29
|
+
*
|
|
30
|
+
* bun scripts/arbitrate-finding.ts --title "…" --body-file /tmp/body.md \
|
|
31
|
+
* --repo owner/name --votes /tmp/votes.json
|
|
32
|
+
*
|
|
33
|
+
* `--votes` is a JSON array of `{ verifier, real, reason, category?,
|
|
34
|
+
* severity? }`. **It is not optional**: a finding nobody voted on has not
|
|
35
|
+
* cleared a quorum, it has not had one, and those are different states.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
import { execFileSync } from "child_process";
|
|
39
|
+
import { readFileSync } from "fs";
|
|
40
|
+
|
|
41
|
+
import {
|
|
42
|
+
arbitrate,
|
|
43
|
+
UNFILED_ID,
|
|
44
|
+
verdictSummary,
|
|
45
|
+
type Finding,
|
|
46
|
+
type Verdict,
|
|
47
|
+
} from "../lib/finding-arbiter";
|
|
48
|
+
import { parseVotes, renderComment, type VerifierVote } from "../lib/finding-jury";
|
|
49
|
+
|
|
50
|
+
interface Args {
|
|
51
|
+
title: string;
|
|
52
|
+
bodyFile: string;
|
|
53
|
+
repo: string;
|
|
54
|
+
existingFile: string | null;
|
|
55
|
+
votesFile: string | null;
|
|
56
|
+
changedFilesArg: string | null;
|
|
57
|
+
outOfScope: string | null;
|
|
58
|
+
projectRoot: string;
|
|
59
|
+
json: boolean;
|
|
60
|
+
comment: boolean;
|
|
61
|
+
limit: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Exit codes, each declared once so no branch can invent its own. */
|
|
65
|
+
export const EXIT_COMMENT = 0;
|
|
66
|
+
export const EXIT_DROP = 1;
|
|
67
|
+
export const EXIT_USAGE = 2;
|
|
68
|
+
export const EXIT_ISSUE = 3;
|
|
69
|
+
|
|
70
|
+
function parseArgs(argv: string[]): Args {
|
|
71
|
+
const get = (flag: string): string | null => {
|
|
72
|
+
const i = argv.indexOf(flag);
|
|
73
|
+
return i !== -1 && i + 1 < argv.length ? argv[i + 1] : null;
|
|
74
|
+
};
|
|
75
|
+
return {
|
|
76
|
+
title: get("--title") ?? "",
|
|
77
|
+
bodyFile: get("--body-file") ?? "",
|
|
78
|
+
repo: get("--repo") ?? "",
|
|
79
|
+
existingFile: get("--existing"),
|
|
80
|
+
votesFile: get("--votes"),
|
|
81
|
+
changedFilesArg: get("--changed-files"),
|
|
82
|
+
outOfScope: get("--out-of-scope"),
|
|
83
|
+
projectRoot: get("--project-root") ?? process.cwd(),
|
|
84
|
+
json: argv.includes("--json"),
|
|
85
|
+
comment: argv.includes("--comment"),
|
|
86
|
+
limit: Number.parseInt(get("--limit") ?? "60", 10),
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Raised when the comparison population could not be read in full. */
|
|
91
|
+
class TruncatedPopulation extends Error {}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The issues the new finding is compared against.
|
|
95
|
+
*
|
|
96
|
+
* Open issues only. A closed issue describes a symptom somebody already
|
|
97
|
+
* decided about, and refusing a finding because it matches a resolved one
|
|
98
|
+
* withholds the report that the fix did not hold.
|
|
99
|
+
*
|
|
100
|
+
* **`--limit` is requested as `limit + 1` and an extra row is a refusal.** A
|
|
101
|
+
* dedupe against a truncated corpus returns "no duplicate" indistinguishably
|
|
102
|
+
* from a real one, and the default of 60 against 27 open issues today is
|
|
103
|
+
* exactly the shape that ships quietly and bites later. This matches the
|
|
104
|
+
* no-input branch in `main`, which already refuses rather than passing.
|
|
105
|
+
*
|
|
106
|
+
* `--existing` takes the same JSON offline, which is how the comparison set
|
|
107
|
+
* is pinned for a test or a measurement; it is the caller's own population
|
|
108
|
+
* and so is never truncation-checked.
|
|
109
|
+
*/
|
|
110
|
+
function existingFindings(args: Args): Finding[] {
|
|
111
|
+
let raw: string;
|
|
112
|
+
if (args.existingFile) {
|
|
113
|
+
raw = readFileSync(args.existingFile, "utf-8");
|
|
114
|
+
} else {
|
|
115
|
+
raw = execFileSync(
|
|
116
|
+
"gh",
|
|
117
|
+
["issue", "list", "--repo", args.repo, "--state", "open",
|
|
118
|
+
"--limit", String(args.limit + 1), "--json", "number,title,body"],
|
|
119
|
+
{ encoding: "utf-8", maxBuffer: 32 * 1024 * 1024 },
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
const parsed = JSON.parse(raw) as Array<{ number?: number; id?: string; title: string; body?: string }>;
|
|
123
|
+
if (!args.existingFile && parsed.length > args.limit) {
|
|
124
|
+
throw new TruncatedPopulation(
|
|
125
|
+
`the tracker returned more than --limit ${args.limit} open issues, so the dedupe population is truncated — ` +
|
|
126
|
+
"raise --limit; a dedupe against a partial corpus reports no duplicate exactly as a complete one does",
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
return parsed.map((i, at) => {
|
|
130
|
+
// A DUPLICATE refusal withholds a finding, so it has to name WHERE the
|
|
131
|
+
// symptom is already reported or it is not reviewable. An entry with no
|
|
132
|
+
// identifier yields the id `"undefined"` and the refusal reads "already
|
|
133
|
+
// reported as undefined, undefined" — a refusal that cannot be acted on,
|
|
134
|
+
// which is worse than no refusal. Caught end-to-end against real tracker
|
|
135
|
+
// data, where a hand-built `--existing` used `id` and the reader wanted
|
|
136
|
+
// `number`; both are accepted now, and neither is guessed.
|
|
137
|
+
const id = i.number !== undefined ? String(i.number) : i.id;
|
|
138
|
+
if (id === undefined || id === "" || id === "undefined") {
|
|
139
|
+
throw new Error(
|
|
140
|
+
`entry ${at} of the existing-findings population has neither \`number\` nor \`id\` — ` +
|
|
141
|
+
"a dedupe that cannot name what a finding collides with produces an unreviewable refusal",
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
return { id, title: i.title, body: i.body ?? "" };
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The ballot, validated rather than cast (#1235).
|
|
150
|
+
*
|
|
151
|
+
* This used to check `Array.isArray` and then `parsed as VerifierVote[]`, so
|
|
152
|
+
* no entry was ever looked at — and `juryReading` counts agreement by
|
|
153
|
+
* truthiness, which made `"real": "false"` a yes. `parseVotes` owns the shape
|
|
154
|
+
* check; it throws, and `main`'s catch turns that into `DROP (UNMEASURED)`,
|
|
155
|
+
* because a ballot that could not be read is not a ballot that passed.
|
|
156
|
+
*/
|
|
157
|
+
function readVotes(path: string): readonly VerifierVote[] {
|
|
158
|
+
try {
|
|
159
|
+
return parseVotes(JSON.parse(readFileSync(path, "utf-8")));
|
|
160
|
+
} catch (error) {
|
|
161
|
+
throw new Error(`--votes ${path}: ${error instanceof Error ? error.message : String(error)}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function emit(args: Args, verdict: Verdict, finding: Finding): void {
|
|
166
|
+
if (args.json) {
|
|
167
|
+
console.log(JSON.stringify(verdict, null, 2));
|
|
168
|
+
} else if (args.comment && verdict.disposition === "COMMENT") {
|
|
169
|
+
console.log(renderComment(finding, verdict.severity ?? "Important", verdict.category));
|
|
170
|
+
} else {
|
|
171
|
+
console.log(verdictSummary(verdict));
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function main(): number {
|
|
176
|
+
const args = parseArgs(process.argv.slice(2));
|
|
177
|
+
if (!args.title || !args.bodyFile) {
|
|
178
|
+
console.error(
|
|
179
|
+
"usage: arbitrate-finding.ts --title <t> --body-file <p> --votes <p> " +
|
|
180
|
+
"[--repo owner/name | --existing issues.json] [--project-root <p>] " +
|
|
181
|
+
"[--changed-files a.ts,b.ts] [--out-of-scope <reason>] [--json|--comment] [--limit N]",
|
|
182
|
+
);
|
|
183
|
+
return EXIT_USAGE;
|
|
184
|
+
}
|
|
185
|
+
// Not usage errors: these are stages with no input, and the doctrine for an
|
|
186
|
+
// absent input is to refuse, not to skip it.
|
|
187
|
+
if (!args.repo && !args.existingFile) {
|
|
188
|
+
console.log("DROP new (UNMEASURED): no --repo and no --existing, so the dedupe has nothing to compare against");
|
|
189
|
+
return EXIT_DROP;
|
|
190
|
+
}
|
|
191
|
+
if (!args.votesFile) {
|
|
192
|
+
console.log("DROP new (UNMEASURED): no --votes, so no verifier group was convened — not reaching a quorum and not having one are different states");
|
|
193
|
+
return EXIT_DROP;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const finding: Finding = {
|
|
197
|
+
id: UNFILED_ID,
|
|
198
|
+
title: args.title,
|
|
199
|
+
body: readFileSync(args.bodyFile, "utf-8"),
|
|
200
|
+
...(args.outOfScope ? { outOfScope: { reason: args.outOfScope } } : {}),
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
let verdict: Verdict;
|
|
204
|
+
try {
|
|
205
|
+
verdict = arbitrate(finding, {
|
|
206
|
+
projectRoot: args.projectRoot,
|
|
207
|
+
existing: existingFindings(args),
|
|
208
|
+
votes: readVotes(args.votesFile),
|
|
209
|
+
...(args.changedFilesArg !== null
|
|
210
|
+
? { changedFiles: args.changedFilesArg.split(",").map(s => s.trim()).filter(Boolean) }
|
|
211
|
+
: {}),
|
|
212
|
+
});
|
|
213
|
+
} catch (error) {
|
|
214
|
+
const why = error instanceof Error ? error.message : String(error);
|
|
215
|
+
console.log(`DROP new (UNMEASURED): ${why}`);
|
|
216
|
+
return EXIT_DROP;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
emit(args, verdict, finding);
|
|
220
|
+
if (verdict.disposition === "ISSUE") return EXIT_ISSUE;
|
|
221
|
+
return verdict.disposition === "COMMENT" ? EXIT_COMMENT : EXIT_DROP;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
process.exit(main());
|