rungate 0.8.2 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/rungate.ts +11 -6
- package/config/prove-evidence.schema.json +21 -3
- package/gates/.gate-salt +1 -1
- package/gates/gate-executor.ts +153 -1
- package/gates/prove.test.ts +12 -6
- package/gates/schema.ts +69 -6
- package/gates/workflow.test.ts +12 -5
- package/hooks/BunTestPathGuard.hook.ts +106 -0
- package/hooks/DocHygiene.hook.ts +2 -1
- package/hooks/GateResultProtection.hook.ts +28 -7
- package/hooks/HookCurrency.hook.ts +48 -1
- package/hooks/TopLevelCdGuard.hook.ts +38 -6
- package/hooks/VerifyPhaseLock.hook.ts +25 -8
- package/hooks/WorkflowStateGuard.hook.ts +24 -7
- package/lib/ai-review-jury.ts +725 -0
- package/lib/ai-review-rounds.ts +791 -4
- package/lib/ai-review-unmeasured.ts +361 -0
- package/lib/bash-write-guard.ts +1235 -102
- package/lib/bun-test-paths.ts +253 -0
- package/lib/cli-commands.ts +252 -11
- package/lib/closing-reference-audit.ts +163 -0
- package/lib/conformity.ts +17 -1
- package/lib/create-sc.ts +45 -12
- package/lib/derived-spec.ts +206 -0
- package/lib/file-claims.ts +335 -0
- package/lib/finding-arbiter.ts +502 -28
- package/lib/finding-jury.ts +61 -5
- package/lib/finding-pr-routing.ts +333 -0
- package/lib/frontmatter.ts +16 -0
- package/lib/gate-check-audit.ts +646 -0
- package/lib/generators/agents-md.ts +29 -0
- package/lib/github.ts +112 -7
- package/lib/hook-tree-currency.ts +494 -0
- package/lib/issue-closure-audit.ts +161 -14
- package/lib/issue-create-override.ts +53 -3
- package/lib/issue-filing-gate.ts +1073 -76
- package/lib/local-review-rounds.ts +523 -0
- package/lib/organize.ts +21 -1
- package/lib/project-state-shape.ts +56 -5
- package/lib/release-preflight.ts +90 -1
- package/lib/repo-path.ts +94 -0
- package/lib/report-provenance.ts +218 -3
- package/lib/risk-tier.ts +302 -0
- package/lib/rounds-to-pass.ts +274 -0
- package/lib/rule-row-drift.ts +199 -0
- package/lib/sc-claim-forms.ts +33 -5
- package/lib/sc-guard.ts +11 -1
- package/lib/sc-id-cross-tree.ts +17 -2
- package/lib/sc-ids.ts +71 -6
- package/lib/sc-reservations.ts +158 -14
- package/lib/scaffold/ai-review-findings.ts +1369 -27
- package/lib/scaffold/ai-review-step.ts +4095 -119
- package/lib/scaffold/ai-review-unmeasured-workflow.ts +243 -0
- package/lib/scaffold/owned-files.ts +14 -0
- package/lib/scaffold/steps.ts +100 -9
- package/lib/scratch-roots.ts +93 -3
- package/lib/spec-updater.ts +22 -2
- package/lib/spec-write-ast.ts +728 -0
- package/lib/spec-write-sweep.ts +470 -0
- package/lib/test-suite-lock.ts +127 -13
- package/lib/ts-ast.ts +26 -0
- package/lib/validators/spec-validators.ts +36 -7
- package/lib/write-spec.ts +138 -0
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +41 -1
- package/scripts/arbitrate-finding.ts +330 -12
- package/scripts/audit-specs.ts +225 -24
- package/scripts/convert-spec.ts +7 -2
- package/scripts/create-sc.ts +14 -2
- package/scripts/create-spec.ts +5 -2
- package/scripts/detect-sc-drift.ts +10 -2
- package/scripts/record-ai-review-unmeasured.ts +370 -0
- package/scripts/split-spec.ts +17 -4
- package/scripts/sync-sc-status.ts +14 -2
|
@@ -0,0 +1,725 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The verifier quorum, in front of the findings that would otherwise BLOCK
|
|
3
|
+
* (#1476, CRC-B4).
|
|
4
|
+
*
|
|
5
|
+
* ## What was missing, and it was not the arithmetic
|
|
6
|
+
*
|
|
7
|
+
* `lib/finding-jury.ts` has held the group size, the quorum, the per-verifier
|
|
8
|
+
* ballot and the disagreement flag since #1187, and #1468 typed the ledger
|
|
9
|
+
* row's `jury` field so a row could hold a reading. Neither put a vote in
|
|
10
|
+
* front of anything. `arbitrate` had exactly one caller in the whole tree and
|
|
11
|
+
* it was `scripts/arbitrate-finding.ts`, hand-run. The spec's own entry said
|
|
12
|
+
* it: *nothing deployed takes a vote*.
|
|
13
|
+
*
|
|
14
|
+
* This is the deployed caller. It sits in the path
|
|
15
|
+
* `bunx rungate ai-review-findings` runs on every scaffolded project's
|
|
16
|
+
* pull request, beside the blocking decision.
|
|
17
|
+
*
|
|
18
|
+
* ## IT RECORDS. IT DOES NOT SUBTRACT. (read this before the rest)
|
|
19
|
+
*
|
|
20
|
+
* CRC-B4 has two halves — *"a separate group of agents votes on each
|
|
21
|
+
* finding"*, and *"it is surfaced only if enough agree it is real"*. This
|
|
22
|
+
* ships the first and deliberately not the second, and the reason is not
|
|
23
|
+
* caution, it is that the second is **unsafe until two other things are
|
|
24
|
+
* true**:
|
|
25
|
+
*
|
|
26
|
+
* - **Independence has to be structural, not textual.** The review action
|
|
27
|
+
* runs with `AI_REVIEW_ALLOWED_TOOLS` = `Read,Grep,Glob,Write` and
|
|
28
|
+
* `AI_REVIEW_DENY_RULES` denying `Task`. An instruction to spawn three
|
|
29
|
+
* verifier subagents therefore cannot execute, and what comes back is
|
|
30
|
+
* one model, in one context, emitting three votes. That ballot is
|
|
31
|
+
* populated, plausible, well-formed and worth nothing — it is the
|
|
32
|
+
* counterfeit the criterion names, reached by a prompt that could not
|
|
33
|
+
* run. So the group is `VERIFIER_GROUP_SIZE` SEPARATE action
|
|
34
|
+
* invocations, each writing its own file, merged here: independence is a
|
|
35
|
+
* property of the generated workflow rather than of a sentence.
|
|
36
|
+
* - **The verifier reads the head checkout.** A ballot that subtracted
|
|
37
|
+
* from the blocking set would make the branch under review an input to
|
|
38
|
+
* the gate on the branch under review: an author who can write code
|
|
39
|
+
* could write code that argues itself clean. Fixing that means reading
|
|
40
|
+
* the cited file at the base ref, which is not built.
|
|
41
|
+
*
|
|
42
|
+
* The half that is shipped is the half that is **permanently lost if it is
|
|
43
|
+
* not done now**: the criterion's own words are *"without them every
|
|
44
|
+
* threshold except the one actually used is permanently UNMEASURED, and
|
|
45
|
+
* re-running to recover the votes re-samples the thing being measured."* A
|
|
46
|
+
* verifier is a sampled instrument, so the second ballot is a second draw.
|
|
47
|
+
* Recording the first one is the prerequisite for CRC-B6's A/B, and it
|
|
48
|
+
* creates no bypass because nothing acts on it.
|
|
49
|
+
*
|
|
50
|
+
* ## Why here and not at the filing chokepoint
|
|
51
|
+
*
|
|
52
|
+
* `hooks/VerifyPhaseLock.hook.ts` → `filingGate` → `adjudicateFiling` is the
|
|
53
|
+
* other place a finding is adjudicated, and it is the wrong one. It is a
|
|
54
|
+
* PreToolUse hook: no verifier group, no network budget, no changed-file
|
|
55
|
+
* list, no branch. Convening a quorum there would INTRODUCE variance rather
|
|
56
|
+
* than reduce it — the deployed adjudicator is a regex over the title plus
|
|
57
|
+
* set arithmetic over symptom keys, with no clock, no model and no network,
|
|
58
|
+
* so twenty runs report one distinct outcome set and zero variance **by
|
|
59
|
+
* construction**. A variance control over it reads zero whether or not it
|
|
60
|
+
* works, and that reading is the signature of the failure mode rather than
|
|
61
|
+
* evidence of success.
|
|
62
|
+
*
|
|
63
|
+
* A false blocking finding on `ai-review`, by contrast, costs a remedy
|
|
64
|
+
* commit, a CI round, a review round and an agent's whole context. Every
|
|
65
|
+
* measured instance of the reviewer disagreeing with itself on a held-fixed
|
|
66
|
+
* tree has had the false positive on the blocking side — four of them, in
|
|
67
|
+
* `specs/CODE-REVIEW-CONVERGENCE-SPEC.md`. That is the cost this group is
|
|
68
|
+
* spent against.
|
|
69
|
+
*
|
|
70
|
+
* ## Four properties, and each is a case in `test/ai-review-jury-gate.test.ts`
|
|
71
|
+
*
|
|
72
|
+
* 1. **Only findings that would otherwise block are voted on.** The caller
|
|
73
|
+
* hands in `blockingReported` — the output of the one decider that already
|
|
74
|
+
* answers "does this block" — so this never re-derives it. A quorum over
|
|
75
|
+
* the advisory tail would spend a full agent context per nit.
|
|
76
|
+
* 2. **Every vote and the disagreement flag reach the ledger at decision
|
|
77
|
+
* time**, for the withheld findings as much as the surfaced ones. The
|
|
78
|
+
* withheld row is the one that cannot be reconstructed later: it is never
|
|
79
|
+
* commented on, never fixed and never filed, so it enters no downstream
|
|
80
|
+
* record at all. Re-running to recover it re-samples the thing being
|
|
81
|
+
* measured, because a verifier is a sampled instrument and the second
|
|
82
|
+
* ballot is a second draw.
|
|
83
|
+
* 3. **A group that could not be convened is UNMEASURED, never a reading.**
|
|
84
|
+
* "Did not reach a quorum" and "did not have one" are different facts
|
|
85
|
+
* with different fixes, and only the first is a decision. The finding
|
|
86
|
+
* keeps blocking either way — because nothing here withdraws one — so
|
|
87
|
+
* the distinction is about what the ledger records, which is the whole
|
|
88
|
+
* output.
|
|
89
|
+
* 4. **A ballot source that was named and could not be read is UNMEASURED
|
|
90
|
+
* and announced**, never an empty ballot set. An empty set would record
|
|
91
|
+
* a group that met and agreed with everything, which is a different and
|
|
92
|
+
* false fact. It does not fail the run: nothing the quorum says changes
|
|
93
|
+
* a verdict, so a quorum that could not be read must not either.
|
|
94
|
+
*
|
|
95
|
+
* ## What this does NOT do
|
|
96
|
+
*
|
|
97
|
+
* It does not convene the group. Spawning verifiers is the generated
|
|
98
|
+
* workflow's job and `lib/finding-jury.ts`'s header says why: a library
|
|
99
|
+
* that spawned them could not be tested without a model in the loop.
|
|
100
|
+
*
|
|
101
|
+
* Nor can it see whether the verifiers were independent. What it CAN see is
|
|
102
|
+
* cardinality — `VERIFIER_GROUP_SIZE` ballot files, each contributing one
|
|
103
|
+
* vote per finding under its own id — and that is why the merge below is
|
|
104
|
+
* over files rather than over one file's array. A single file claiming
|
|
105
|
+
* three votes is exactly the shape one context produces.
|
|
106
|
+
*/
|
|
107
|
+
|
|
108
|
+
import { readFileSync, readdirSync } from "fs";
|
|
109
|
+
import { join } from "path";
|
|
110
|
+
|
|
111
|
+
import { symptomHash, UNFILED_ID, type ArbiterRefusal, type Finding } from "./finding-arbiter";
|
|
112
|
+
import {
|
|
113
|
+
classifyCategory,
|
|
114
|
+
juryCategory,
|
|
115
|
+
juryReading,
|
|
116
|
+
parseVotes,
|
|
117
|
+
severityOf,
|
|
118
|
+
type JuryReading,
|
|
119
|
+
type VerifierVote,
|
|
120
|
+
} from "./finding-jury";
|
|
121
|
+
import { filingBatch, recordAdjudication, type FilingLedgerRow } from "./issue-filing-gate";
|
|
122
|
+
import {
|
|
123
|
+
AI_REVIEW_FINDING_FIELDS,
|
|
124
|
+
AI_REVIEW_JURY_NOTICE_TITLE,
|
|
125
|
+
advisoryKey,
|
|
126
|
+
} from "./scaffold/ai-review-findings";
|
|
127
|
+
import {
|
|
128
|
+
AI_REVIEW_BALLOT_BASENAME,
|
|
129
|
+
AI_REVIEW_BALLOT_KEY,
|
|
130
|
+
AI_REVIEW_BALLOT_VOTES_KEY,
|
|
131
|
+
} from "./scaffold/ai-review-step";
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* The key the ballot payload hangs its per-finding votes off.
|
|
135
|
+
*
|
|
136
|
+
* Declared in `./scaffold/ai-review-step`, beside the prompt that asks a
|
|
137
|
+
* verifier group to write the file, and re-exported here rather than
|
|
138
|
+
* restated. #295 is the measured instance of what a second spelling costs: a
|
|
139
|
+
* reader asked for a key no writer ever produced, the list was empty on
|
|
140
|
+
* every run, and an absent key read as an empty list read as clean.
|
|
141
|
+
*/
|
|
142
|
+
export { AI_REVIEW_BALLOT_KEY } from "./scaffold/ai-review-step";
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* The annotation title the jury's own reading is printed under.
|
|
146
|
+
*
|
|
147
|
+
* Declared in `./scaffold/ai-review-findings` beside the `console.log` that
|
|
148
|
+
* emits it and re-exported here, never re-declared. A title spelled in two
|
|
149
|
+
* places is two contracts, and nothing fails to compile when they diverge —
|
|
150
|
+
* #295, in the module next door.
|
|
151
|
+
*/
|
|
152
|
+
export const AI_REVIEW_JURY_TITLE = AI_REVIEW_JURY_NOTICE_TITLE;
|
|
153
|
+
|
|
154
|
+
/** What a finding nobody voted on is called. One string, one binding. */
|
|
155
|
+
export const NO_GROUP_CONVENED = "no verifier group was convened for this finding";
|
|
156
|
+
|
|
157
|
+
/** Opens every refusal of the ballot source itself, so a caller can match it. */
|
|
158
|
+
export const BALLOT_UNREADABLE_PREFIX = "the verifier ballot could not be read";
|
|
159
|
+
|
|
160
|
+
/** Per-finding ballots, as the payload declares them. Values are unvalidated. */
|
|
161
|
+
export type BallotSet = Record<string, unknown>;
|
|
162
|
+
|
|
163
|
+
export interface BallotReading {
|
|
164
|
+
/** `null` when the source could not be read. Never an empty set standing in. */
|
|
165
|
+
ballots: BallotSet | null;
|
|
166
|
+
/** The sentence naming what went wrong, or `null`. */
|
|
167
|
+
unmeasured: string | null;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Read the ballot payload, or refuse it.
|
|
172
|
+
*
|
|
173
|
+
* **An unreadable source is never an empty ballot set.** They differ in the
|
|
174
|
+
* direction that matters: an empty set withholds nothing, which is
|
|
175
|
+
* indistinguishable from a group that unanimously upheld every finding, and
|
|
176
|
+
* a run reporting that about a file it never parsed is the Silent Default
|
|
177
|
+
* Substitution `.claude/rules/config-access.md` forbids.
|
|
178
|
+
*
|
|
179
|
+
* The VOTES are not validated here. `parseVotes` owns that boundary and is
|
|
180
|
+
* the only thing in the tree that reads an untrusted vote — a second reader
|
|
181
|
+
* agreeing with it would be two deciders over one question
|
|
182
|
+
* (`.claude/rules/checks-must-be-able-to-fail.md`, the fourth shape), and a
|
|
183
|
+
* malformed vote refuses ONE finding's ballot rather than the whole file.
|
|
184
|
+
*
|
|
185
|
+
* What IS checked here is the envelope: an entry names a finding, carries a
|
|
186
|
+
* votes list, and names it only once. Those are facts about the ballot as a
|
|
187
|
+
* document rather than about any vote on it, and a document this reader
|
|
188
|
+
* cannot index is a file it did not read.
|
|
189
|
+
*/
|
|
190
|
+
export function readBallots(raw: string | null): BallotReading {
|
|
191
|
+
if (raw === null) {
|
|
192
|
+
return { ballots: null, unmeasured: `${BALLOT_UNREADABLE_PREFIX}: no ballot payload was produced` };
|
|
193
|
+
}
|
|
194
|
+
let doc: unknown;
|
|
195
|
+
try {
|
|
196
|
+
doc = JSON.parse(raw);
|
|
197
|
+
} catch (e) {
|
|
198
|
+
return { ballots: null, unmeasured: `${BALLOT_UNREADABLE_PREFIX}: it is not JSON (${(e as Error).message})` };
|
|
199
|
+
}
|
|
200
|
+
if (doc === null || typeof doc !== "object" || Array.isArray(doc)) {
|
|
201
|
+
return {
|
|
202
|
+
ballots: null,
|
|
203
|
+
unmeasured: `${BALLOT_UNREADABLE_PREFIX}: the payload is ${
|
|
204
|
+
Array.isArray(doc) ? "an array" : String(doc)
|
|
205
|
+
}, not an object`,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
const obj = doc as Record<string, unknown>;
|
|
209
|
+
if (!(AI_REVIEW_BALLOT_KEY in obj)) {
|
|
210
|
+
return {
|
|
211
|
+
ballots: null,
|
|
212
|
+
unmeasured:
|
|
213
|
+
`${BALLOT_UNREADABLE_PREFIX}: it carries no '${AI_REVIEW_BALLOT_KEY}' key ` +
|
|
214
|
+
`(it carries ${Object.keys(obj).join(", ") || "no keys at all"}). An absent key is not ` +
|
|
215
|
+
`an empty ballot set, and an empty ballot set is not a group that agreed with everything`,
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
const list = obj[AI_REVIEW_BALLOT_KEY];
|
|
219
|
+
if (!Array.isArray(list)) {
|
|
220
|
+
return {
|
|
221
|
+
ballots: null,
|
|
222
|
+
unmeasured:
|
|
223
|
+
`${BALLOT_UNREADABLE_PREFIX}: '${AI_REVIEW_BALLOT_KEY}' is ${
|
|
224
|
+
list === null ? "null" : typeof list
|
|
225
|
+
}, not a list of per-finding ballots`,
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const f = AI_REVIEW_FINDING_FIELDS;
|
|
230
|
+
const ballots: BallotSet = {};
|
|
231
|
+
for (let at = 0; at < list.length; at++) {
|
|
232
|
+
const entry = list[at];
|
|
233
|
+
if (entry === null || typeof entry !== "object" || Array.isArray(entry)) {
|
|
234
|
+
return {
|
|
235
|
+
ballots: null,
|
|
236
|
+
unmeasured: `${BALLOT_UNREADABLE_PREFIX}: ballot ${at} is ${
|
|
237
|
+
Array.isArray(entry) ? "an array" : String(entry)
|
|
238
|
+
}, not an object`,
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
const e = entry as Record<string, unknown>;
|
|
242
|
+
// The pair IS the identity, so a ballot that does not carry both names
|
|
243
|
+
// no finding. Refused rather than skipped: a skipped entry is a group
|
|
244
|
+
// that met and whose votes nobody counted, which reads from the outcome
|
|
245
|
+
// exactly like a group that was never convened.
|
|
246
|
+
for (const name of [f.file, f.summary] as const) {
|
|
247
|
+
if (typeof e[name] !== "string" || (e[name] as string).trim() === "") {
|
|
248
|
+
return {
|
|
249
|
+
ballots: null,
|
|
250
|
+
unmeasured:
|
|
251
|
+
`${BALLOT_UNREADABLE_PREFIX}: ballot ${at} carries no '${name}', so it names no ` +
|
|
252
|
+
`finding. The declared file and summary are how a ballot is matched to the ` +
|
|
253
|
+
`finding it is about`,
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
if (!Array.isArray(e[AI_REVIEW_BALLOT_VOTES_KEY])) {
|
|
258
|
+
return {
|
|
259
|
+
ballots: null,
|
|
260
|
+
unmeasured:
|
|
261
|
+
`${BALLOT_UNREADABLE_PREFIX}: ballot ${at} carries no '${AI_REVIEW_BALLOT_VOTES_KEY}' ` +
|
|
262
|
+
`list. A ballot entry with no votes on it is not an abstention, it is a malformed entry`,
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
// ONE identity function, computed here from the pair the writer copied
|
|
266
|
+
// rather than reproduced by the writer. See `AI_REVIEW_BALLOT_VOTES_KEY`.
|
|
267
|
+
const key = advisoryKey(e);
|
|
268
|
+
if (key in ballots) {
|
|
269
|
+
return {
|
|
270
|
+
ballots: null,
|
|
271
|
+
unmeasured:
|
|
272
|
+
`${BALLOT_UNREADABLE_PREFIX}: ballot ${at} is a second ballot for the finding at ` +
|
|
273
|
+
`${JSON.stringify(e[f.file])}, and nothing here can choose between two readings of ` +
|
|
274
|
+
`one question`,
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
ballots[key] = e[AI_REVIEW_BALLOT_VOTES_KEY];
|
|
278
|
+
}
|
|
279
|
+
return { ballots, unmeasured: null };
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export interface JuryGateVerdict {
|
|
283
|
+
/** The finding's identity, as `advisoryKey` computes it. */
|
|
284
|
+
key: string;
|
|
285
|
+
/** The finding's own one-sentence summary, so a ledger row is readable. */
|
|
286
|
+
title: string;
|
|
287
|
+
/** The reading, or `null` when no group was convened for this finding. */
|
|
288
|
+
reading: JuryReading | null;
|
|
289
|
+
/** `true` when the finding keeps blocking. UNMEASURED is `true`. */
|
|
290
|
+
surfaced: boolean;
|
|
291
|
+
/** Why, in words a reader can act on without re-running anything. */
|
|
292
|
+
reason: string;
|
|
293
|
+
/** The refusal recorded on the ledger row, or `null` when it survived. */
|
|
294
|
+
refusal: ArbiterRefusal | null;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export interface JuryGateResult {
|
|
298
|
+
/**
|
|
299
|
+
* One entry per blocking finding, **in input order** — the sentence saying
|
|
300
|
+
* it was voted down, or `null` for a finding that keeps blocking.
|
|
301
|
+
*
|
|
302
|
+
* **Positional, not keyed, and this was a defect before it was a design.**
|
|
303
|
+
* The first cut returned a map from `advisoryKey` to a reason, and the
|
|
304
|
+
* consumer re-derived that hash in its own second pass to look the verdict
|
|
305
|
+
* back up. Two findings that collide under it — the same `file` and
|
|
306
|
+
* `summary`, or a genuine FNV-1a collision between different pairs — then
|
|
307
|
+
* had one vote decide both: whichever the group actually voted on, the
|
|
308
|
+
* other was withheld or upheld on a verdict that was never about it.
|
|
309
|
+
*
|
|
310
|
+
* `lib/finding-arbiter.ts` had already answered this, in the module this
|
|
311
|
+
* one is built on, and answered it the other way: `SHARED_BALLOT` says
|
|
312
|
+
* *"a vote is about a finding. Spreading one across a heterogeneous batch
|
|
313
|
+
* does not give the other findings a quorum, it gives them somebody
|
|
314
|
+
* else's"*, and its remedy is *"one group per finding, in input order"*.
|
|
315
|
+
* A second module deciding the same question differently is the fourth
|
|
316
|
+
* shape in `.claude/rules/checks-must-be-able-to-fail.md`.
|
|
317
|
+
*
|
|
318
|
+
* So the identity is derived exactly once, HERE, where a collision inside
|
|
319
|
+
* the blocking set can be seen and refused — and the decision is carried
|
|
320
|
+
* out by position, where nothing can collide at all.
|
|
321
|
+
*/
|
|
322
|
+
withheld: (string | null)[];
|
|
323
|
+
verdicts: JuryGateVerdict[];
|
|
324
|
+
/** One sentence per finding voted on, for the annotation. */
|
|
325
|
+
detail: string[];
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** The `Finding` shape the arbiter's identity and category functions want. */
|
|
329
|
+
function asFinding(item: Record<string, unknown>): Finding {
|
|
330
|
+
const f = AI_REVIEW_FINDING_FIELDS;
|
|
331
|
+
const where = item[f.file] !== undefined ? `${String(item[f.file])}:${String(item[f.line] ?? "?")}` : "";
|
|
332
|
+
return {
|
|
333
|
+
id: UNFILED_ID,
|
|
334
|
+
title: String(item[f.summary] ?? ""),
|
|
335
|
+
body: [where, String(item[f.failureScenario] ?? "")].filter(Boolean).join(" — "),
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Count the group's votes on each finding that would otherwise block.
|
|
341
|
+
*
|
|
342
|
+
* `ballots` is `null` for "no ballot set was supplied", under which every
|
|
343
|
+
* finding reads as NO_GROUP_CONVENED and keeps blocking. That is the same
|
|
344
|
+
* answer an empty set gives and it is reached by a different route, which is
|
|
345
|
+
* why both are a per-finding UNMEASURED rather than a single verdict about
|
|
346
|
+
* the run: a ballot set can legitimately cover some findings and not others,
|
|
347
|
+
* and collapsing that to one reading would lose which.
|
|
348
|
+
*
|
|
349
|
+
* **No case passes a `groupSize` or a `quorum`.** The defaults are what the
|
|
350
|
+
* deployed path uses, so a mutation of `VERIFIER_GROUP_SIZE` reaches every
|
|
351
|
+
* case in the suite — which is the degenerate control
|
|
352
|
+
* `specs/CODE-REVIEW-CONVERGENCE-SPEC.md` asks for. A gate whose fixtures
|
|
353
|
+
* declare their own cardinality is green with a group of one.
|
|
354
|
+
*/
|
|
355
|
+
export function juryGate(
|
|
356
|
+
blocking: readonly unknown[],
|
|
357
|
+
ballots: BallotSet | null,
|
|
358
|
+
): JuryGateResult {
|
|
359
|
+
const withheld: (string | null)[] = [];
|
|
360
|
+
const verdicts: JuryGateVerdict[] = [];
|
|
361
|
+
const detail: string[] = [];
|
|
362
|
+
|
|
363
|
+
const asObject = (item: unknown): Record<string, unknown> =>
|
|
364
|
+
item !== null && typeof item === "object" && !Array.isArray(item)
|
|
365
|
+
? (item as Record<string, unknown>)
|
|
366
|
+
: {};
|
|
367
|
+
|
|
368
|
+
// The identity is derived ONCE, over the whole blocking set, so a key two
|
|
369
|
+
// findings share can be SEEN. A ballot filed under an ambiguous key names
|
|
370
|
+
// no single finding, and crediting it to whichever happened to come first
|
|
371
|
+
// is the shared-ballot defect `lib/finding-arbiter.ts` refuses by name.
|
|
372
|
+
// Both findings keep blocking, which is the direction this is allowed to
|
|
373
|
+
// be wrong in.
|
|
374
|
+
const seen = new Map<string, number>();
|
|
375
|
+
for (const item of blocking) {
|
|
376
|
+
const key = advisoryKey(asObject(item));
|
|
377
|
+
seen.set(key, (seen.get(key) ?? 0) + 1);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const record = (v: JuryGateVerdict): void => {
|
|
381
|
+
verdicts.push(v);
|
|
382
|
+
detail.push(v.reason);
|
|
383
|
+
withheld.push(v.surfaced ? null : v.reason);
|
|
384
|
+
};
|
|
385
|
+
|
|
386
|
+
for (const item of blocking) {
|
|
387
|
+
const finding = asObject(item);
|
|
388
|
+
const key = advisoryKey(finding);
|
|
389
|
+
const title = String(finding[AI_REVIEW_FINDING_FIELDS.summary] ?? "");
|
|
390
|
+
|
|
391
|
+
if ((seen.get(key) ?? 0) > 1) {
|
|
392
|
+
record({
|
|
393
|
+
key,
|
|
394
|
+
title,
|
|
395
|
+
reading: null,
|
|
396
|
+
surfaced: true,
|
|
397
|
+
reason:
|
|
398
|
+
`${seen.get(key)} of the blocking findings share the identity [key ${key}], so a ` +
|
|
399
|
+
`ballot filed under it names no one of them. All of them keep blocking: a vote is ` +
|
|
400
|
+
`about a finding, and spreading one across several gives the others somebody ` +
|
|
401
|
+
`else's quorum rather than their own`,
|
|
402
|
+
refusal: "UNMEASURED",
|
|
403
|
+
});
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
const cast = ballots === null ? undefined : ballots[key];
|
|
408
|
+
|
|
409
|
+
if (cast === undefined) {
|
|
410
|
+
record({
|
|
411
|
+
key,
|
|
412
|
+
title,
|
|
413
|
+
reading: null,
|
|
414
|
+
surfaced: true,
|
|
415
|
+
reason:
|
|
416
|
+
`${NO_GROUP_CONVENED} [key ${key}]. It keeps blocking: a group that did not meet ` +
|
|
417
|
+
`is UNMEASURED, which is neither a quorum of zero nor a pass`,
|
|
418
|
+
refusal: "UNMEASURED",
|
|
419
|
+
});
|
|
420
|
+
continue;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
let votes: readonly VerifierVote[];
|
|
424
|
+
try {
|
|
425
|
+
votes = parseVotes(cast);
|
|
426
|
+
} catch (e) {
|
|
427
|
+
record({
|
|
428
|
+
key,
|
|
429
|
+
title,
|
|
430
|
+
reading: null,
|
|
431
|
+
surfaced: true,
|
|
432
|
+
reason:
|
|
433
|
+
`the verifier ballot [key ${key}] could not be read, so no vote was counted on it: ` +
|
|
434
|
+
`${(e as Error).message}. It keeps blocking`,
|
|
435
|
+
refusal: "UNMEASURED",
|
|
436
|
+
});
|
|
437
|
+
continue;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
const reading = juryReading(votes);
|
|
441
|
+
if (reading.reading === "UNMEASURED") {
|
|
442
|
+
record({
|
|
443
|
+
key,
|
|
444
|
+
title,
|
|
445
|
+
reading,
|
|
446
|
+
surfaced: true,
|
|
447
|
+
reason:
|
|
448
|
+
`the verifier group [key ${key}] did not produce a countable ballot: ` +
|
|
449
|
+
`${reading.unmeasuredReasons.join("; ")}. It keeps blocking`,
|
|
450
|
+
refusal: "UNMEASURED",
|
|
451
|
+
});
|
|
452
|
+
continue;
|
|
453
|
+
}
|
|
454
|
+
if (!reading.passed) {
|
|
455
|
+
record({
|
|
456
|
+
key,
|
|
457
|
+
title,
|
|
458
|
+
reading,
|
|
459
|
+
surfaced: false,
|
|
460
|
+
reason:
|
|
461
|
+
`${reading.agree} of ${reading.votes} verifiers called it real and the quorum is ` +
|
|
462
|
+
`${reading.quorum}, so it is withheld [key ${key}]. It is still reported below and ` +
|
|
463
|
+
`blocks nothing`,
|
|
464
|
+
refusal: "NO_QUORUM",
|
|
465
|
+
});
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
record({
|
|
469
|
+
key,
|
|
470
|
+
title,
|
|
471
|
+
reading,
|
|
472
|
+
surfaced: true,
|
|
473
|
+
reason:
|
|
474
|
+
`${reading.agree} of ${reading.votes} verifiers called it real, quorum ` +
|
|
475
|
+
`${reading.quorum}${reading.disagreed ? " (the group disagreed)" : ""}, so it blocks ` +
|
|
476
|
+
`[key ${key}]`,
|
|
477
|
+
refusal: null,
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
return { withheld, verdicts, detail };
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* One adjudication, as a ledger row.
|
|
486
|
+
*
|
|
487
|
+
* **One site, and it is the whole of the recording**, so a mutant that
|
|
488
|
+
* neutralises `jury` here neutralises the thing under test and nothing else —
|
|
489
|
+
* the same property `scripts/arbitrate-finding.ts`'s `ledgerRow` holds and
|
|
490
|
+
* for the same reason.
|
|
491
|
+
*
|
|
492
|
+
* `FilingLedgerRow` and not a new shape. The value of the row populations —
|
|
493
|
+
* the chokepoint's, the hand-run arbiter's and this one's — is the corpus
|
|
494
|
+
* they form together, and a second vocabulary for `category` / `refusal` /
|
|
495
|
+
* `withheldBy` would make that union a join nobody can write.
|
|
496
|
+
*
|
|
497
|
+
* Written for a withheld finding as well as a surfaced one, and that is the
|
|
498
|
+
* criterion rather than a detail. The cheapest wrong version records what it
|
|
499
|
+
* surfaces, which satisfies every row a reader goes looking for and leaves
|
|
500
|
+
* the withheld population exactly as unrecoverable as it was.
|
|
501
|
+
*/
|
|
502
|
+
export function juryLedgerRow(
|
|
503
|
+
item: unknown,
|
|
504
|
+
verdict: JuryGateVerdict,
|
|
505
|
+
at: string,
|
|
506
|
+
index: number,
|
|
507
|
+
): FilingLedgerRow {
|
|
508
|
+
const finding = asFinding(
|
|
509
|
+
item !== null && typeof item === "object" && !Array.isArray(item)
|
|
510
|
+
? (item as Record<string, unknown>)
|
|
511
|
+
: {},
|
|
512
|
+
);
|
|
513
|
+
const votes = verdict.reading?.ballot ?? [];
|
|
514
|
+
const batch = filingBatch(at);
|
|
515
|
+
const emitted = verdict.surfaced;
|
|
516
|
+
return {
|
|
517
|
+
at,
|
|
518
|
+
key: batch === null ? null : `${batch}/${index}`,
|
|
519
|
+
id: UNFILED_ID,
|
|
520
|
+
title: verdict.title,
|
|
521
|
+
symptomHash: symptomHash(finding),
|
|
522
|
+
category: votes.length > 0 ? juryCategory(finding, votes) : classifyCategory(finding),
|
|
523
|
+
// The payload's severity vocabulary — critical / high / medium / low /
|
|
524
|
+
// info — is a different axis from `Severity`, so it is NOT mapped onto
|
|
525
|
+
// one. Only a verifier's own tag fills this, and `null` is the honest
|
|
526
|
+
// answer when none carried one.
|
|
527
|
+
severity: severityOf(finding, votes),
|
|
528
|
+
disposition: emitted ? "COMMENT" : "DROP",
|
|
529
|
+
refusal: verdict.refusal,
|
|
530
|
+
emitted,
|
|
531
|
+
withheldBy: emitted ? null : (verdict.refusal ?? "UNACCOUNTED"),
|
|
532
|
+
jury: verdict.reading,
|
|
533
|
+
duplicateOf: [],
|
|
534
|
+
// No override exists on this path. `arbiter-override` is a
|
|
535
|
+
// `gh issue create` label the chokepoint reads, and nothing here has one.
|
|
536
|
+
overridden: false,
|
|
537
|
+
// A review round adjudicates candidates, not `gh issue create` attempts,
|
|
538
|
+
// so there is no earlier filing of this title for one to be a re-draft
|
|
539
|
+
// of. `null` is the reading here, not an unperformed lookup (#1536).
|
|
540
|
+
refiledAfter: null,
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/** What the adjudicator's `main` consumes. Plain data, so it can be injected. */
|
|
545
|
+
export interface JuryGateOutcome {
|
|
546
|
+
/**
|
|
547
|
+
* One entry per blocking finding, in input order — the sentence saying
|
|
548
|
+
* the group WOULD have withheld it, or `null`.
|
|
549
|
+
*
|
|
550
|
+
* "Would have". Nothing acts on this; see the module header.
|
|
551
|
+
*/
|
|
552
|
+
wouldWithhold: (string | null)[];
|
|
553
|
+
/** Non-null when no countable ballot could be read. Never fatal. */
|
|
554
|
+
unmeasured: string | null;
|
|
555
|
+
detail: string[];
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Collect the ballots one verifier directory per verifier, merged by finding.
|
|
560
|
+
*
|
|
561
|
+
* **The group is N files, and that is the only independence claim anything
|
|
562
|
+
* downstream can check.** One file carrying `VERIFIER_GROUP_SIZE` votes is
|
|
563
|
+
* the shape a single model in a single context produces, and once written
|
|
564
|
+
* it is indistinguishable from a real group — so the contract is one file
|
|
565
|
+
* per verifier, and the merge happens here where the count can be seen.
|
|
566
|
+
*
|
|
567
|
+
* A directory with fewer than `VERIFIER_GROUP_SIZE` ballot files produces
|
|
568
|
+
* short ballots, which `juryReading` refuses as `INCOMPLETE_GROUP`. That is
|
|
569
|
+
* the correct reading and not a workaround: a verifier that did not report
|
|
570
|
+
* is neither agreement nor dissent.
|
|
571
|
+
*
|
|
572
|
+
* Entries are merged in directory-name order, which is stable, so a given
|
|
573
|
+
* finding's votes arrive in the same sequence on every run.
|
|
574
|
+
*/
|
|
575
|
+
export function readBallotDirectory(ballotDir: string): BallotReading {
|
|
576
|
+
let entries: string[];
|
|
577
|
+
try {
|
|
578
|
+
entries = readdirSync(ballotDir).sort();
|
|
579
|
+
} catch (e) {
|
|
580
|
+
return {
|
|
581
|
+
ballots: null,
|
|
582
|
+
unmeasured:
|
|
583
|
+
`${BALLOT_UNREADABLE_PREFIX}: the verifier directory ${ballotDir} could not be listed ` +
|
|
584
|
+
`(${(e as Error).message}). A directory this run was told to read and could not is a ` +
|
|
585
|
+
`group that did not meet, not a group that upheld every finding`,
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
const merged: BallotSet = {};
|
|
590
|
+
let files = 0;
|
|
591
|
+
for (const entry of entries) {
|
|
592
|
+
const path = join(ballotDir, entry, AI_REVIEW_BALLOT_BASENAME);
|
|
593
|
+
let raw: string;
|
|
594
|
+
try {
|
|
595
|
+
raw = readFileSync(path, "utf-8");
|
|
596
|
+
} catch {
|
|
597
|
+
// A directory with no ballot in it is a verifier that did not report.
|
|
598
|
+
// That is not an error here — it shortens the ballot, and
|
|
599
|
+
// `INCOMPLETE_GROUP` is the reading that names it one level down.
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
files += 1;
|
|
603
|
+
const one = readBallots(raw);
|
|
604
|
+
if (one.ballots === null) {
|
|
605
|
+
return {
|
|
606
|
+
ballots: null,
|
|
607
|
+
unmeasured: `${one.unmeasured} (verifier ${JSON.stringify(entry)})`,
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
for (const [key, votes] of Object.entries(one.ballots)) {
|
|
611
|
+
const cast = votes as unknown[];
|
|
612
|
+
// ONE VOTE PER FILE, and a file carrying more is REFUSED rather
|
|
613
|
+
// than truncated.
|
|
614
|
+
//
|
|
615
|
+
// This is the counterfeit CRC-B4 names, arriving through the file
|
|
616
|
+
// format instead of through a constant `agree`. One file listing
|
|
617
|
+
// `VERIFIER_GROUP_SIZE` votes under distinct names reads as a full
|
|
618
|
+
// group one level down: the names differ so `REPEATED_VERIFIER`
|
|
619
|
+
// cannot fire, the count is right so `INCOMPLETE_GROUP` cannot,
|
|
620
|
+
// and nothing below this line knows how many files the votes came
|
|
621
|
+
// out of. The file count is the ONLY independence signal this
|
|
622
|
+
// layer has, so it is spent here or not at all.
|
|
623
|
+
//
|
|
624
|
+
// Refused and not truncated, because the two readings differ:
|
|
625
|
+
// keeping the first vote would report a group that met and fell
|
|
626
|
+
// short, which is a reading, and this is a ballot nobody can
|
|
627
|
+
// count. Found by this repository's own `ai-review` against the
|
|
628
|
+
// first cut of this function, whose docblock already said a
|
|
629
|
+
// self-written group "is indistinguishable from a real group"
|
|
630
|
+
// while the code merged the array whole.
|
|
631
|
+
//
|
|
632
|
+
// **Here and not in `readBallots`**, which is the other candidate
|
|
633
|
+
// and the tempting one — it is the single-file reader, so the
|
|
634
|
+
// invariant would sit with the thing it describes. Two reasons it
|
|
635
|
+
// does not. The refusal has to NAME the verifier that filed the
|
|
636
|
+
// stuffed file, and the directory name is knowledge this loop has
|
|
637
|
+
// and `readBallots` does not; putting the cap down there means the
|
|
638
|
+
// caller rewrites the message to add the name, which is two places
|
|
639
|
+
// shaping one refusal. And the violated property is provenance,
|
|
640
|
+
// not syntax: `VERIFIER_GROUP_SIZE` votes for one finding is a
|
|
641
|
+
// perfectly well-formed ballot, and what makes it a counterfeit is
|
|
642
|
+
// that they arrived in one file. Only a reader counting files can
|
|
643
|
+
// see that. `readBallotDirectory` is the only production caller of
|
|
644
|
+
// `readBallots`, so there is exactly one decider either way —
|
|
645
|
+
// this is about which one can state the fact.
|
|
646
|
+
if (cast.length !== 1) {
|
|
647
|
+
return {
|
|
648
|
+
ballots: null,
|
|
649
|
+
unmeasured:
|
|
650
|
+
`${BALLOT_UNREADABLE_PREFIX}: verifier ${JSON.stringify(entry)} filed ${cast.length} ` +
|
|
651
|
+
`votes for one finding in a single ${AI_REVIEW_BALLOT_BASENAME}. The contract is one ` +
|
|
652
|
+
`vote per file because the file count is the only evidence the votes came from ` +
|
|
653
|
+
`separate contexts: a group one writer can produce alone is the quorum wired to a ` +
|
|
654
|
+
`constant that CRC-B4 refuses, wearing ${cast.length} names`,
|
|
655
|
+
};
|
|
656
|
+
}
|
|
657
|
+
const carried = merged[key];
|
|
658
|
+
merged[key] = [...(Array.isArray(carried) ? carried : []), ...cast];
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
if (files === 0) {
|
|
663
|
+
return {
|
|
664
|
+
ballots: null,
|
|
665
|
+
unmeasured:
|
|
666
|
+
`${BALLOT_UNREADABLE_PREFIX}: no verifier under ${ballotDir} wrote a ` +
|
|
667
|
+
`${AI_REVIEW_BALLOT_BASENAME}, so no group met. An empty ballot set is not a group ` +
|
|
668
|
+
`that agreed with every finding`,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
return { ballots: merged, unmeasured: null };
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/**
|
|
675
|
+
* The deployed gate: read the ballots, count them, record them, report them.
|
|
676
|
+
*
|
|
677
|
+
* Injected into `main` rather than imported by it. `lib/scaffold/
|
|
678
|
+
* ai-review-findings.ts` is copied into a scratch directory by three mutation
|
|
679
|
+
* batteries, which carry its one relative dependency with it; an import of
|
|
680
|
+
* this module would drag `lib/issue-filing-gate.ts` and its whole tree behind
|
|
681
|
+
* it and the mutants would die on module resolution — a failure that reads
|
|
682
|
+
* identically to a guard behaving differently
|
|
683
|
+
* (`.claude/rules/self-referential-tests.md`, Tier 1, last row).
|
|
684
|
+
*
|
|
685
|
+
* The seam is not a loophole, because there is nothing on the other side of
|
|
686
|
+
* it to bypass: the gate changes no verdict. An entry point that forgets to
|
|
687
|
+
* wire it records no votes and says so under
|
|
688
|
+
* `AI_REVIEW_JURY_UNMEASURED_TITLE`.
|
|
689
|
+
*
|
|
690
|
+
* **Including when this function throws.** That is not a property of this
|
|
691
|
+
* file — nothing here is exception-safe, and `juryReading`, `advisoryKey`
|
|
692
|
+
* and the whole of `juryLedgerRow` can all raise — it is a property of the
|
|
693
|
+
* `try` around the call in `lib/scaffold/ai-review-findings.ts`, asserted
|
|
694
|
+
* by `JG-6h`. Said here because this is where a reader will look for it,
|
|
695
|
+
* and because the first cut of that sentence was true of three states out
|
|
696
|
+
* of four.
|
|
697
|
+
*
|
|
698
|
+
* **A ledger that cannot be written does not change the verdict**, which is
|
|
699
|
+
* trivially true here and is said anyway, because it will stop being
|
|
700
|
+
* trivial the day the ballot is allowed to subtract. The record exists to
|
|
701
|
+
* measure the gate, and a measurement apparatus that blocks the thing it
|
|
702
|
+
* measures is worse than no measurement — `recordAdjudication`'s own policy.
|
|
703
|
+
*/
|
|
704
|
+
export function aiReviewJuryGate(blocking: readonly unknown[], ballotDir: string): JuryGateOutcome {
|
|
705
|
+
const source = readBallotDirectory(ballotDir);
|
|
706
|
+
if (source.ballots === null) {
|
|
707
|
+
return { wouldWithhold: [], unmeasured: source.unmeasured, detail: [] };
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
const result = juryGate(blocking, source.ballots);
|
|
711
|
+
const at = new Date().toISOString();
|
|
712
|
+
const detail = [...result.detail];
|
|
713
|
+
let unwritten = 0;
|
|
714
|
+
result.verdicts.forEach((verdict, index) => {
|
|
715
|
+
if (!recordAdjudication(juryLedgerRow(blocking[index], verdict, at, index))) unwritten += 1;
|
|
716
|
+
});
|
|
717
|
+
if (unwritten > 0) {
|
|
718
|
+
detail.push(
|
|
719
|
+
`${unwritten} of ${result.verdicts.length} verifier ballots could not be appended to the ` +
|
|
720
|
+
`finding ledger, so those votes are UNMEASURED for every later threshold. Nothing else ` +
|
|
721
|
+
`changes: a record that cannot be written must not change what it records`,
|
|
722
|
+
);
|
|
723
|
+
}
|
|
724
|
+
return { wouldWithhold: result.withheld, unmeasured: null, detail };
|
|
725
|
+
}
|