rungate 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -0
- package/gates/SCHEMA-GUIDE.md +485 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +283 -0
- package/gates/brief-assembler.test.ts +487 -0
- package/gates/brief-assembler.ts +448 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +250 -0
- package/gates/error-classifier.ts +99 -0
- package/gates/gate-executor.ts +3071 -0
- package/gates/orchestrator.test.ts +968 -0
- package/gates/orchestrator.ts +827 -0
- package/gates/preload.ts +89 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prompts.sha256 +3 -0
- package/gates/prove.test.ts +164 -0
- package/gates/rook-state.test.ts +108 -0
- package/gates/run-attribution.ts +290 -0
- package/gates/run-gate.ts +163 -0
- package/gates/schema-parity.test.ts +378 -0
- package/gates/schema.ts +1209 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +125 -0
- package/gates/ship-orchestrator.test.ts +399 -0
- package/gates/ship-orchestrator.ts +384 -0
- package/gates/spec-compliance-probe.ts +159 -0
- package/gates/test-fixtures/broken-no-environments.json +110 -0
- package/gates/test-fixtures/consumer-directory-layout.json +90 -0
- package/gates/test-fixtures/valid-standard.json +91 -0
- package/gates/test-utils.ts +43 -0
- package/gates/typecheck-gate.ts +111 -0
- package/gates/witness.ts +349 -0
- package/gates/workflow.test.ts +2018 -0
- package/hooks/BashWriteGuard.hook.ts +19 -0
- package/hooks/GateResultProtection.hook.ts +27 -1
- package/hooks/VerifyPhaseLock.hook.ts +51 -11
- package/hooks/WorkflowStateGuard.hook.ts +24 -0
- package/lib/ai-review-rounds.ts +462 -0
- package/lib/bash-file-read.ts +17 -3
- package/lib/bash-write-guard.ts +1285 -50
- package/lib/distribution-standing.ts +75 -4
- package/lib/failure-intake.ts +12 -4
- package/lib/file-extensions.ts +81 -0
- package/lib/finding-arbiter.ts +1015 -0
- package/lib/finding-jury.ts +537 -0
- package/lib/gate-enforcement.ts +171 -43
- package/lib/gate-result-protection.ts +12 -2
- package/lib/generators/code-map.ts +55 -4
- package/lib/generators/types.ts +27 -1
- package/lib/github.ts +55 -0
- package/lib/interpreters.ts +64 -0
- package/lib/issue-create-override.ts +69 -8
- package/lib/issue-filing-gate.ts +605 -0
- package/lib/js-top-level-code.ts +20 -2
- package/lib/junit-checks.ts +239 -1
- package/lib/package-closure.ts +114 -0
- package/lib/publish-window.ts +287 -0
- package/lib/scaffold/ai-review-findings.ts +142 -3
- package/lib/scaffold/ai-review-step.ts +1012 -18
- package/lib/scaffold/managed-workflow.ts +163 -4
- package/lib/scaffold/steps.ts +498 -22
- package/lib/scanner.ts +13 -4
- package/lib/ticked-sc-symbols.ts +361 -0
- package/lib/transcript-checker.ts +6 -0
- package/lib/workflow-actions.ts +18 -0
- package/package.json +3 -1
- package/scripts/arbitrate-finding.ts +224 -0
- package/scripts/generate-code-map.ts +18 -10
- package/scripts/sync-sc-status.ts +82 -8
|
@@ -0,0 +1,1015 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether a finding is worth filing as an issue — stage 1, mechanical (#1187).
|
|
3
|
+
*
|
|
4
|
+
* ## What this answers, and what it deliberately does not
|
|
5
|
+
*
|
|
6
|
+
* Measured at 15:24 on 2026-10-22: **14 issues filed per hour against 6
|
|
7
|
+
* closed**, 27 open. Nothing between an agent producing a finding and the
|
|
8
|
+
* finding becoming an issue asks whether it is worth filing. #1187 names four
|
|
9
|
+
* filing failures from one session and records that a correctness vote —
|
|
10
|
+
* "several agents hunt, a separate group votes" — catches **none** of them.
|
|
11
|
+
* Every one of the four was a real observation about the tree its reader was
|
|
12
|
+
* looking at. The questions that would have stopped them are *is this new*,
|
|
13
|
+
* *is this here* and *is this current*, and all three are decidable by one
|
|
14
|
+
* process with no model in it.
|
|
15
|
+
*
|
|
16
|
+
* So the checks run in cost order, each on what survived the one before:
|
|
17
|
+
*
|
|
18
|
+
* | stage | question | instrument | on failure |
|
|
19
|
+
* |---|---|---|---|
|
|
20
|
+
* | 1 | is this new? | string sets over the finding bodies | `DUPLICATE` |
|
|
21
|
+
* | 2 | do enough verifiers call it real? | `lib/finding-jury.ts` counts votes the caller cast | `NO_QUORUM` |
|
|
22
|
+
* | 3 | is it the kind we post at all? | the category, verifier-supplied or keyed | `DROPPED_CATEGORY` |
|
|
23
|
+
* | 4 | which tree is it about? | `git grep` against `origin/main` | sets `scope`, never refuses |
|
|
24
|
+
* | 5 | is this current? | the file's line count | `STALE_CITATION` |
|
|
25
|
+
*
|
|
26
|
+
* Stage 1 first and the vote second is the one ordering choice worth
|
|
27
|
+
* defending: a vote is the most expensive instrument the harness has, and
|
|
28
|
+
* spending three agent contexts to confirm a finding that a string-set
|
|
29
|
+
* comparison already knows is a duplicate is the failure #1187 measured.
|
|
30
|
+
*
|
|
31
|
+
* ## THE DEFAULT IS A COMMENT, NOT AN ISSUE
|
|
32
|
+
*
|
|
33
|
+
* #1187 asked which findings are worth **filing**, and that framing is wrong.
|
|
34
|
+
* The design it was drawn from files nothing: a surviving finding becomes an
|
|
35
|
+
* inline comment that **blocks the merge until it is answered** — fixed, or
|
|
36
|
+
* the thread closed with a reason. There is no backlog because findings never
|
|
37
|
+
* become backlog. We added the filing step, and it is why the tracker
|
|
38
|
+
* regenerates: measured 2026-10-22, 23 of 27 open issues were created the
|
|
39
|
+
* same day and 10 named one subsystem.
|
|
40
|
+
*
|
|
41
|
+
* So `Disposition` defaults to `COMMENT`, `ISSUE` requires an explicit
|
|
42
|
+
* declared reason from the caller (`Finding.outOfScope`), and what used to be
|
|
43
|
+
* the `OFF_MAIN` **refusal** is now the `scope` **reading** — a finding about
|
|
44
|
+
* an unmerged branch was never a bad finding, it was a comment on that branch
|
|
45
|
+
* all along. See `FindingScope`.
|
|
46
|
+
*
|
|
47
|
+
* ## Why the key is the measured symptom and not the title
|
|
48
|
+
*
|
|
49
|
+
* The two duplicate pairs this was built against both merged today:
|
|
50
|
+
*
|
|
51
|
+
* - **#1175 / #1177** — two agents, the same 5,060-command corpus sweep, the
|
|
52
|
+
* same 144-of-234 reading, filed five minutes apart under **completely
|
|
53
|
+
* different titles**. A title-keyed deduper misses this, and
|
|
54
|
+
* `test/finding-arbiter.test.ts` runs a title-similarity deduper over the
|
|
55
|
+
* pair to show that it does.
|
|
56
|
+
* - **#1195 / #1196** — an agent and a human filed the same two regressions
|
|
57
|
+
* from the same merge, one minute apart. Different titles again; the same
|
|
58
|
+
* `commandWord` / `isDirDestination` / `statSync` symptom underneath.
|
|
59
|
+
*
|
|
60
|
+
* What survives a rewrite of the prose is what the finding **measured**: the
|
|
61
|
+
* symbols it names, the files it names, and the integers it reports. So a
|
|
62
|
+
* finding emits a set of keys of those three kinds and two findings are
|
|
63
|
+
* duplicates when they share at least `DUPLICATE_MIN_SHARED_KEYS` of them.
|
|
64
|
+
*
|
|
65
|
+
* A set-overlap test and not a hash of the set, because the sets are never
|
|
66
|
+
* equal — #1177 carries a measurement table #1175 renders as prose. An exact
|
|
67
|
+
* key over the symptom catches neither pair, which is the same failure as the
|
|
68
|
+
* title key one level in.
|
|
69
|
+
*
|
|
70
|
+
* ## WHAT THIS DOES NOT COVER
|
|
71
|
+
*
|
|
72
|
+
* - **Two findings about one symbol that are genuinely different defects.**
|
|
73
|
+
* A hot file collects findings, and shared symbols are what a hot file
|
|
74
|
+
* produces. The threshold is tuned on a 40-issue corpus in
|
|
75
|
+
* `test/finding-arbiter.test.ts` and the false positives it produces there
|
|
76
|
+
* are listed in that file by number rather than tuned away. **The direction
|
|
77
|
+
* of the error matters:** a false DUPLICATE withholds a real finding, so
|
|
78
|
+
* `arbitrate` returns the colliding issue numbers and the shared keys —
|
|
79
|
+
* the refusal is reviewable, never silent.
|
|
80
|
+
* - **A finding whose symbol exists on `origin/main` under a different
|
|
81
|
+
* name.** A rename between the reviewed branch and `main` reads here as
|
|
82
|
+
* `OFF_MAIN`.
|
|
83
|
+
* - **Prose-only findings.** A finding that names no symbol and no file is
|
|
84
|
+
* `UNMEASURED`, which FAILS — it does not skip. That is #1187's own
|
|
85
|
+
* doctrine and the one line in here that must not be softened: a finding
|
|
86
|
+
* nobody can evaluate is not a finding that passes.
|
|
87
|
+
*/
|
|
88
|
+
|
|
89
|
+
import { existsSync, readFileSync } from "fs";
|
|
90
|
+
import { spawnSync } from "child_process";
|
|
91
|
+
import { resolve, sep } from "path";
|
|
92
|
+
|
|
93
|
+
import { contentHash } from "./content-hash";
|
|
94
|
+
import {
|
|
95
|
+
promoteCandidates,
|
|
96
|
+
type PromotableCandidate,
|
|
97
|
+
type PromotionReading,
|
|
98
|
+
type WithheldReason,
|
|
99
|
+
} from "./rule-promotion";
|
|
100
|
+
import {
|
|
101
|
+
classifyCategory,
|
|
102
|
+
DROPPED_CATEGORIES,
|
|
103
|
+
juryCategory,
|
|
104
|
+
juryReading,
|
|
105
|
+
severityOf,
|
|
106
|
+
SEVERITY_RANK,
|
|
107
|
+
type Disposition,
|
|
108
|
+
type FindingCategory,
|
|
109
|
+
type JuryConfig,
|
|
110
|
+
type JuryReading,
|
|
111
|
+
type Severity,
|
|
112
|
+
type VerifierVote,
|
|
113
|
+
} from "./finding-jury";
|
|
114
|
+
|
|
115
|
+
/** A finding, as little of one as the checks need. */
|
|
116
|
+
export interface Finding {
|
|
117
|
+
/** How the caller names it — an issue number, or `new` for an unfiled one. */
|
|
118
|
+
id: string;
|
|
119
|
+
title: string;
|
|
120
|
+
body: string;
|
|
121
|
+
/**
|
|
122
|
+
* The ONE route to `ISSUE`, and it must be declared, never inferred.
|
|
123
|
+
*
|
|
124
|
+
* Filing is the exception: the work genuinely cannot ride along on this PR
|
|
125
|
+
* — a different subsystem, or a design decision somebody else owns. An
|
|
126
|
+
* *inferred* exception is exactly how a filing default comes back, so there
|
|
127
|
+
* is no heuristic here and there is no default: a caller that wants an
|
|
128
|
+
* issue says so and says why, in a sentence that ends up in the issue body.
|
|
129
|
+
*/
|
|
130
|
+
outOfScope?: { reason: string };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* The id an unfiled finding carries, and the one id that does not identify.
|
|
135
|
+
*
|
|
136
|
+
* Exported so the CLI and the dedupe share one binding. Two findings both
|
|
137
|
+
* labelled `"new"` are two different findings, not one — and treating the
|
|
138
|
+
* label as an identity is what made the in-batch dedupe a no-op in exactly
|
|
139
|
+
* the configuration it was written for.
|
|
140
|
+
*/
|
|
141
|
+
export const UNFILED_ID = "new";
|
|
142
|
+
|
|
143
|
+
export type SymptomKeyKind = "sym" | "file" | "qty";
|
|
144
|
+
|
|
145
|
+
/** How many cited symbols the branch-scope reading will pay a `git grep` for. */
|
|
146
|
+
const MAX_SYMBOLS_PROBED = 8;
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* How many `path:line` citations the currency reading will pay a file read for.
|
|
150
|
+
*
|
|
151
|
+
* The asymmetry this fixes ran the wrong way: `MAX_SYMBOLS_PROBED` bounded the
|
|
152
|
+
* *cheap* probe while the expensive one — a `readFileSync`, or a
|
|
153
|
+
* `git show origin/main:<path>` per citation — was unbounded. An AI reviewer's
|
|
154
|
+
* output is the one input on this path nobody audits, so a body naming four
|
|
155
|
+
* hundred `file.ts:N` pairs was a free way to wedge the arbiter.
|
|
156
|
+
*/
|
|
157
|
+
const MAX_LINES_PROBED = 8;
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Identifiers that are code-shaped and carry no information about which
|
|
161
|
+
* defect is being described. Excluded from the key rather than down-weighted,
|
|
162
|
+
* because a weight is a second threshold to justify.
|
|
163
|
+
*/
|
|
164
|
+
const UNINFORMATIVE_SYMBOLS = new Set([
|
|
165
|
+
"true", "false", "null", "undefined", "string", "number", "boolean",
|
|
166
|
+
"console", "process", "export", "import", "return", "const", "function",
|
|
167
|
+
"origin/main", "main", "HEAD",
|
|
168
|
+
]);
|
|
169
|
+
|
|
170
|
+
// ── Symptom keys ─────────────────────────────────────────────────────────
|
|
171
|
+
|
|
172
|
+
/** Spans between backticks, where a body puts the things it measured. */
|
|
173
|
+
function codeSpans(body: string): string[] {
|
|
174
|
+
const out: string[] = [];
|
|
175
|
+
for (const m of body.matchAll(/`{1,3}([^`]+)`{1,3}/g)) out.push(m[1]);
|
|
176
|
+
return out;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Identifiers worth keying on: `camelCase`, `UPPER_SNAKE`, `snake_case`.
|
|
181
|
+
*
|
|
182
|
+
* Two exclusions, both measured on the 40-issue corpus rather than guessed:
|
|
183
|
+
*
|
|
184
|
+
* - **A bare lowercase word is not a symbol.** That costs `dequote` and
|
|
185
|
+
* `mkdtemp` on the two pairs above and buys exclusion of every English word
|
|
186
|
+
* that ever appears inside backticks. Both pairs clear the threshold
|
|
187
|
+
* without them.
|
|
188
|
+
* - **An all-caps token must contain `_`.** Without this, the hyphenated
|
|
189
|
+
* filename `specs/HOOK-ARCHITECTURE-SPEC.md` yields the three "symbols"
|
|
190
|
+
* `HOOK`, `ARCHITECTURE` and `SPEC`, and `CODE-MAP.md` yields two more —
|
|
191
|
+
* so two findings citing one document agree on five keys while sharing no
|
|
192
|
+
* symptom. It also drops the status vocabulary (`CLEAN`, `BLOCK`, `PASS`,
|
|
193
|
+
* `FAIL`, `EOF`, `JSON`), which every finding in a review pipeline uses.
|
|
194
|
+
* The path itself is still keyed, once, as a `file:` key.
|
|
195
|
+
*
|
|
196
|
+
* The cost is real and is the direction to watch: a genuine unsnaked
|
|
197
|
+
* constant such as `OPERATORS` is no longer a key. It is paid because the
|
|
198
|
+
* measured error was 44 false pairs out of 49.
|
|
199
|
+
*/
|
|
200
|
+
function identifiersIn(span: string): string[] {
|
|
201
|
+
const out: string[] = [];
|
|
202
|
+
for (const m of span.matchAll(/\b[A-Za-z_$][A-Za-z0-9_$]{2,}\b/g)) {
|
|
203
|
+
const t = m[0];
|
|
204
|
+
if (UNINFORMATIVE_SYMBOLS.has(t)) continue;
|
|
205
|
+
const shaped = /[a-z][A-Z]/.test(t) || /_/.test(t);
|
|
206
|
+
if (shaped) out.push(t);
|
|
207
|
+
}
|
|
208
|
+
return out;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Repository-shaped paths, anywhere in the body. */
|
|
212
|
+
function pathsIn(body: string): string[] {
|
|
213
|
+
const out: string[] = [];
|
|
214
|
+
for (const m of body.matchAll(/(?:[\w.@-]+\/)+[\w.-]+\.(?:ts|tsx|js|mjs|cjs|md|json|ya?ml|sh)\b/g)) {
|
|
215
|
+
out.push(m[0].toLowerCase());
|
|
216
|
+
}
|
|
217
|
+
return out;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Integers a finding reports as a reading.
|
|
222
|
+
*
|
|
223
|
+
* The exclusions are what make this a measurement and not a number: an issue
|
|
224
|
+
* reference (`#1188`), an SC or AC id, a date component, and anything under
|
|
225
|
+
* ten. Without the first of those, every finding that cites the same prior
|
|
226
|
+
* issue collides with every other — which is a reading of the cross-reference
|
|
227
|
+
* graph, not of the defect.
|
|
228
|
+
*/
|
|
229
|
+
function quantitiesIn(body: string): number[] {
|
|
230
|
+
const out: number[] = [];
|
|
231
|
+
// Blank out the shapes whose digits are identifiers rather than readings,
|
|
232
|
+
// keeping the length so no two tokens are glued together by the removal.
|
|
233
|
+
const masked = body
|
|
234
|
+
.replace(/#\d+/g, m => " ".repeat(m.length))
|
|
235
|
+
.replace(/\b(?:SC|AC|PR|H|R)-?\d+/gi, m => " ".repeat(m.length))
|
|
236
|
+
.replace(/\d{4}-\d{2}-\d{2}(?:T[\d:.]+Z?)?/g, m => " ".repeat(m.length))
|
|
237
|
+
.replace(/:\d+/g, m => " ".repeat(m.length))
|
|
238
|
+
.replace(/\bv?\d+\.\d+(?:\.\d+)?/g, m => " ".repeat(m.length));
|
|
239
|
+
for (const m of masked.matchAll(/\b\d[\d,]*\b/g)) {
|
|
240
|
+
const n = Number.parseInt(m[0].replace(/,/g, ""), 10);
|
|
241
|
+
if (Number.isFinite(n) && n >= 10) out.push(n);
|
|
242
|
+
}
|
|
243
|
+
return out;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* The keys a finding is deduplicated on.
|
|
248
|
+
*
|
|
249
|
+
* Sorted and de-duplicated, so two runs over one finding agree byte for byte
|
|
250
|
+
* and a caller can print the intersection without re-deriving it.
|
|
251
|
+
*/
|
|
252
|
+
export function symptomKeys(finding: Finding): string[] {
|
|
253
|
+
const keys = new Set<string>();
|
|
254
|
+
// The title contributes symbols and paths but NOT quantities: a title
|
|
255
|
+
// states a headline number the body also states, and counting it twice
|
|
256
|
+
// would let two findings agree on one reading reported in two places.
|
|
257
|
+
for (const span of [...codeSpans(finding.body), finding.title]) {
|
|
258
|
+
for (const id of identifiersIn(span)) keys.add(`sym:${id}`);
|
|
259
|
+
}
|
|
260
|
+
for (const p of [...pathsIn(finding.body), ...pathsIn(finding.title)]) keys.add(`file:${p}`);
|
|
261
|
+
for (const q of quantitiesIn(finding.body)) keys.add(`qty:${q}`);
|
|
262
|
+
return [...keys].sort();
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** A stable identity for the symptom as a whole — the one hashing binding. */
|
|
266
|
+
export function symptomHash(finding: Finding): string {
|
|
267
|
+
return contentHash(symptomKeys(finding).join("\n"));
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Keys two findings have in common, sorted. */
|
|
271
|
+
export function sharedKeys(a: Finding, b: Finding): string[] {
|
|
272
|
+
const left = new Set(symptomKeys(a));
|
|
273
|
+
return symptomKeys(b).filter(k => left.has(k));
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Do two findings describe the same measured symptom?
|
|
278
|
+
*
|
|
279
|
+
* **The two key kinds answer different questions and the rule needs both.**
|
|
280
|
+
* A shared `sym:` or `file:` key says the findings are about the same area
|
|
281
|
+
* of code. A shared `qty:` key says they report the same *reading*. Only the
|
|
282
|
+
* second is a measurement, and #1187 asks for a key derived from the
|
|
283
|
+
* measured symptom.
|
|
284
|
+
*
|
|
285
|
+
* The corpus says why that distinction is the whole rule. Over the 40 most
|
|
286
|
+
* recent issues, a plain "share at least three keys of any kind" test flags
|
|
287
|
+
* **19 pairs, of which 2 are real**. Six of the seventeen are one cluster —
|
|
288
|
+
* #1175, #1176, #1177, #1184, #1195, #1196 all landed on
|
|
289
|
+
* `lib/bash-write-guard.ts` in a single afternoon and name the same
|
|
290
|
+
* functions while describing genuinely different defects. Every one of those
|
|
291
|
+
* six shares **zero or one** quantity. Requiring two drops the cluster
|
|
292
|
+
* whole, and both real pairs clear it: #1175/#1177 agree on 144, 234 and
|
|
293
|
+
* 5,060, and #1195/#1196 on two more.
|
|
294
|
+
*
|
|
295
|
+
* ## The honest caveat
|
|
296
|
+
*
|
|
297
|
+
* The two target pairs were fixed as the requirement before any of this was
|
|
298
|
+
* written. **The kind-split was not** — it was chosen after reading the
|
|
299
|
+
* corpus it is measured on, so the 5-pair reading below is a fit, not a
|
|
300
|
+
* forecast. What makes it worth shipping anyway is that the surviving false
|
|
301
|
+
* positives are one explainable class rather than a residue: all three
|
|
302
|
+
* involve #1187, which quotes other findings' numbers verbatim because it is
|
|
303
|
+
* *about* them. A finding that re-reports another's readings is the shape
|
|
304
|
+
* this rule cannot tell from a duplicate, and nothing here claims otherwise.
|
|
305
|
+
*/
|
|
306
|
+
export const DUPLICATE_MIN_SHARED_QUANTITIES = 2;
|
|
307
|
+
export const DUPLICATE_MIN_SHARED_LOCATIONS = 1;
|
|
308
|
+
|
|
309
|
+
/** Does `text` reference `id` as an issue number? */
|
|
310
|
+
function references(text: string, id: string): boolean {
|
|
311
|
+
if (!/^\d+$/.test(id)) return false;
|
|
312
|
+
return new RegExp(`#${id}\\b`).test(text);
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Two findings that cite each other are not duplicates.
|
|
317
|
+
*
|
|
318
|
+
* An author who writes `#1177` knows #1177 exists; what they are filing is a
|
|
319
|
+
* comment on it, a follow-on from it, or — in the case that made this rule
|
|
320
|
+
* necessary — an issue *about* a set of other findings, quoting their
|
|
321
|
+
* readings in order to argue from them. #1187 is that shape: it reproduces
|
|
322
|
+
* #1175's 234 and 5,060 and #1176's 60 verbatim, and under measurement alone
|
|
323
|
+
* it is indistinguishable from a duplicate of each.
|
|
324
|
+
*
|
|
325
|
+
* Measured: removes 2 false positives of 7 and neither target pair, which is
|
|
326
|
+
* the asymmetry worth having — it can only ever *allow* a filing, so the
|
|
327
|
+
* worst case is a duplicate that gets through, never a real finding
|
|
328
|
+
* withheld.
|
|
329
|
+
*/
|
|
330
|
+
export function duplicateOverlap(a: Finding, b: Finding): { duplicate: boolean; keys: string[] } {
|
|
331
|
+
const keys = sharedKeys(a, b);
|
|
332
|
+
if (references(a.body, b.id) || references(b.body, a.id)) return { duplicate: false, keys };
|
|
333
|
+
const quantities = keys.filter(k => k.startsWith("qty:")).length;
|
|
334
|
+
const locations = keys.length - quantities;
|
|
335
|
+
return {
|
|
336
|
+
duplicate:
|
|
337
|
+
quantities >= DUPLICATE_MIN_SHARED_QUANTITIES &&
|
|
338
|
+
locations >= DUPLICATE_MIN_SHARED_LOCATIONS,
|
|
339
|
+
keys,
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/**
|
|
344
|
+
* Issues in `existing` whose symptom collides with this one, strongest first.
|
|
345
|
+
*
|
|
346
|
+
* Extracted from `arbitrate`'s stage 1 so that stage has **one binding**.
|
|
347
|
+
* `arbitrate` runs it with a verifier group behind it;
|
|
348
|
+
* `lib/issue-filing-gate.ts` runs it at the `gh issue create` chokepoint,
|
|
349
|
+
* where no group has been convened and the dedupe is the only stage that is
|
|
350
|
+
* decidable at all. Two spellings of this loop would be two deciders over one
|
|
351
|
+
* question, and a mutation of either would leave the other holding the case
|
|
352
|
+
* up (`.claude/rules/checks-must-be-able-to-fail.md`, fourth shape) — which
|
|
353
|
+
* is the same reasoning that deleted the redundant `isAbsolute` check in
|
|
354
|
+
* `containedCitationPath` above.
|
|
355
|
+
*
|
|
356
|
+
* Identity is REFERENCE identity, and the id clause is secondary.
|
|
357
|
+
*
|
|
358
|
+
* Keying self-skip on the id string alone made the in-batch dedupe a no-op in
|
|
359
|
+
* the one configuration it was written for: the CLI labels every unfiled
|
|
360
|
+
* finding `UNFILED_ID`, so two findings in one batch both carried `"new"`,
|
|
361
|
+
* each skipped the other as itself, and both survived. Measured on the merged
|
|
362
|
+
* code with the real #1195/#1196 bodies — 2 survivors under `"new"`/`"new"`,
|
|
363
|
+
* 1 under their real ids. The acceptance case was green for a true reason
|
|
364
|
+
* about an input the CLI never produces.
|
|
365
|
+
*
|
|
366
|
+
* The sort is here rather than at the call site for the same reason: a caller
|
|
367
|
+
* that forgot it would name a weaker match than the one it had, and "the
|
|
368
|
+
* reader lands on the best match" is a property of the collision list, not of
|
|
369
|
+
* whoever asked for it.
|
|
370
|
+
*/
|
|
371
|
+
export function duplicateCollisions(
|
|
372
|
+
finding: Finding,
|
|
373
|
+
existing: readonly Finding[],
|
|
374
|
+
): Array<{ id: string; keys: string[] }> {
|
|
375
|
+
const collisions: Array<{ id: string; keys: string[] }> = [];
|
|
376
|
+
for (const other of existing) {
|
|
377
|
+
if (other === finding) continue;
|
|
378
|
+
if (other.id === finding.id && finding.id !== UNFILED_ID) continue;
|
|
379
|
+
const overlap = duplicateOverlap(finding, other);
|
|
380
|
+
if (overlap.duplicate) collisions.push({ id: other.id, keys: overlap.keys });
|
|
381
|
+
}
|
|
382
|
+
// Strongest overlap first, so the reader lands on the best match.
|
|
383
|
+
collisions.sort((a, b) => b.keys.length - a.keys.length || a.id.localeCompare(b.id));
|
|
384
|
+
return collisions;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
// ── Stage 1b: branch scope ───────────────────────────────────────────────
|
|
388
|
+
|
|
389
|
+
function git(root: string, args: string[]): { ok: boolean; out: string } {
|
|
390
|
+
const r = spawnSync("git", ["-C", root, ...args], { encoding: "utf-8", timeout: 20_000 });
|
|
391
|
+
return { ok: r.status === 0, out: r.stdout ?? "" };
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/** Symbols cited by a finding, in body order, capped. */
|
|
395
|
+
export function citedSymbols(finding: Finding): string[] {
|
|
396
|
+
return symptomKeys(finding)
|
|
397
|
+
.filter(k => k.startsWith("sym:"))
|
|
398
|
+
.map(k => k.slice(4))
|
|
399
|
+
.slice(0, MAX_SYMBOLS_PROBED);
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
/** `path:line` citations a finding makes, capped at `MAX_LINES_PROBED`. */
|
|
403
|
+
export function citedLines(finding: Finding): Array<{ path: string; line: number }> {
|
|
404
|
+
const out: Array<{ path: string; line: number }> = [];
|
|
405
|
+
const seen = new Set<string>();
|
|
406
|
+
for (const m of finding.body.matchAll(
|
|
407
|
+
/((?:[\w.@-]+\/)*[\w.-]+\.(?:ts|tsx|js|mjs|cjs|md|json|ya?ml|sh)):(\d+)/g,
|
|
408
|
+
)) {
|
|
409
|
+
const key = `${m[1].toLowerCase()}:${m[2]}`;
|
|
410
|
+
if (seen.has(key)) continue;
|
|
411
|
+
seen.add(key);
|
|
412
|
+
out.push({ path: m[1], line: Number.parseInt(m[2], 10) });
|
|
413
|
+
if (out.length >= MAX_LINES_PROBED) break;
|
|
414
|
+
}
|
|
415
|
+
return out;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/**
|
|
419
|
+
* The absolute path a citation resolves to, or `null` if it escapes the root.
|
|
420
|
+
*
|
|
421
|
+
* The citation text comes from an AI reviewer's output and reaches
|
|
422
|
+
* `readFileSync`. The regex above cannot emit a leading `/`, so an absolute
|
|
423
|
+
* path never arrives — but `../` does, and `../../../../Users/…/settings.json`
|
|
424
|
+
* carries an allowlisted extension and matches cleanly. The payoff is only a
|
|
425
|
+
* line count and an existence oracle rather than content, which is why this is
|
|
426
|
+
* two lines and not a subsystem; it is here because an unaudited input should
|
|
427
|
+
* not be choosing which file the arbiter opens.
|
|
428
|
+
*
|
|
429
|
+
* **There is deliberately no separate `isAbsolute` rejection**, and this was
|
|
430
|
+
* measured rather than reasoned: one was written, and removing it changed
|
|
431
|
+
* **0 of 63** cases. `resolve(root, "/etc/hosts.json")` is `/etc/hosts.json`,
|
|
432
|
+
* which fails containment on its own — so the two guards answered one
|
|
433
|
+
* question, no behavioural test could tell them apart, and a mutation of
|
|
434
|
+
* either left the other holding the case up. That is the fourth shape in
|
|
435
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`, and the remedy for it is
|
|
436
|
+
* deletion, not a second assertion.
|
|
437
|
+
*/
|
|
438
|
+
export function containedCitationPath(projectRoot: string, cited: string): string | null {
|
|
439
|
+
const root = resolve(projectRoot);
|
|
440
|
+
const abs = resolve(root, cited);
|
|
441
|
+
return abs === root || abs.startsWith(root + sep) ? abs : null;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
// ── The verdict ──────────────────────────────────────────────────────────
|
|
445
|
+
|
|
446
|
+
export type ArbiterRefusal =
|
|
447
|
+
/** The symptom is already filed. `duplicateOf` names where. */
|
|
448
|
+
| "DUPLICATE"
|
|
449
|
+
/** The verifier group did not reach quorum on "is this real?". */
|
|
450
|
+
| "NO_QUORUM"
|
|
451
|
+
/** Style, diagnostics or cleanup. Dropped outright, never posted. */
|
|
452
|
+
| "DROPPED_CATEGORY"
|
|
453
|
+
/** A cited line is past the end of the file it cites. */
|
|
454
|
+
| "STALE_CITATION"
|
|
455
|
+
/** Something could not be evaluated. Fails; never skips. */
|
|
456
|
+
| "UNMEASURED";
|
|
457
|
+
|
|
458
|
+
/**
|
|
459
|
+
* Which tree the finding describes.
|
|
460
|
+
*
|
|
461
|
+
* `OFF_MAIN` used to be a **refusal**, and the inversion of that is the
|
|
462
|
+
* central correction in this change. A finding whose symbols exist only on an
|
|
463
|
+
* unmerged branch is not a bad finding — it is a finding about the branch,
|
|
464
|
+
* which is precisely what an inline PR comment is for. Refusing it was the
|
|
465
|
+
* old shape, where the only two outcomes were "file an issue" and "nothing".
|
|
466
|
+
*
|
|
467
|
+
* It survives as a reading because it still decides one thing: a finding
|
|
468
|
+
* about code that exists only on this branch **cannot** be out-of-scope work
|
|
469
|
+
* for this PR, so it can never be routed to `ISSUE` however the caller marks
|
|
470
|
+
* it.
|
|
471
|
+
*/
|
|
472
|
+
export type FindingScope = "main" | "branch";
|
|
473
|
+
|
|
474
|
+
export interface Verdict {
|
|
475
|
+
id: string;
|
|
476
|
+
/**
|
|
477
|
+
* What to do with it. **One field, because this is one decision** — a
|
|
478
|
+
* separate `file: boolean` alongside it would be a second decider over the
|
|
479
|
+
* same input, and a mutation of either would leave the other holding the
|
|
480
|
+
* case up (`.claude/rules/checks-must-be-able-to-fail.md`, fourth shape).
|
|
481
|
+
*/
|
|
482
|
+
disposition: Disposition;
|
|
483
|
+
refusal: ArbiterRefusal | null;
|
|
484
|
+
/** Why, in words a reader can act on without re-running anything. */
|
|
485
|
+
reason: string;
|
|
486
|
+
/** `null` when the branch-scope reading was not reached. */
|
|
487
|
+
scope: FindingScope | null;
|
|
488
|
+
category: FindingCategory;
|
|
489
|
+
/** `null` when nothing could tag it; a dropped finding has no severity. */
|
|
490
|
+
severity: Severity | null;
|
|
491
|
+
/** The vote count, or `null` when no group was convened. */
|
|
492
|
+
jury: JuryReading | null;
|
|
493
|
+
/** Issue ids whose symptom this one collides with. */
|
|
494
|
+
duplicateOf: string[];
|
|
495
|
+
/** The keys that produced the collision. Empty when there was none. */
|
|
496
|
+
collidingKeys: string[];
|
|
497
|
+
/**
|
|
498
|
+
* How many findings carry this symptom, this one included. #1187's
|
|
499
|
+
* acceptance: a duplicate increments a count rather than opening an issue.
|
|
500
|
+
*/
|
|
501
|
+
occurrences: number;
|
|
502
|
+
symptomHash: string;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
export interface ArbitrateOptions {
|
|
506
|
+
/** Repository root. `origin/main` is read from here; never checked out. */
|
|
507
|
+
projectRoot: string;
|
|
508
|
+
/** Findings already posted or filed. The dedupe compares against these. */
|
|
509
|
+
existing: readonly Finding[];
|
|
510
|
+
/**
|
|
511
|
+
* The separate verifier group's votes on "is this real?".
|
|
512
|
+
*
|
|
513
|
+
* **Absent is UNMEASURED, not a pass.** A finding nobody voted on has not
|
|
514
|
+
* cleared a quorum; it has not had one. Those are different states and only
|
|
515
|
+
* one of them is reportable. There is deliberately no `skipJury` escape —
|
|
516
|
+
* `skipRepoChecks` below exists because a caller may genuinely have no
|
|
517
|
+
* repository, and no caller genuinely has no verifiers.
|
|
518
|
+
*/
|
|
519
|
+
votes?: readonly VerifierVote[];
|
|
520
|
+
/** Group size and quorum. Both default; see `lib/finding-jury.ts`. */
|
|
521
|
+
jury?: JuryConfig;
|
|
522
|
+
/**
|
|
523
|
+
* The PR's changed files. Supplying them is what makes `Pre-existing`
|
|
524
|
+
* decidable: a finding citing none of them is about code the change never
|
|
525
|
+
* touched. Absent, the pre-existing axis is simply not read.
|
|
526
|
+
*/
|
|
527
|
+
changedFiles?: readonly string[];
|
|
528
|
+
/**
|
|
529
|
+
* Skip the two `git`-backed checks. For a caller with no repository, and
|
|
530
|
+
* for the corpus measurement, which is about the key and nothing else.
|
|
531
|
+
* The skipped checks report `UNMEASURED`, they do not report a pass.
|
|
532
|
+
*/
|
|
533
|
+
skipRepoChecks?: boolean;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
/**
|
|
537
|
+
* The single place that decides. One exported function, one binding.
|
|
538
|
+
*
|
|
539
|
+
* Checks run in cost order and short-circuit, so a duplicate is never paid a
|
|
540
|
+
* `git grep` for. That ordering is the whole design: #1187's measurement is
|
|
541
|
+
* that the cheap questions catch the failures, and the expensive instrument
|
|
542
|
+
* was being spent first.
|
|
543
|
+
*/
|
|
544
|
+
export function arbitrate(finding: Finding, options: ArbitrateOptions): Verdict {
|
|
545
|
+
const hash = symptomHash(finding);
|
|
546
|
+
const votes = options.votes;
|
|
547
|
+
const category = votes ? juryCategory(finding, votes) : classifyCategory(finding);
|
|
548
|
+
const base = {
|
|
549
|
+
id: finding.id,
|
|
550
|
+
duplicateOf: [] as string[],
|
|
551
|
+
collidingKeys: [] as string[],
|
|
552
|
+
occurrences: 1,
|
|
553
|
+
symptomHash: hash,
|
|
554
|
+
scope: null as FindingScope | null,
|
|
555
|
+
category,
|
|
556
|
+
severity: null as Severity | null,
|
|
557
|
+
jury: null as JuryReading | null,
|
|
558
|
+
};
|
|
559
|
+
const drop = (refusal: ArbiterRefusal, reason: string, extra: Partial<Verdict> = {}): Verdict => ({
|
|
560
|
+
...base,
|
|
561
|
+
...extra,
|
|
562
|
+
disposition: "DROP",
|
|
563
|
+
refusal,
|
|
564
|
+
reason,
|
|
565
|
+
});
|
|
566
|
+
|
|
567
|
+
const keys = symptomKeys(finding);
|
|
568
|
+
if (keys.length === 0) {
|
|
569
|
+
return drop("UNMEASURED", "the finding names no symbol, no file and no reading — nothing here can be evaluated");
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
// ── 1. is this new? ──
|
|
573
|
+
const collisions = duplicateCollisions(finding, options.existing);
|
|
574
|
+
if (collisions.length > 0) {
|
|
575
|
+
return drop(
|
|
576
|
+
"DUPLICATE",
|
|
577
|
+
`the symptom is already reported as ${collisions.map(c => c.id).join(", ")} — ` +
|
|
578
|
+
`shared ${collisions[0].keys.length} key(s): ${collisions[0].keys.slice(0, 6).join(", ")}`,
|
|
579
|
+
{
|
|
580
|
+
duplicateOf: collisions.map(c => c.id),
|
|
581
|
+
collidingKeys: collisions[0].keys,
|
|
582
|
+
occurrences: collisions.length + 1,
|
|
583
|
+
},
|
|
584
|
+
);
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// ── 2. do enough of the verifiers agree it is real? ──
|
|
588
|
+
if (votes === undefined) {
|
|
589
|
+
return drop("UNMEASURED", "no verifier group was convened, so no quorum was reached — not reaching one and not having one are different states");
|
|
590
|
+
}
|
|
591
|
+
const jury = juryReading(votes, options.jury);
|
|
592
|
+
if (jury.reading === "UNMEASURED") {
|
|
593
|
+
return drop("UNMEASURED", `the vote could not be read: ${jury.unmeasuredReasons.join("; ")}`, { jury });
|
|
594
|
+
}
|
|
595
|
+
if (!jury.passed) {
|
|
596
|
+
return drop(
|
|
597
|
+
"NO_QUORUM",
|
|
598
|
+
`${jury.agree} of ${jury.votes} verifiers called it real, quorum is ${jury.quorum}`,
|
|
599
|
+
{ jury },
|
|
600
|
+
);
|
|
601
|
+
}
|
|
602
|
+
const withJury = { ...base, jury };
|
|
603
|
+
|
|
604
|
+
// ── 3. is it the kind of finding that gets posted at all? ──
|
|
605
|
+
if (DROPPED_CATEGORIES.has(category)) {
|
|
606
|
+
return drop(
|
|
607
|
+
"DROPPED_CATEGORY",
|
|
608
|
+
`${category} findings are dropped outright: they cost a review round and change no behaviour`,
|
|
609
|
+
{ jury },
|
|
610
|
+
);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
if (options.skipRepoChecks === true) {
|
|
614
|
+
return drop("UNMEASURED", "branch scope and citation currency were not read: skipRepoChecks", { jury });
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
// ── 4. which tree is it about? ──
|
|
618
|
+
const symbols = citedSymbols(finding);
|
|
619
|
+
if (symbols.length === 0) {
|
|
620
|
+
return drop("UNMEASURED", "the finding cites no symbol, so whether it describes `origin/main` cannot be read", { jury });
|
|
621
|
+
}
|
|
622
|
+
const probe = git(options.projectRoot, ["rev-parse", "--verify", "--quiet", "origin/main"]);
|
|
623
|
+
if (!probe.ok) {
|
|
624
|
+
return drop(
|
|
625
|
+
"UNMEASURED",
|
|
626
|
+
`no origin/main at ${options.projectRoot} — branch scope is unreadable, which fails rather than passes`,
|
|
627
|
+
{ jury },
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
const present = symbols.filter(
|
|
631
|
+
s => git(options.projectRoot, ["grep", "--quiet", "--fixed-strings", "-e", s, "origin/main"]).ok,
|
|
632
|
+
);
|
|
633
|
+
const scope: FindingScope = present.length > 0 ? "main" : "branch";
|
|
634
|
+
|
|
635
|
+
// ── 5. is this current? ──
|
|
636
|
+
const stale: string[] = [];
|
|
637
|
+
for (const c of citedLines(finding)) {
|
|
638
|
+
const abs = containedCitationPath(options.projectRoot, c.path);
|
|
639
|
+
if (abs === null) continue;
|
|
640
|
+
let text: string | null = null;
|
|
641
|
+
if (existsSync(abs)) {
|
|
642
|
+
text = readFileSync(abs, "utf-8");
|
|
643
|
+
} else {
|
|
644
|
+
const shown = git(options.projectRoot, ["show", `origin/main:${c.path}`]);
|
|
645
|
+
if (shown.ok) text = shown.out;
|
|
646
|
+
}
|
|
647
|
+
// A path that resolves nowhere is not a stale citation — the evidence
|
|
648
|
+
// gate is where an unresolvable citation is adjudicated, and guessing
|
|
649
|
+
// here would put a second decider in front of it.
|
|
650
|
+
if (text === null) continue;
|
|
651
|
+
const lines = text.split("\n").length;
|
|
652
|
+
if (c.line > lines) stale.push(`${c.path}:${c.line} (file is ${lines} lines)`);
|
|
653
|
+
}
|
|
654
|
+
if (stale.length > 0) {
|
|
655
|
+
return drop(
|
|
656
|
+
"STALE_CITATION",
|
|
657
|
+
`cited line(s) past the end of the file: ${stale.join("; ")} — re-read before posting`,
|
|
658
|
+
{ jury, scope },
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
// ── 6. where does it go? COMMENT unless somebody declared otherwise. ──
|
|
663
|
+
const severity = severityOf(finding, votes, options.changedFiles) ?? "Important";
|
|
664
|
+
const outOfScope = finding.outOfScope;
|
|
665
|
+
if (outOfScope !== undefined && outOfScope.reason.trim() !== "" && scope === "main") {
|
|
666
|
+
return {
|
|
667
|
+
...withJury,
|
|
668
|
+
scope,
|
|
669
|
+
severity,
|
|
670
|
+
disposition: "ISSUE",
|
|
671
|
+
refusal: null,
|
|
672
|
+
reason: `out of scope for this change, declared: ${outOfScope.reason}`,
|
|
673
|
+
};
|
|
674
|
+
}
|
|
675
|
+
return {
|
|
676
|
+
...withJury,
|
|
677
|
+
scope,
|
|
678
|
+
severity,
|
|
679
|
+
disposition: "COMMENT",
|
|
680
|
+
refusal: null,
|
|
681
|
+
reason:
|
|
682
|
+
scope === "main"
|
|
683
|
+
? `${jury.agree} of ${jury.votes} verifiers agree; cites ${present.join(", ")} on origin/main`
|
|
684
|
+
: `${jury.agree} of ${jury.votes} verifiers agree; cites only branch symbols (${symbols.join(", ")}), so it is about this change`,
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
/** One line a caller can print without re-deriving any of the above. */
|
|
689
|
+
export function verdictSummary(v: Verdict): string {
|
|
690
|
+
const tag = v.severity ? `${v.severity}/${v.category}` : v.category;
|
|
691
|
+
return v.refusal === null
|
|
692
|
+
? `${v.disposition} ${v.id} [${tag}]: ${v.reason}`
|
|
693
|
+
: `DROP ${v.id} (${v.refusal}): ${v.reason}`;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
// ── The batch, and the cap that is not written here ──────────────────────
|
|
697
|
+
|
|
698
|
+
/**
|
|
699
|
+
* How many findings one batch may file.
|
|
700
|
+
*
|
|
701
|
+
* Six, which is the measured hourly CLOSE rate at the time #1187 was filed —
|
|
702
|
+
* 14 filed against 6 closed. A filing budget above the closing rate is not a
|
|
703
|
+
* budget, it is a slower queue growth, and the queue was the constraint.
|
|
704
|
+
*
|
|
705
|
+
* Deliberately NOT `PROMOTION_BUDGET_RULES`. That sixteen is a context-window
|
|
706
|
+
* reading about how many rules an agent can hold at once
|
|
707
|
+
* (`docs/research/context-management.md`); it answers a different question
|
|
708
|
+
* and reusing the literal would make one number the decider for two
|
|
709
|
+
* unrelated decisions. The *mechanism* is shared; the number is not.
|
|
710
|
+
*/
|
|
711
|
+
export const FILING_BUDGET_FINDINGS = 6;
|
|
712
|
+
|
|
713
|
+
/**
|
|
714
|
+
* Why a candidate did not reach a reader.
|
|
715
|
+
*
|
|
716
|
+
* `NO_QUORUM` and `DROPPED_CATEGORY` are **separate values and must stay
|
|
717
|
+
* separate**. They are the two filters this module adds, and collapsing them
|
|
718
|
+
* into one "suppressed" reason would make the quorum and the category filter
|
|
719
|
+
* inseparable in exactly the A/B that exists to price them apart.
|
|
720
|
+
*
|
|
721
|
+
* `WithheldReason` is `lib/rule-promotion.ts`'s own vocabulary, reused rather
|
|
722
|
+
* than restated: the promoter decides which candidates it withheld and why,
|
|
723
|
+
* and a parallel enum here would be a second name for one decision.
|
|
724
|
+
*
|
|
725
|
+
* - `BATCH_UNMEASURED` — the candidate was fine and the BATCH was not. A
|
|
726
|
+
* candidate refused `UNMEASURED` in its own right carries `UNMEASURED`;
|
|
727
|
+
* the two are deliberately distinguishable, because the first set is what
|
|
728
|
+
* explains the second.
|
|
729
|
+
* - `UNACCOUNTED` — the promoter returned this candidate in neither
|
|
730
|
+
* `promoted` nor `withheld`. It partitions its input, so this cannot
|
|
731
|
+
* happen; it is named instead of defaulted so that if the partition ever
|
|
732
|
+
* breaks the ledger says so rather than reporting a plausible reason.
|
|
733
|
+
*/
|
|
734
|
+
export type WithheldBy =
|
|
735
|
+
| ArbiterRefusal
|
|
736
|
+
| WithheldReason
|
|
737
|
+
| "BATCH_UNMEASURED"
|
|
738
|
+
| "UNACCOUNTED";
|
|
739
|
+
|
|
740
|
+
/**
|
|
741
|
+
* One row per finding the batch considered, **including the ones it withheld**.
|
|
742
|
+
*
|
|
743
|
+
* ## Why this exists at all
|
|
744
|
+
*
|
|
745
|
+
* The quorum's cardinality (`VERIFIER_GROUP_SIZE`, `VERIFIER_QUORUM`) is a
|
|
746
|
+
* defended default and not a measurement, and the category filter's only
|
|
747
|
+
* backwards reading is a safety one — 0 dropped of 385 known-real findings.
|
|
748
|
+
* Both gaps have the same cause: **nothing records a finding before it is
|
|
749
|
+
* filed.** The tracker is post-filing by construction, there are no inline
|
|
750
|
+
* review comments to mine, and a candidate the quorum votes down leaves no
|
|
751
|
+
* artefact anywhere. So the population those numbers would be derived from
|
|
752
|
+
* does not exist in history and **cannot be reconstructed from it** — it has
|
|
753
|
+
* to be written down at the moment the candidate is withheld, or never.
|
|
754
|
+
*
|
|
755
|
+
* This is that record. It is deliberately a list of rows and not a tally: a
|
|
756
|
+
* count cannot be joined against an outcome later, and the join is the whole
|
|
757
|
+
* point.
|
|
758
|
+
*
|
|
759
|
+
* ## How it is read
|
|
760
|
+
*
|
|
761
|
+
* `evals/ai-review-control/outcomes.ts`'s `compareReviewOutcomes(before,
|
|
762
|
+
* after)` prices a filter, and it returns UNMEASURED on a population
|
|
763
|
+
* mismatch by design — so both sides are built from **this same ledger**,
|
|
764
|
+
* differing only in which rows carry `suppressed`. Three summaries, two
|
|
765
|
+
* comparisons:
|
|
766
|
+
*
|
|
767
|
+
* | summary | rows marked suppressed |
|
|
768
|
+
* |---|---|
|
|
769
|
+
* | baseline | none |
|
|
770
|
+
* | quorum-only | `withheldBy === "NO_QUORUM"` |
|
|
771
|
+
* | both | `withheldBy` is `NO_QUORUM` or `DROPPED_CATEGORY` |
|
|
772
|
+
*
|
|
773
|
+
* `compare(baseline, quorumOnly)` prices the quorum; `compare(quorumOnly,
|
|
774
|
+
* both)` prices the category filter on top of it. The attribution lives in
|
|
775
|
+
* **which pair you compare**, which is why that function's `suppressed` is a
|
|
776
|
+
* reasonless boolean and why the reason lives here instead.
|
|
777
|
+
*
|
|
778
|
+
* ## WHAT THIS DOES NOT COVER
|
|
779
|
+
*
|
|
780
|
+
* - **Findings no hunter produced.** This records what was *considered*, not
|
|
781
|
+
* what was missable. A bug nobody reported is invisible here, and no
|
|
782
|
+
* amount of this data will price recall.
|
|
783
|
+
* - **Whether a withheld candidate was right to be withheld.** That needs an
|
|
784
|
+
* outcome, which arrives later and from somewhere else. This supplies the
|
|
785
|
+
* left-hand side of that join and nothing more.
|
|
786
|
+
*/
|
|
787
|
+
export interface CandidateRecord {
|
|
788
|
+
/**
|
|
789
|
+
* A key unique across every ledger produced under the same `batch`, for a
|
|
790
|
+
* consumer that scores a corpus keyed by one field.
|
|
791
|
+
*
|
|
792
|
+
* **`null` when the caller named no batch**, and that is the honest
|
|
793
|
+
* reading rather than a gap. Neither of the other two fields can do this
|
|
794
|
+
* job and both look as though they can:
|
|
795
|
+
*
|
|
796
|
+
* - `id` is `UNFILED_ID` for everything that has not been filed, which is
|
|
797
|
+
* every candidate the ledger exists to record.
|
|
798
|
+
* - `symptomHash` is not unique either. Measured on an exact re-report
|
|
799
|
+
* under two different titles: **1 distinct hash across 2 rows**, the
|
|
800
|
+
* original and the duplicate that collided with it. And it does not even
|
|
801
|
+
* identify the collision — the real #1195/#1196 pair hashes to
|
|
802
|
+
* `11f1ca58d515` and `609a1838a74d`, two different values for one
|
|
803
|
+
* symptom, because the dedupe matches on SHARED keys and not on the
|
|
804
|
+
* whole set.
|
|
805
|
+
*
|
|
806
|
+
* A duplicate key does not degrade a score, it inverts one: in
|
|
807
|
+
* `evals/ai-review-control/outcomes.ts` two rows sharing a key meant the
|
|
808
|
+
* second overwrote the first, so a finding with a real fixing commit read
|
|
809
|
+
* `NO_ACTION` and landed in `cost` — noise manufactured out of a fix.
|
|
810
|
+
* That module now refuses a repeated key outright, which is why the
|
|
811
|
+
* failure here must be an absent key and never a plausible one.
|
|
812
|
+
*
|
|
813
|
+
* It is position-derived rather than content-derived on purpose: unique by
|
|
814
|
+
* construction, with no collision bookkeeping to be a second decider.
|
|
815
|
+
*/
|
|
816
|
+
key: string | null;
|
|
817
|
+
id: string;
|
|
818
|
+
symptomHash: string;
|
|
819
|
+
category: FindingCategory;
|
|
820
|
+
severity: Severity | null;
|
|
821
|
+
disposition: Disposition;
|
|
822
|
+
refusal: ArbiterRefusal | null;
|
|
823
|
+
/** Did it reach a reader — a PR comment or a filed issue? */
|
|
824
|
+
emitted: boolean;
|
|
825
|
+
/** Why not. `null` exactly when `emitted` is true. */
|
|
826
|
+
withheldBy: WithheldBy | null;
|
|
827
|
+
/** The vote, when one was taken. `null` when no group was convened. */
|
|
828
|
+
jury: JuryReading | null;
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
export interface BatchReading {
|
|
832
|
+
/** Per-finding verdicts, in input order. */
|
|
833
|
+
verdicts: Verdict[];
|
|
834
|
+
/**
|
|
835
|
+
* EVERY candidate the batch considered, in input order, withheld ones
|
|
836
|
+
* included. Exactly one row per input finding — `compareReviewOutcomes`
|
|
837
|
+
* refuses a population mismatch, and this is the population.
|
|
838
|
+
*/
|
|
839
|
+
candidates: CandidateRecord[];
|
|
840
|
+
/**
|
|
841
|
+
* Ids to post as inline comments, **ranked by severity** — Important, then
|
|
842
|
+
* Pre-existing, then Nit. This is the default output and it is deliberately
|
|
843
|
+
* NOT budgeted: a comment blocks the merge until it is answered, so it
|
|
844
|
+
* leaves the system the same day. `FILING_BUDGET_FINDINGS` caps the other
|
|
845
|
+
* list, which is the one that accumulates.
|
|
846
|
+
*/
|
|
847
|
+
toComment: string[];
|
|
848
|
+
/** `lib/rule-promotion.ts`'s reading over the declared-out-of-scope ones. */
|
|
849
|
+
promotion: PromotionReading;
|
|
850
|
+
/** Ids cleared to file as issues, after the checks and the budget. */
|
|
851
|
+
toFile: string[];
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
/**
|
|
855
|
+
* The order the batch is walked in.
|
|
856
|
+
*
|
|
857
|
+
* **Order is load-bearing and therefore pinned here**: with an
|
|
858
|
+
* accumulate-as-you-walk dedupe, whichever of a duplicate pair is seen first
|
|
859
|
+
* is the one that survives, so leaving it to array position would let the
|
|
860
|
+
* caller's iteration order silently decide which finding exists. Numeric ids
|
|
861
|
+
* compare numerically so `9` precedes `10`; everything else falls back to
|
|
862
|
+
* lexical, and the original index breaks remaining ties so the sort is total.
|
|
863
|
+
*/
|
|
864
|
+
function walkOrder(findings: readonly Finding[]): Array<{ finding: Finding; index: number }> {
|
|
865
|
+
return findings
|
|
866
|
+
.map((finding, index) => ({ finding, index }))
|
|
867
|
+
.sort((a, b) => {
|
|
868
|
+
const [x, y] = [Number(a.finding.id), Number(b.finding.id)];
|
|
869
|
+
const numeric = Number.isInteger(x) && Number.isInteger(y);
|
|
870
|
+
return (numeric ? x - y : a.finding.id.localeCompare(b.finding.id)) || a.index - b.index;
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/**
|
|
875
|
+
* Arbitrate a batch, then hand the survivors to the promoter that already
|
|
876
|
+
* exists.
|
|
877
|
+
*
|
|
878
|
+
* **The ranking, the eviction and the UNMEASURED folding are not implemented
|
|
879
|
+
* here.** `promoteCandidates` does all three, has them under test, and
|
|
880
|
+
* already gets the hard part right — an upstream that read UNMEASURED makes
|
|
881
|
+
* the whole promotion UNMEASURED rather than a smaller one. #1187 and #1162
|
|
882
|
+
* both say the same thing about this machinery: wire it, do not rewrite it.
|
|
883
|
+
* What stage 1 adds is the three checks in front and a budget whose number
|
|
884
|
+
* is its own.
|
|
885
|
+
*
|
|
886
|
+
* A finding refused by any check is withheld before it reaches the promoter,
|
|
887
|
+
* so it cannot be ranked above one that passed them.
|
|
888
|
+
*
|
|
889
|
+
* ## The dedupe accumulates as it walks, and that is the whole point
|
|
890
|
+
*
|
|
891
|
+
* The first cut of this compared every finding against `options.existing` and
|
|
892
|
+
* nothing else. `existing` is a **fixed snapshot**, so two identical findings
|
|
893
|
+
* in one batch both read as new — which is exactly the incident that
|
|
894
|
+
* motivated #1187: #1195 and #1196, the same symptom, filed **one minute
|
|
895
|
+
* apart**, close enough that neither was in the tracker when the other was
|
|
896
|
+
* arbitrated. A tracker-only dedupe would not have stopped it, and
|
|
897
|
+
* `test/finding-arbiter.test.ts`'s `AR-9` drives the real pair through this
|
|
898
|
+
* function to prove the fix does.
|
|
899
|
+
*
|
|
900
|
+
* So each survivor joins the comparison set for everything walked after it.
|
|
901
|
+
* See `walkOrder` for why the order is pinned rather than inherited.
|
|
902
|
+
*/
|
|
903
|
+
export function arbitrateBatch(
|
|
904
|
+
findings: readonly Finding[],
|
|
905
|
+
options: ArbitrateOptions & { budget?: number; now?: string; batch?: string },
|
|
906
|
+
): BatchReading {
|
|
907
|
+
const date = options.now ?? new Date().toISOString();
|
|
908
|
+
const seen: Finding[] = [...options.existing];
|
|
909
|
+
const verdicts: Verdict[] = new Array(findings.length);
|
|
910
|
+
|
|
911
|
+
for (const { finding, index } of walkOrder(findings)) {
|
|
912
|
+
const verdict = arbitrate(finding, { ...options, existing: seen });
|
|
913
|
+
verdicts[index] = verdict;
|
|
914
|
+
// Only something that will actually be posted or filed occupies the
|
|
915
|
+
// space. A dropped finding leaves no artefact for a later one to
|
|
916
|
+
// duplicate, so it must not shadow one.
|
|
917
|
+
if (verdict.disposition !== "DROP") seen.push(finding);
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
// ── Everything below is keyed by POSITION, never by id ─────────────────
|
|
921
|
+
//
|
|
922
|
+
// `UNFILED_ID` is the id of every finding that has not been filed yet, so
|
|
923
|
+
// ids are not unique within a batch: the #1195/#1196 incident arrives as
|
|
924
|
+
// two rows both reading `"new"`, one of which is dropped as a duplicate of
|
|
925
|
+
// the other. A lookup by id resolves both to whichever came first, which
|
|
926
|
+
// would title one candidate with the other's title and mark a withheld
|
|
927
|
+
// candidate as emitted. The index is the only identity a candidate has
|
|
928
|
+
// before it is filed.
|
|
929
|
+
const withIndex = verdicts.map((verdict, index) => ({ verdict, index }));
|
|
930
|
+
const bySeverity = (a: Verdict, b: Verdict) =>
|
|
931
|
+
SEVERITY_RANK[a.severity ?? "Important"] - SEVERITY_RANK[b.severity ?? "Important"];
|
|
932
|
+
const commented = withIndex
|
|
933
|
+
.filter(({ verdict }) => verdict.disposition === "COMMENT")
|
|
934
|
+
.sort((a, b) => bySeverity(a.verdict, b.verdict));
|
|
935
|
+
const toComment = commented.map(({ verdict }) => verdict.id);
|
|
936
|
+
|
|
937
|
+
const declared: PromotableCandidate[] = withIndex
|
|
938
|
+
.filter(({ verdict }) => verdict.disposition === "ISSUE")
|
|
939
|
+
.map(({ verdict, index }) => ({
|
|
940
|
+
rule: findings[index]!.title,
|
|
941
|
+
source: `finding:${index}`,
|
|
942
|
+
hash: verdict.symptomHash,
|
|
943
|
+
evidence: { kind: "finding" as const, date, citation: verdict.reason, issues: [] },
|
|
944
|
+
}));
|
|
945
|
+
|
|
946
|
+
// The arbiter's own reading is an upstream, so a batch in which nothing
|
|
947
|
+
// could be evaluated does not report a measured zero.
|
|
948
|
+
const unmeasured = verdicts.filter(v => v.refusal === "UNMEASURED");
|
|
949
|
+
const promotion = promoteCandidates(declared, {
|
|
950
|
+
budget: options.budget ?? FILING_BUDGET_FINDINGS,
|
|
951
|
+
upstream: {
|
|
952
|
+
reading: unmeasured.length > 0 ? "UNMEASURED" : "MEASURED",
|
|
953
|
+
unmeasuredReasons: unmeasured.map(v => `${v.id}: ${v.reason}`),
|
|
954
|
+
},
|
|
955
|
+
});
|
|
956
|
+
|
|
957
|
+
// `promoteCandidates` returns a POPULATED `promoted` under an UNMEASURED
|
|
958
|
+
// reading — only its `budget < 1` branch empties it. So the discriminator
|
|
959
|
+
// has to be consulted here or it is computed and then ignored, and this
|
|
960
|
+
// function's own header would be claiming that a batch which could not be
|
|
961
|
+
// evaluated "does not report a measured zero" while it reported a measured
|
|
962
|
+
// LIST. Gated here rather than in `promoteCandidates`, which has other
|
|
963
|
+
// callers whose semantics are not ours to change.
|
|
964
|
+
const position = (c: PromotableCandidate) => Number(c.source.replace(/^finding:/, ""));
|
|
965
|
+
const filed = promotion.reading === "MEASURED" ? promotion.promoted.map(position) : [];
|
|
966
|
+
const emitted = new Set<number>([...commented.map(c => c.index), ...filed]);
|
|
967
|
+
|
|
968
|
+
// The promoter's own per-candidate reason, read rather than re-derived. A
|
|
969
|
+
// second opinion here about why something was not promoted would be two
|
|
970
|
+
// deciders over one question, and the one that is not consulted is the one
|
|
971
|
+
// that silently rots (`.claude/rules/checks-must-be-able-to-fail.md`).
|
|
972
|
+
const promoterReason = new Map<number, WithheldReason>(
|
|
973
|
+
promotion.withheld.map(w => [position(w.candidate), w.reason]),
|
|
974
|
+
);
|
|
975
|
+
|
|
976
|
+
const candidates: CandidateRecord[] = verdicts.map((verdict, index) => ({
|
|
977
|
+
// The batch name is REQUIRED for a key rather than defaulted, because a
|
|
978
|
+
// default would make every batch's first row collide with every other
|
|
979
|
+
// batch's first row — the corpus a consumer builds is many batches
|
|
980
|
+
// concatenated, and a plausible key is the one failure mode that
|
|
981
|
+
// silently shrinks it.
|
|
982
|
+
key: options.batch === undefined ? null : `${options.batch}/${index}`,
|
|
983
|
+
id: verdict.id,
|
|
984
|
+
symptomHash: verdict.symptomHash,
|
|
985
|
+
category: verdict.category,
|
|
986
|
+
severity: verdict.severity,
|
|
987
|
+
disposition: verdict.disposition,
|
|
988
|
+
refusal: verdict.refusal,
|
|
989
|
+
emitted: emitted.has(index),
|
|
990
|
+
withheldBy: emitted.has(index)
|
|
991
|
+
? null
|
|
992
|
+
: // A refusal is the candidate's OWN reading and outranks everything
|
|
993
|
+
// after it: this one never reached the promoter. Note that a
|
|
994
|
+
// candidate refused `UNMEASURED` is recorded as `UNMEASURED`, not
|
|
995
|
+
// `BATCH_UNMEASURED` — the first says this finding could not be
|
|
996
|
+
// evaluated, the second says the batch could not be, and conflating
|
|
997
|
+
// them would lose the only row that says WHY the batch folded.
|
|
998
|
+
(verdict.refusal ??
|
|
999
|
+
// Then the batch fold. It outranks the promoter's reason because an
|
|
1000
|
+
// UNMEASURED batch ranked an incomplete population, so "over budget"
|
|
1001
|
+
// read off that ranking is a number nobody should act on.
|
|
1002
|
+
(promotion.reading === "UNMEASURED"
|
|
1003
|
+
? "BATCH_UNMEASURED"
|
|
1004
|
+
: (promoterReason.get(index) ?? "UNACCOUNTED"))),
|
|
1005
|
+
jury: verdict.jury,
|
|
1006
|
+
}));
|
|
1007
|
+
|
|
1008
|
+
return {
|
|
1009
|
+
verdicts,
|
|
1010
|
+
candidates,
|
|
1011
|
+
toComment,
|
|
1012
|
+
promotion,
|
|
1013
|
+
toFile: filed.map(index => findings[index]!.id),
|
|
1014
|
+
};
|
|
1015
|
+
}
|