rungate 0.8.2 → 0.8.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/rungate.ts +11 -6
- package/config/prove-evidence.schema.json +21 -3
- package/gates/.gate-salt +1 -1
- package/gates/gate-executor.ts +153 -1
- package/gates/prove.test.ts +12 -6
- package/gates/schema.ts +69 -6
- package/gates/workflow.test.ts +12 -5
- package/hooks/BunTestPathGuard.hook.ts +106 -0
- package/hooks/DocHygiene.hook.ts +2 -1
- package/hooks/GateResultProtection.hook.ts +28 -7
- package/hooks/HookCurrency.hook.ts +48 -1
- package/hooks/TopLevelCdGuard.hook.ts +38 -6
- package/hooks/VerifyPhaseLock.hook.ts +25 -8
- package/hooks/WorkflowStateGuard.hook.ts +24 -7
- package/lib/ai-review-jury.ts +725 -0
- package/lib/ai-review-rounds.ts +791 -4
- package/lib/ai-review-unmeasured.ts +361 -0
- package/lib/bash-write-guard.ts +1235 -102
- package/lib/bun-test-paths.ts +253 -0
- package/lib/cli-commands.ts +252 -11
- package/lib/closing-reference-audit.ts +163 -0
- package/lib/conformity.ts +17 -1
- package/lib/create-sc.ts +45 -12
- package/lib/derived-spec.ts +206 -0
- package/lib/file-claims.ts +335 -0
- package/lib/finding-arbiter.ts +502 -28
- package/lib/finding-jury.ts +61 -5
- package/lib/finding-pr-routing.ts +333 -0
- package/lib/frontmatter.ts +16 -0
- package/lib/gate-check-audit.ts +646 -0
- package/lib/generators/agents-md.ts +29 -0
- package/lib/github.ts +112 -7
- package/lib/hook-tree-currency.ts +494 -0
- package/lib/issue-closure-audit.ts +161 -14
- package/lib/issue-create-override.ts +53 -3
- package/lib/issue-filing-gate.ts +1073 -76
- package/lib/local-review-rounds.ts +523 -0
- package/lib/organize.ts +21 -1
- package/lib/project-state-shape.ts +56 -5
- package/lib/release-preflight.ts +90 -1
- package/lib/repo-path.ts +94 -0
- package/lib/report-provenance.ts +218 -3
- package/lib/risk-tier.ts +302 -0
- package/lib/rounds-to-pass.ts +274 -0
- package/lib/rule-row-drift.ts +199 -0
- package/lib/sc-claim-forms.ts +33 -5
- package/lib/sc-guard.ts +11 -1
- package/lib/sc-id-cross-tree.ts +17 -2
- package/lib/sc-ids.ts +71 -6
- package/lib/sc-reservations.ts +158 -14
- package/lib/scaffold/ai-review-findings.ts +1369 -27
- package/lib/scaffold/ai-review-step.ts +4095 -119
- package/lib/scaffold/ai-review-unmeasured-workflow.ts +243 -0
- package/lib/scaffold/owned-files.ts +14 -0
- package/lib/scaffold/steps.ts +100 -9
- package/lib/scratch-roots.ts +93 -3
- package/lib/spec-updater.ts +22 -2
- package/lib/spec-write-ast.ts +728 -0
- package/lib/spec-write-sweep.ts +470 -0
- package/lib/test-suite-lock.ts +127 -13
- package/lib/ts-ast.ts +26 -0
- package/lib/validators/spec-validators.ts +36 -7
- package/lib/write-spec.ts +138 -0
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +41 -1
- package/scripts/arbitrate-finding.ts +330 -12
- package/scripts/audit-specs.ts +225 -24
- package/scripts/convert-spec.ts +7 -2
- package/scripts/create-sc.ts +14 -2
- package/scripts/create-spec.ts +5 -2
- package/scripts/detect-sc-drift.ts +10 -2
- package/scripts/record-ai-review-unmeasured.ts +370 -0
- package/scripts/split-spec.ts +17 -4
- package/scripts/sync-sc-status.ts +14 -2
package/lib/risk-tier.ts
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Risk tiers, and the classifier that reads a change to set one (#1437).
|
|
3
|
+
*
|
|
4
|
+
* Covers CRC-E1, CRC-E2, CRC-E4 (specs/CODE-REVIEW-CONVERGENCE-SPEC.md,
|
|
5
|
+
* Layer 5 — human approval scaled by risk).
|
|
6
|
+
*
|
|
7
|
+
* ## What problem this is pointed at
|
|
8
|
+
*
|
|
9
|
+
* Measured on this repository on 2026-10-23 at 19:15Z: 15 open pull requests,
|
|
10
|
+
* 2 with auto-merge armed. Nine were red on `ai-review` and not one of them
|
|
11
|
+
* for a defect in its own code — five because `main` had moved
|
|
12
|
+
* `.github/workflows/ai-review.yml` underneath them, and five because they
|
|
13
|
+
* EDIT that file, which the action refuses by construction.
|
|
14
|
+
*
|
|
15
|
+
* The thing deciding whether each of those merges was a person reading
|
|
16
|
+
* fifteen diffs. That is the bottleneck a tier is for: a change that needs
|
|
17
|
+
* nobody should not wait for somebody.
|
|
18
|
+
*
|
|
19
|
+
* ## CRC-E2 — the change sets the tier, not the author
|
|
20
|
+
*
|
|
21
|
+
* `classifyChange` takes a change. There is no author, actor, login, team or
|
|
22
|
+
* branch name in `ChangeUnderReview`, and `test/risk-tier.test.ts` asserts
|
|
23
|
+
* that the key set contains no such field, because the criterion is about
|
|
24
|
+
* what the function is ALLOWED to read and a docblock promising it cannot be
|
|
25
|
+
* checked. A classifier that can see who wrote something will eventually be
|
|
26
|
+
* asked to be lenient about it.
|
|
27
|
+
*
|
|
28
|
+
* ## CRC-E4 — a CI workflow file always needs a human
|
|
29
|
+
*
|
|
30
|
+
* `ALWAYS_HIGH_PREFIXES` is the floor and it is not reachable from config.
|
|
31
|
+
* Owner-listed sensitive paths ADD to it; nothing subtracts. A deployment
|
|
32
|
+
* that could configure its way out of "a CI workflow file needs a person"
|
|
33
|
+
* has a setting whose only use is to switch the criterion off, and the
|
|
34
|
+
* criterion exists because that is the change class where switching it off is
|
|
35
|
+
* most attractive and worst.
|
|
36
|
+
*
|
|
37
|
+
* Today this repository satisfies CRC-E4 by accident and in its worst form:
|
|
38
|
+
* `ai-review` refuses any pull request whose copy of its own workflow file
|
|
39
|
+
* differs from the default branch's, so a change to the reviewer cannot pass
|
|
40
|
+
* the reviewer, and the documented escape — the `ai-review-override` label —
|
|
41
|
+
* WAIVES the check rather than routing to a person. A waiver records nothing
|
|
42
|
+
* about whether anybody read the diff. A tier does.
|
|
43
|
+
*
|
|
44
|
+
* ## UNMEASURED is not LOW
|
|
45
|
+
*
|
|
46
|
+
* A change list that could not be read is not a small change. Every refusal
|
|
47
|
+
* here returns `tier: null` with the entries that caused it, and a caller
|
|
48
|
+
* that treats `null` as "fine" has rebuilt the fail-open this module exists
|
|
49
|
+
* to avoid. `.claude/rules/config-access.md` calls the alternative *Silent
|
|
50
|
+
* Default Substitution*; in a risk classifier it is the whole failure.
|
|
51
|
+
*
|
|
52
|
+
* ## Where the numbers came from, and CRC-E9
|
|
53
|
+
*
|
|
54
|
+
* `LOW_MAX_FILES` and `LOW_MAX_LINES` start strict and are not evidence-based
|
|
55
|
+
* yet — nothing in this repository has measured what size of change reaches
|
|
56
|
+
* main without a defect. CRC-E9 ("tiers start strict and loosen with
|
|
57
|
+
* evidence") is the criterion that makes them move, and it is deliberately
|
|
58
|
+
* not implemented here: loosening needs a recorded outcome to loosen against,
|
|
59
|
+
* and the ledger that would hold it does not carry tiers yet. Starting strict
|
|
60
|
+
* and staying there is the honest state, and it is the direction this is
|
|
61
|
+
* allowed to be wrong in.
|
|
62
|
+
*/
|
|
63
|
+
|
|
64
|
+
import { underPrefix, normaliseRepoPath } from "./repo-path";
|
|
65
|
+
|
|
66
|
+
/** Low needs no human, medium one, high two. */
|
|
67
|
+
export type RiskTier = "low" | "medium" | "high";
|
|
68
|
+
|
|
69
|
+
export interface TierPolicy {
|
|
70
|
+
/** CRC-E1: how many PEOPLE must approve. An agent is never one of them. */
|
|
71
|
+
readonly humansRequired: 0 | 1 | 2;
|
|
72
|
+
readonly meaning: string;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* CRC-E1: the tiers, written down, in one place.
|
|
77
|
+
*
|
|
78
|
+
* `humansRequired` is a count of people. CRC-E6 — an agent's approval never
|
|
79
|
+
* counts as a person's — is a property of whoever reads this number, and this
|
|
80
|
+
* repository cannot currently observe it: branch protection declares no
|
|
81
|
+
* required reviews, so there is no approval state for an agent to pollute.
|
|
82
|
+
* The field is named `humansRequired` rather than `approvalsRequired` so that
|
|
83
|
+
* the day there is one, the distinction is already in the type.
|
|
84
|
+
*/
|
|
85
|
+
export const RISK_TIERS: Record<RiskTier, TierPolicy> = {
|
|
86
|
+
low: {
|
|
87
|
+
humansRequired: 0,
|
|
88
|
+
meaning: "Checks are the review. Arm auto-merge and move on.",
|
|
89
|
+
},
|
|
90
|
+
medium: {
|
|
91
|
+
humansRequired: 1,
|
|
92
|
+
meaning: "One person reads the diff before it merges.",
|
|
93
|
+
},
|
|
94
|
+
high: {
|
|
95
|
+
humansRequired: 2,
|
|
96
|
+
meaning: "Two people read the diff. No waiver substitutes for either.",
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* CRC-E4's floor. Any changed path at or under one of these is HIGH, and no
|
|
102
|
+
* configuration can lower it.
|
|
103
|
+
*
|
|
104
|
+
* `.github/workflows/` is a GitHub fact, not a consumer-specific value, which
|
|
105
|
+
* is why it is built in rather than read from a project's config — a consumer
|
|
106
|
+
* that has never configured anything still gets the criterion.
|
|
107
|
+
*/
|
|
108
|
+
export const ALWAYS_HIGH_PREFIXES: readonly string[] = [".github/workflows"];
|
|
109
|
+
|
|
110
|
+
/** Below these a change may be LOW. They start strict — see CRC-E9 above. */
|
|
111
|
+
export const LOW_MAX_FILES = 5;
|
|
112
|
+
export const LOW_MAX_LINES = 150;
|
|
113
|
+
|
|
114
|
+
export type RiskReason =
|
|
115
|
+
/** CRC-E4, the built-in floor. */
|
|
116
|
+
| "CI_WORKFLOW_FILE"
|
|
117
|
+
/** CRC-E4, an owner-listed path. */
|
|
118
|
+
| "OWNER_SENSITIVE_PATH"
|
|
119
|
+
/** Too many files or lines to call small. */
|
|
120
|
+
| "SIZE"
|
|
121
|
+
/** The size of the change was not supplied, so it is not known to be small. */
|
|
122
|
+
| "SIZE_UNKNOWN"
|
|
123
|
+
/** Nothing raised it. */
|
|
124
|
+
| "NO_ESCALATION";
|
|
125
|
+
|
|
126
|
+
export type RiskRefusal =
|
|
127
|
+
/** The list could not be read at all — a caller passed null. */
|
|
128
|
+
| "CHANGED_FILES_UNREADABLE"
|
|
129
|
+
/** The list was read and is empty. A change touching nothing is not small. */
|
|
130
|
+
| "CHANGED_FILES_EMPTY"
|
|
131
|
+
/** At least one entry is not a repository path. */
|
|
132
|
+
| "CHANGED_FILE_UNREADABLE"
|
|
133
|
+
/**
|
|
134
|
+
* An owner CONFIGURED a sensitive path the classifier cannot read.
|
|
135
|
+
*
|
|
136
|
+
* This is the refusal it is most tempting to skip, and the most dangerous
|
|
137
|
+
* to. `underPrefix` returns false on a prefix it cannot read, so a
|
|
138
|
+
* configured `billing/**` or `src/auth/*` would match nothing, every path
|
|
139
|
+
* under it would read as not-sensitive, and the owner would see LOW on
|
|
140
|
+
* exactly the changes they wrote the setting to catch. The classifier
|
|
141
|
+
* being silent is what makes it worse than having no setting at all.
|
|
142
|
+
*/
|
|
143
|
+
| "SENSITIVE_PATH_UNREADABLE";
|
|
144
|
+
|
|
145
|
+
export interface ChangeUnderReview {
|
|
146
|
+
/**
|
|
147
|
+
* Every path the change touches. `null` means the list could not be read,
|
|
148
|
+
* which is UNMEASURED and never LOW.
|
|
149
|
+
*/
|
|
150
|
+
readonly changedFiles: readonly unknown[] | null;
|
|
151
|
+
/** Added plus deleted lines. Absent means unknown, which is never LOW. */
|
|
152
|
+
readonly linesChanged?: number;
|
|
153
|
+
/** Owner-listed sensitive paths. These ADD to the floor; none subtracts. */
|
|
154
|
+
readonly sensitivePaths?: readonly string[];
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export interface RiskReading {
|
|
158
|
+
/** `null` is UNMEASURED. It is not a tier and it is not LOW. */
|
|
159
|
+
readonly tier: RiskTier | null;
|
|
160
|
+
readonly humansRequired: number | null;
|
|
161
|
+
readonly reasons: readonly RiskReason[];
|
|
162
|
+
readonly refusals: readonly RiskRefusal[];
|
|
163
|
+
/** Human-readable lines naming what was read and what it implied. */
|
|
164
|
+
readonly detail: readonly string[];
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* CRC-E4, as ONE exported decision site.
|
|
169
|
+
*
|
|
170
|
+
* Every escalation to HIGH goes through this function, so a mutation of it is
|
|
171
|
+
* a mutation of the whole criterion and `test/risk-tier.test.ts` can prove the
|
|
172
|
+
* classifier is able to fail. A later branch that inlines its own prefix
|
|
173
|
+
* comparison bypasses that mutation and the criterion goes green having
|
|
174
|
+
* witnessed nothing — `.claude/rules/checks-must-be-able-to-fail.md`.
|
|
175
|
+
*
|
|
176
|
+
* Returns `null` when nothing about the path forces HIGH. An unreadable path
|
|
177
|
+
* also returns `null` here: it is not a REASON for HIGH, it is a refusal, and
|
|
178
|
+
* `classifyChange` handles it one layer up so the two states stay distinct.
|
|
179
|
+
*/
|
|
180
|
+
export function alwaysHighReason(
|
|
181
|
+
path: unknown,
|
|
182
|
+
sensitivePaths: readonly string[] = [],
|
|
183
|
+
): RiskReason | null {
|
|
184
|
+
for (const prefix of ALWAYS_HIGH_PREFIXES) {
|
|
185
|
+
if (underPrefix(path, prefix)) return "CI_WORKFLOW_FILE";
|
|
186
|
+
}
|
|
187
|
+
for (const prefix of sensitivePaths) {
|
|
188
|
+
if (underPrefix(path, prefix)) return "OWNER_SENSITIVE_PATH";
|
|
189
|
+
}
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Read a change and say which tier it is in, or refuse.
|
|
195
|
+
*
|
|
196
|
+
* The order is deliberate and is the same ordering lesson as the filing gate:
|
|
197
|
+
* **refusals are read before escalations are counted.** A list with an
|
|
198
|
+
* unreadable entry is UNMEASURED even if another entry already forced HIGH,
|
|
199
|
+
* because "HIGH, and also I could not read part of this" is a reading the
|
|
200
|
+
* caller must see rather than one it should be spared.
|
|
201
|
+
*/
|
|
202
|
+
export function classifyChange(change: ChangeUnderReview): RiskReading {
|
|
203
|
+
const detail: string[] = [];
|
|
204
|
+
const { changedFiles, linesChanged, sensitivePaths = [] } = change;
|
|
205
|
+
|
|
206
|
+
if (changedFiles === null) {
|
|
207
|
+
return unmeasured(["CHANGED_FILES_UNREADABLE"], [
|
|
208
|
+
"The changed-file list could not be read. A change whose extent is " +
|
|
209
|
+
"unknown is UNMEASURED, never low risk.",
|
|
210
|
+
]);
|
|
211
|
+
}
|
|
212
|
+
if (changedFiles.length === 0) {
|
|
213
|
+
return unmeasured(["CHANGED_FILES_EMPTY"], [
|
|
214
|
+
"The changed-file list was read and is empty. Either the change " +
|
|
215
|
+
"touches nothing, or the list was produced wrongly; those are " +
|
|
216
|
+
"different facts and neither is low risk.",
|
|
217
|
+
]);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// The CONFIGURED side is read before the change is. `underPrefix` refuses a
|
|
221
|
+
// prefix it cannot read by returning false, which at this call site would
|
|
222
|
+
// mean "not sensitive" — so the decision it defers to the caller is taken
|
|
223
|
+
// here, explicitly, and taken toward refusing.
|
|
224
|
+
const badPrefixes = sensitivePaths.filter(
|
|
225
|
+
s => normaliseRepoPath(typeof s === "string" ? s.replace(/\/+$/, "") : s) === null,
|
|
226
|
+
);
|
|
227
|
+
if (badPrefixes.length > 0) {
|
|
228
|
+
return unmeasured(["SENSITIVE_PATH_UNREADABLE"], [
|
|
229
|
+
`${badPrefixes.length} configured sensitive path(s) could not be read ` +
|
|
230
|
+
`as a repository path: ` +
|
|
231
|
+
`${badPrefixes.slice(0, 3).map(p => JSON.stringify(p)).join(", ")}. ` +
|
|
232
|
+
`Globs are not supported — write the directory. A prefix the ` +
|
|
233
|
+
`classifier cannot read is a prefix it cannot enforce, and matching ` +
|
|
234
|
+
`nothing would report LOW on exactly the paths the setting exists ` +
|
|
235
|
+
`to catch.`,
|
|
236
|
+
]);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const unreadable = changedFiles.filter(p => normaliseRepoPath(p) === null);
|
|
240
|
+
if (unreadable.length > 0) {
|
|
241
|
+
return unmeasured(["CHANGED_FILE_UNREADABLE"], [
|
|
242
|
+
`${unreadable.length} of ${changedFiles.length} changed-file entr(ies) ` +
|
|
243
|
+
`could not be read as a repository path: ` +
|
|
244
|
+
`${unreadable.slice(0, 3).map(p => JSON.stringify(p)).join(", ")}. ` +
|
|
245
|
+
`A path the classifier cannot read is a path it cannot compare ` +
|
|
246
|
+
`against a sensitive prefix.`,
|
|
247
|
+
]);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const reasons = new Set<RiskReason>();
|
|
251
|
+
for (const path of changedFiles) {
|
|
252
|
+
const reason = alwaysHighReason(path, sensitivePaths);
|
|
253
|
+
if (reason !== null) {
|
|
254
|
+
reasons.add(reason);
|
|
255
|
+
detail.push(
|
|
256
|
+
`${normaliseRepoPath(path)} forces HIGH (${reason}): ` +
|
|
257
|
+
`${RISK_TIERS.high.meaning}`,
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
if (reasons.size > 0) return at("high", [...reasons], detail);
|
|
262
|
+
|
|
263
|
+
if (typeof linesChanged !== "number" || !Number.isFinite(linesChanged)) {
|
|
264
|
+
detail.push(
|
|
265
|
+
"The size of this change was not supplied, so it is not known to be " +
|
|
266
|
+
"small. Unknown size is MEDIUM, not LOW.",
|
|
267
|
+
);
|
|
268
|
+
return at("medium", ["SIZE_UNKNOWN"], detail);
|
|
269
|
+
}
|
|
270
|
+
if (changedFiles.length > LOW_MAX_FILES || linesChanged > LOW_MAX_LINES) {
|
|
271
|
+
detail.push(
|
|
272
|
+
`${changedFiles.length} file(s) and ${linesChanged} line(s) changed, ` +
|
|
273
|
+
`over the LOW ceiling of ${LOW_MAX_FILES} file(s) and ` +
|
|
274
|
+
`${LOW_MAX_LINES} line(s).`,
|
|
275
|
+
);
|
|
276
|
+
return at("medium", ["SIZE"], detail);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
detail.push(
|
|
280
|
+
`${changedFiles.length} file(s) and ${linesChanged} line(s) changed, no ` +
|
|
281
|
+
`sensitive path touched. ${RISK_TIERS.low.meaning}`,
|
|
282
|
+
);
|
|
283
|
+
return at("low", ["NO_ESCALATION"], detail);
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
function at(
|
|
287
|
+
tier: RiskTier,
|
|
288
|
+
reasons: RiskReason[],
|
|
289
|
+
detail: string[],
|
|
290
|
+
): RiskReading {
|
|
291
|
+
return {
|
|
292
|
+
tier,
|
|
293
|
+
humansRequired: RISK_TIERS[tier].humansRequired,
|
|
294
|
+
reasons,
|
|
295
|
+
refusals: [],
|
|
296
|
+
detail,
|
|
297
|
+
};
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function unmeasured(refusals: RiskRefusal[], detail: string[]): RiskReading {
|
|
301
|
+
return { tier: null, humansRequired: null, reasons: [], refusals, detail };
|
|
302
|
+
}
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Rounds-to-pass, reported beside recall (CRC-M11).
|
|
3
|
+
*
|
|
4
|
+
* Covers CRC-M11 (specs/CODE-REVIEW-CONVERGENCE-SPEC.md). The audit entry it
|
|
5
|
+
* answers reads: *"The round number is computed per run and retained nowhere.
|
|
6
|
+
* Nothing tracks rounds-to-pass beside recall."* The first half stopped being
|
|
7
|
+
* true when the ledger landed — `readLedger` returns a `LedgerRound[]` and
|
|
8
|
+
* #1405 taught it to state how many rounds it evicted. This module is the
|
|
9
|
+
* second half: turning those per-pull-request histories into the three
|
|
10
|
+
* numbers the spec's target is written in.
|
|
11
|
+
*
|
|
12
|
+
* Those numbers, from the source the spec is drawn from:
|
|
13
|
+
*
|
|
14
|
+
* | | before | after |
|
|
15
|
+
* |---|---|---|
|
|
16
|
+
* | passed after one round | 65% | 78% |
|
|
17
|
+
* | five or more rounds | 8.9% | 1.5% |
|
|
18
|
+
* | **mean rounds to pass** | **2.08** | **1.32** |
|
|
19
|
+
*
|
|
20
|
+
* ## The reason this is not a one-line average
|
|
21
|
+
*
|
|
22
|
+
* **Averaging over pull requests that passed is survivorship, and it biases
|
|
23
|
+
* the number in the flattering direction.** Every pull request still open at
|
|
24
|
+
* round 6 is precisely the kind this metric exists to find, and it is exactly
|
|
25
|
+
* the kind an average over merged pull requests drops. A review process that
|
|
26
|
+
* got worse — more rounds, more abandonment — can therefore show a FALLING
|
|
27
|
+
* mean rounds-to-pass, because the slow ones leave the population instead of
|
|
28
|
+
* raising the number.
|
|
29
|
+
*
|
|
30
|
+
* So the mean is reported over the ones that passed, because that is what
|
|
31
|
+
* "rounds to pass" means, and it is reported **beside the count that did not
|
|
32
|
+
* pass and the rounds they have spent**. A mean with no `stillOpen` beside it
|
|
33
|
+
* is not this metric, and `summariseRoundsToPass` has no way to emit one.
|
|
34
|
+
*
|
|
35
|
+
* `fiveOrMoreRate` is deliberately computed over the WHOLE measured
|
|
36
|
+
* population, passed or not. It is the tail statistic, and a tail measured
|
|
37
|
+
* only on survivors has had its tail cut off.
|
|
38
|
+
*
|
|
39
|
+
* ## What is UNMEASURED here, and why truncation is the sharp one
|
|
40
|
+
*
|
|
41
|
+
* A ledger that cannot say how many rounds it dropped gives a round count
|
|
42
|
+
* that is a **lower bound, not a number**. Counting it at its visible length
|
|
43
|
+
* silently understates every long history — the same direction as the
|
|
44
|
+
* survivorship bias above, which is why it is refused rather than estimated.
|
|
45
|
+
* `AI_REVIEW_TRUNCATION_STATED` with a `dropped` count IS measurable:
|
|
46
|
+
* `rounds.length + dropped` is the total, and that is what #1405 was for.
|
|
47
|
+
*
|
|
48
|
+
* An absent ledger and an unreadable one are different facts and neither is
|
|
49
|
+
* "one round" — `.claude/rules/config-access.md`'s *Silent Default
|
|
50
|
+
* Substitution*, in a metric where the default is the most common value.
|
|
51
|
+
*
|
|
52
|
+
* A round whose entry recorded `findings: null` (`measured: false`) still
|
|
53
|
+
* COUNTS as a round. The reviewer ran; what is unreadable is what it found,
|
|
54
|
+
* which is a question about recall and not about how many times round the
|
|
55
|
+
* loop went.
|
|
56
|
+
*/
|
|
57
|
+
|
|
58
|
+
import {
|
|
59
|
+
AI_REVIEW_LEDGER_ABSENT,
|
|
60
|
+
AI_REVIEW_LEDGER_READ,
|
|
61
|
+
AI_REVIEW_LEDGER_UNREADABLE,
|
|
62
|
+
AI_REVIEW_TRUNCATION_STATED,
|
|
63
|
+
type LedgerReading,
|
|
64
|
+
} from "./ai-review-rounds";
|
|
65
|
+
|
|
66
|
+
/** The tail bucket the spec's target names. */
|
|
67
|
+
export const LONG_REVIEW_ROUNDS = 5;
|
|
68
|
+
|
|
69
|
+
export type RoundsRefusal =
|
|
70
|
+
/** No ledger comment on the pull request at all. */
|
|
71
|
+
| "LEDGER_ABSENT"
|
|
72
|
+
/** A ledger was found and could not be parsed. */
|
|
73
|
+
| "LEDGER_UNREADABLE"
|
|
74
|
+
/** The ledger cannot say how many rounds it evicted, so its length is a floor. */
|
|
75
|
+
| "ROUNDS_TRUNCATION_UNKNOWN"
|
|
76
|
+
/** The ledger parsed and holds no rounds, which is not a pull request reviewed once. */
|
|
77
|
+
| "ROUNDS_EMPTY"
|
|
78
|
+
/** A stated eviction count that is not a count. */
|
|
79
|
+
| "ROUNDS_DROPPED_UNREADABLE";
|
|
80
|
+
|
|
81
|
+
/** One pull request's history, as the caller read it. */
|
|
82
|
+
export interface PullRequestRounds {
|
|
83
|
+
readonly number: number;
|
|
84
|
+
readonly ledger: LedgerReading;
|
|
85
|
+
/**
|
|
86
|
+
* Did this pull request PASS — merge, or otherwise reach the state the
|
|
87
|
+
* rounds were spent getting to?
|
|
88
|
+
*
|
|
89
|
+
* Deliberately not `merged`: a consumer whose definition of passing is not
|
|
90
|
+
* "merged" should be able to say so without this module having an opinion
|
|
91
|
+
* about its workflow.
|
|
92
|
+
*/
|
|
93
|
+
readonly passed: boolean;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export interface UnmeasuredPullRequest {
|
|
97
|
+
readonly number: number;
|
|
98
|
+
readonly refusal: RoundsRefusal;
|
|
99
|
+
readonly detail: string;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export interface RoundsToPassReading {
|
|
103
|
+
/**
|
|
104
|
+
* Mean rounds taken by the pull requests that PASSED. `null` when none did,
|
|
105
|
+
* or none was measurable — never 0, which is a real and different value.
|
|
106
|
+
*/
|
|
107
|
+
readonly meanRoundsToPass: number | null;
|
|
108
|
+
/** Share of PASSED pull requests that took exactly one round. `null` if none. */
|
|
109
|
+
readonly passedAfterOneRoundRate: number | null;
|
|
110
|
+
/**
|
|
111
|
+
* Share of ALL measured pull requests at or over `LONG_REVIEW_ROUNDS`,
|
|
112
|
+
* passed or not. Over the whole population on purpose — see the docblock.
|
|
113
|
+
*/
|
|
114
|
+
readonly fiveOrMoreRate: number | null;
|
|
115
|
+
/** How many measured pull requests passed, and how many have not. */
|
|
116
|
+
readonly passedCount: number;
|
|
117
|
+
readonly stillOpenCount: number;
|
|
118
|
+
/** Rounds spent so far by the measured pull requests that have NOT passed. */
|
|
119
|
+
readonly stillOpenRounds: readonly number[];
|
|
120
|
+
/** Round count -> how many measured pull requests took it, passed or not. */
|
|
121
|
+
readonly distribution: ReadonlyMap<number, number>;
|
|
122
|
+
/** Every pull request that could not be counted, and why. */
|
|
123
|
+
readonly unmeasured: readonly UnmeasuredPullRequest[];
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* How many rounds this pull request has had, or a refusal.
|
|
128
|
+
*
|
|
129
|
+
* Exported because it is the single decision site this module's battery
|
|
130
|
+
* mutates: every count in the summary comes through here, so a mutation of it
|
|
131
|
+
* is a mutation of the whole metric. A later caller that counts
|
|
132
|
+
* `rounds.length` itself bypasses the truncation refusal and the measurement
|
|
133
|
+
* goes quietly optimistic — `.claude/rules/checks-must-be-able-to-fail.md`.
|
|
134
|
+
*/
|
|
135
|
+
export function roundsTaken(
|
|
136
|
+
pr: PullRequestRounds,
|
|
137
|
+
): { rounds: number } | { refusal: RoundsRefusal; detail: string } {
|
|
138
|
+
const { ledger } = pr;
|
|
139
|
+
|
|
140
|
+
if (ledger.status === AI_REVIEW_LEDGER_ABSENT) {
|
|
141
|
+
return {
|
|
142
|
+
refusal: "LEDGER_ABSENT",
|
|
143
|
+
detail:
|
|
144
|
+
"No ledger comment on this pull request. That is not a pull request " +
|
|
145
|
+
"reviewed once — it is one nothing recorded, and the two differ by " +
|
|
146
|
+
"exactly the value this metric is most often wrong by.",
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
if (ledger.status === AI_REVIEW_LEDGER_UNREADABLE) {
|
|
150
|
+
return {
|
|
151
|
+
refusal: "LEDGER_UNREADABLE",
|
|
152
|
+
detail: `The ledger was found and could not be read: ${ledger.reason}`,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
if (ledger.truncation.status !== AI_REVIEW_TRUNCATION_STATED) {
|
|
157
|
+
return {
|
|
158
|
+
refusal: "ROUNDS_TRUNCATION_UNKNOWN",
|
|
159
|
+
detail:
|
|
160
|
+
`The ledger holds ${ledger.rounds.length} round(s) and cannot say how ` +
|
|
161
|
+
`many it evicted (${ledger.truncation.reason}), so that figure is a ` +
|
|
162
|
+
`LOWER BOUND and not a count. Counting it would understate exactly ` +
|
|
163
|
+
`the long histories this metric exists to surface.`,
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
const { dropped } = ledger.truncation;
|
|
168
|
+
if (!Number.isInteger(dropped) || dropped < 0) {
|
|
169
|
+
return {
|
|
170
|
+
refusal: "ROUNDS_DROPPED_UNREADABLE",
|
|
171
|
+
detail:
|
|
172
|
+
`The ledger states an eviction count of ${JSON.stringify(dropped)}, ` +
|
|
173
|
+
`which is not a number of rounds.`,
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
const total = ledger.rounds.length + dropped;
|
|
178
|
+
if (total === 0) {
|
|
179
|
+
return {
|
|
180
|
+
refusal: "ROUNDS_EMPTY",
|
|
181
|
+
detail:
|
|
182
|
+
"The ledger parsed, states nothing was evicted, and holds no rounds. " +
|
|
183
|
+
"A pull request with no rounds has not passed review in one; it has " +
|
|
184
|
+
"not had one.",
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
return { rounds: total };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* The three numbers the spec's target is written in, plus what it took to
|
|
192
|
+
* refuse the rest.
|
|
193
|
+
*
|
|
194
|
+
* An empty or wholly-unmeasurable population returns nulls, not zeros. A mean
|
|
195
|
+
* of nothing is not 0, and a rate over nothing is not 0% — both of those read
|
|
196
|
+
* as "perfect" on the dashboard this feeds.
|
|
197
|
+
*/
|
|
198
|
+
export function summariseRoundsToPass(
|
|
199
|
+
prs: readonly PullRequestRounds[],
|
|
200
|
+
): RoundsToPassReading {
|
|
201
|
+
const unmeasured: UnmeasuredPullRequest[] = [];
|
|
202
|
+
const distribution = new Map<number, number>();
|
|
203
|
+
const passedRounds: number[] = [];
|
|
204
|
+
const stillOpenRounds: number[] = [];
|
|
205
|
+
|
|
206
|
+
for (const pr of prs) {
|
|
207
|
+
const taken = roundsTaken(pr);
|
|
208
|
+
if ("refusal" in taken) {
|
|
209
|
+
unmeasured.push({ number: pr.number, refusal: taken.refusal, detail: taken.detail });
|
|
210
|
+
continue;
|
|
211
|
+
}
|
|
212
|
+
distribution.set(taken.rounds, (distribution.get(taken.rounds) ?? 0) + 1);
|
|
213
|
+
(pr.passed ? passedRounds : stillOpenRounds).push(taken.rounds);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const measured = passedRounds.length + stillOpenRounds.length;
|
|
217
|
+
const mean =
|
|
218
|
+
passedRounds.length === 0
|
|
219
|
+
? null
|
|
220
|
+
: passedRounds.reduce((a, b) => a + b, 0) / passedRounds.length;
|
|
221
|
+
|
|
222
|
+
return {
|
|
223
|
+
meanRoundsToPass: mean,
|
|
224
|
+
passedAfterOneRoundRate:
|
|
225
|
+
passedRounds.length === 0
|
|
226
|
+
? null
|
|
227
|
+
: passedRounds.filter(r => r === 1).length / passedRounds.length,
|
|
228
|
+
// Whole population, passed or not. A tail measured on survivors only has
|
|
229
|
+
// had its tail cut off — the one line of this module worth re-reading.
|
|
230
|
+
fiveOrMoreRate:
|
|
231
|
+
measured === 0
|
|
232
|
+
? null
|
|
233
|
+
: [...passedRounds, ...stillOpenRounds].filter(r => r >= LONG_REVIEW_ROUNDS)
|
|
234
|
+
.length / measured,
|
|
235
|
+
passedCount: passedRounds.length,
|
|
236
|
+
stillOpenCount: stillOpenRounds.length,
|
|
237
|
+
stillOpenRounds,
|
|
238
|
+
distribution,
|
|
239
|
+
unmeasured,
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* The reading as lines, for the eval runner to print beside recall.
|
|
245
|
+
*
|
|
246
|
+
* The unmeasured count is on the FIRST line, not a footnote. A rounds-to-pass
|
|
247
|
+
* of 1.32 over four pull requests out of sixty is a different claim from the
|
|
248
|
+
* same number over sixty, and a reader who has to scroll to learn which will
|
|
249
|
+
* quote the one they saw first.
|
|
250
|
+
*/
|
|
251
|
+
export function formatRoundsToPass(reading: RoundsToPassReading): string[] {
|
|
252
|
+
const pct = (v: number | null) => (v === null ? "UNMEASURED" : `${(v * 100).toFixed(1)}%`);
|
|
253
|
+
const num = (v: number | null) => (v === null ? "UNMEASURED" : v.toFixed(2));
|
|
254
|
+
const measured = reading.passedCount + reading.stillOpenCount;
|
|
255
|
+
|
|
256
|
+
const lines = [
|
|
257
|
+
`rounds-to-pass: measured ${measured}, unmeasured ${reading.unmeasured.length}` +
|
|
258
|
+
` (passed ${reading.passedCount}, still open ${reading.stillOpenCount})`,
|
|
259
|
+
` mean rounds to pass ${num(reading.meanRoundsToPass)} (target 1.32, from 2.08)`,
|
|
260
|
+
` passed after 1 round ${pct(reading.passedAfterOneRoundRate)} (target 78%, from 65%)`,
|
|
261
|
+
` ${LONG_REVIEW_ROUNDS}+ rounds ${pct(reading.fiveOrMoreRate)} ` +
|
|
262
|
+
`(target 1.5%, from 8.9%; over ALL measured, not only those that passed)`,
|
|
263
|
+
];
|
|
264
|
+
if (reading.stillOpenCount > 0) {
|
|
265
|
+
lines.push(
|
|
266
|
+
` still open at rounds: ${[...reading.stillOpenRounds].sort((a, b) => b - a).join(", ")}` +
|
|
267
|
+
` — these are excluded from the mean BY DEFINITION and are why it flatters`,
|
|
268
|
+
);
|
|
269
|
+
}
|
|
270
|
+
for (const u of reading.unmeasured) {
|
|
271
|
+
lines.push(` #${u.number} UNMEASURED ${u.refusal}: ${u.detail}`);
|
|
272
|
+
}
|
|
273
|
+
return lines;
|
|
274
|
+
}
|