rungate 0.8.6 → 0.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -1
- package/lib/ai-review-rounds.ts +121 -1
- package/lib/ai-review-suppression.ts +80 -127
- package/lib/ai-review-threads.ts +296 -0
- package/lib/cli-commands.ts +21 -0
- package/lib/github.ts +25 -0
- package/lib/prose-self-distance.ts +414 -0
- package/lib/review-tool-agreement.ts +364 -15
- package/lib/scaffold/ai-review-findings.ts +167 -0
- package/lib/scaffold/ai-review-step.ts +454 -34
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +12 -17
- package/scripts/ai-review-threads.ts +211 -0
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/ai-review-threads.ts — posting a review's findings onto the diff.
|
|
3
|
+
*
|
|
4
|
+
* #1612. Measured 2026-10-25 across every merged pull request on the
|
|
5
|
+
* consumer that runs the generated `ai-review` job: zero review threads,
|
|
6
|
+
* ever. The job's only output has been a GitHub Actions annotation, which
|
|
7
|
+
* lives as long as the run log. A repository that switches
|
|
8
|
+
* `required_conversation_resolution` on against that gets a gate whose
|
|
9
|
+
* population is always empty — satisfied by construction.
|
|
10
|
+
*
|
|
11
|
+
* This module owns the half `reviewThreadRequests` does not: what happened
|
|
12
|
+
* when the comments were sent. That matters because **zero comments posted
|
|
13
|
+
* is the same observation in four different situations** — the review found
|
|
14
|
+
* nothing, the review could not run, the token could not write, or this code
|
|
15
|
+
* is broken. One channel, four mechanisms, which is the sixth shape in
|
|
16
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`. Collapsing them into a
|
|
17
|
+
* boolean is how a producer stays dead without anybody noticing, which is
|
|
18
|
+
* precisely the history being fixed here.
|
|
19
|
+
*
|
|
20
|
+
* So the outcome is NAMED, and exactly one of the names is a refusal.
|
|
21
|
+
*
|
|
22
|
+
* The HTTP call is injected rather than performed. Every decision below is
|
|
23
|
+
* therefore reachable from a test over a recorded response, including the
|
|
24
|
+
* two that only a live fork or a line outside a hunk would otherwise produce.
|
|
25
|
+
*/
|
|
26
|
+
import type { ThreadReading } from "./ai-review-rounds";
|
|
27
|
+
import { reviewThreadRequests, threadKeyFrom } from "./scaffold/ai-review-findings";
|
|
28
|
+
import type { AiReviewFindingsReading, AiReviewThreadRequest } from "./scaffold/ai-review-findings";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* One review thread as the GraphQL query returns it, before anything is
|
|
32
|
+
* trusted about it. Both fields are `unknown`: a partial response drops
|
|
33
|
+
* fields rather than failing, so the shape is asserted here and nowhere else.
|
|
34
|
+
*/
|
|
35
|
+
export interface RawReviewThread {
|
|
36
|
+
isResolved: unknown;
|
|
37
|
+
/** The body of the thread's FIRST comment — the one the poster wrote. */
|
|
38
|
+
body: unknown;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Turn what GitHub returned into the reading `carryForwardFromThreads` takes.
|
|
43
|
+
*
|
|
44
|
+
* @param raw the threads, or `null`/`undefined` when the list could not be
|
|
45
|
+
* read at all — a network failure, a missing token, a GraphQL error.
|
|
46
|
+
* @param reason what to call the unreadable case. Only used when `raw` is
|
|
47
|
+
* absent, and it is a parameter because the caller is the only thing that
|
|
48
|
+
* knows which way the fetch failed.
|
|
49
|
+
*
|
|
50
|
+
* ## An unmarked thread is DROPPED, not keyed to the empty string
|
|
51
|
+
*
|
|
52
|
+
* Only a thread carrying the #1618 marker names a finding. Every thread
|
|
53
|
+
* posted before #1618 and every thread a person opened carries none, and
|
|
54
|
+
* they are real threads that are not evidence about any finding. Collapsing
|
|
55
|
+
* them to `""` would make one resolved human comment answer every unkeyable
|
|
56
|
+
* finding on the pull request at once.
|
|
57
|
+
*
|
|
58
|
+
* ## Only the literal `true` is resolution
|
|
59
|
+
*
|
|
60
|
+
* A partial response drops fields rather than failing, and a finding answered
|
|
61
|
+
* by an omission is the silent direction of error. Anything that is not
|
|
62
|
+
* exactly `true` reads as open.
|
|
63
|
+
*/
|
|
64
|
+
export function threadReading(
|
|
65
|
+
raw: readonly RawReviewThread[] | null | undefined,
|
|
66
|
+
reason: string,
|
|
67
|
+
): ThreadReading {
|
|
68
|
+
if (raw === null || raw === undefined) return { status: "UNMEASURED", reason };
|
|
69
|
+
const threads: { key: string; isResolved: boolean }[] = [];
|
|
70
|
+
for (const t of raw) {
|
|
71
|
+
// A NULL NODE IS A THREAD, NOT A CRASH. A GraphQL connection returns a
|
|
72
|
+
// null node for an entry the token cannot see — a thread on a deleted
|
|
73
|
+
// file, or one hidden by a permission — and `t.body` on it throws a
|
|
74
|
+
// TypeError straight past this function's callers. That is the one
|
|
75
|
+
// failure mode this module is built to not have: an undecidable thread
|
|
76
|
+
// is dropped, which carries its finding, and the whole point is that the
|
|
77
|
+
// reading degrades rather than exploding. Dropping is also right on the
|
|
78
|
+
// merits — a node nobody can read names no finding and answers none.
|
|
79
|
+
if (t === null || typeof t !== "object") continue;
|
|
80
|
+
const key = threadKeyFrom(t.body);
|
|
81
|
+
if (key === null) continue;
|
|
82
|
+
threads.push({ key, isResolved: t.isResolved === true });
|
|
83
|
+
}
|
|
84
|
+
return { status: "MEASURED", threads };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* The three ways the file on the runner can fail to be a thread list.
|
|
89
|
+
*
|
|
90
|
+
* One sentence per FACT, and no two facts share one. "Nobody fetched it",
|
|
91
|
+
* "the fetch was cut off mid-write" and "the fetch returned an error object
|
|
92
|
+
* instead of threads" are three different things to go and look at, and a
|
|
93
|
+
* reader that prints the same sentence for all three has told the next person
|
|
94
|
+
* only that something went wrong. Exported so a case can assert on the
|
|
95
|
+
* sentence rather than on a substring of it — a `toContain` goes green on a
|
|
96
|
+
* message that was narrowed to say less.
|
|
97
|
+
*/
|
|
98
|
+
export const THREADS_NOT_READ =
|
|
99
|
+
"the pull request's review threads were not read, so whether a finding was already raised is unknown";
|
|
100
|
+
export const THREADS_UNPARSEABLE =
|
|
101
|
+
"the review-thread list was written but does not parse, so it was cut off or is not the list it claims to be";
|
|
102
|
+
export const THREADS_NOT_A_LIST =
|
|
103
|
+
"the review-thread list parsed and is not a list, which is the shape a GraphQL error response has";
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* The same reading, taken from the bytes a bash step left on the runner.
|
|
107
|
+
*
|
|
108
|
+
* `threadReading` takes parsed nodes because that is the shape a test can
|
|
109
|
+
* drive; this is the production path, and it exists so that the JSON parse —
|
|
110
|
+
* the one step between GitHub's answer and the decision — is somewhere a
|
|
111
|
+
* negative control can reach. Every failure produces a NAMED sentence and an
|
|
112
|
+
* UNMEASURED reading, never an empty list: an empty list is the assertion
|
|
113
|
+
* "this pull request has no threads", which suppresses nothing while claiming
|
|
114
|
+
* to have looked, and is exactly how a repeat-suppressor goes quietly dead.
|
|
115
|
+
*
|
|
116
|
+
* An empty STRING is absence, not a parse failure. A step that wrote no bytes
|
|
117
|
+
* and a step that wrote nothing are the same event seen twice.
|
|
118
|
+
*/
|
|
119
|
+
export function threadReadingFromJson(text: string | null | undefined): ThreadReading {
|
|
120
|
+
if (text === null || text === undefined || text === "") {
|
|
121
|
+
return { status: "UNMEASURED", reason: THREADS_NOT_READ };
|
|
122
|
+
}
|
|
123
|
+
let parsed: unknown;
|
|
124
|
+
try {
|
|
125
|
+
parsed = JSON.parse(text);
|
|
126
|
+
} catch {
|
|
127
|
+
return { status: "UNMEASURED", reason: THREADS_UNPARSEABLE };
|
|
128
|
+
}
|
|
129
|
+
if (!Array.isArray(parsed)) return { status: "UNMEASURED", reason: THREADS_NOT_A_LIST };
|
|
130
|
+
// The per-node decisions stay in ONE place. A second loop here would be a
|
|
131
|
+
// second decider over "is this thread usable", and a mutation of either
|
|
132
|
+
// would leave the other holding every case up.
|
|
133
|
+
return threadReading(parsed as readonly RawReviewThread[], THREADS_NOT_READ);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The single non-zero exit. Declared once, so a mutant can set it to `0` and
|
|
138
|
+
* a case can observe the refusal disappearing — `.claude/rules/checks-must-be-able-to-fail.md`,
|
|
139
|
+
* *For a script whose job is to exit non-zero*. A `process.exit(1)` or a
|
|
140
|
+
* second literal anywhere in this module would bypass that mutation and the
|
|
141
|
+
* case would go green over a refusal it never witnessed.
|
|
142
|
+
*/
|
|
143
|
+
export const AI_REVIEW_THREADS_REFUSAL = 1;
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* What a posting round was, in the words that distinguish the four ways it
|
|
147
|
+
* can produce no comments.
|
|
148
|
+
*
|
|
149
|
+
* | Outcome | Comments | Refusal | What it means |
|
|
150
|
+
* |---|---|---|---|
|
|
151
|
+
* | `POSTED` | ≥ 1 | no | the producer works |
|
|
152
|
+
* | `NOTHING_TO_POST` | 0 | no | the review located nothing to anchor |
|
|
153
|
+
* | `UNMEASURED` | 0 | no | there was no reading to post FROM |
|
|
154
|
+
* | `DEGRADED_NO_WRITE` | 0 | no | the token cannot write; see below |
|
|
155
|
+
* | `FAILED` | any | **yes** | the producer is broken |
|
|
156
|
+
*
|
|
157
|
+
* `UNMEASURED` is not a quieter `NOTHING_TO_POST`. A review that did not run
|
|
158
|
+
* has not found zero findings, and a tree that cannot tell those apart is the
|
|
159
|
+
* *Silent Default Substitution* anti-pattern in `.claude/rules/config-access.md`.
|
|
160
|
+
*/
|
|
161
|
+
export type AiReviewThreadOutcome =
|
|
162
|
+
| "POSTED"
|
|
163
|
+
| "NOTHING_TO_POST"
|
|
164
|
+
| "UNMEASURED"
|
|
165
|
+
| "DEGRADED_NO_WRITE"
|
|
166
|
+
| "FAILED";
|
|
167
|
+
|
|
168
|
+
/** What the API said. `status` is `null` for a failure that never got one. */
|
|
169
|
+
export type AiReviewThreadResponse =
|
|
170
|
+
| { ok: true }
|
|
171
|
+
| { ok: false; status: number | null; message: string };
|
|
172
|
+
|
|
173
|
+
/** Sends one comment. The only thing in this module that touches a network. */
|
|
174
|
+
export type AiReviewThreadPoster = (
|
|
175
|
+
request: AiReviewThreadRequest,
|
|
176
|
+
) => Promise<AiReviewThreadResponse>;
|
|
177
|
+
|
|
178
|
+
export interface AiReviewThreadsResult {
|
|
179
|
+
outcome: AiReviewThreadOutcome;
|
|
180
|
+
/** Comments this round meant to create. */
|
|
181
|
+
attempted: number;
|
|
182
|
+
/** Comments the API accepted. */
|
|
183
|
+
posted: number;
|
|
184
|
+
/** Of those, how many only succeeded after dropping to file level. */
|
|
185
|
+
demoted: number;
|
|
186
|
+
/** One sentence per thing worth reading, in the order it happened. */
|
|
187
|
+
detail: string[];
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** HTTP 422 — the documented answer to a `line` outside a diff hunk. */
|
|
191
|
+
const UNPROCESSABLE = 422;
|
|
192
|
+
/** HTTP 403 — the documented answer to a token without `pull-requests: write`. */
|
|
193
|
+
const FORBIDDEN = 403;
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Post every comment a reading calls for, and say which of the four
|
|
197
|
+
* no-comment states this was if none appeared.
|
|
198
|
+
*
|
|
199
|
+
* Three behaviours are worth stating because each has a cheaper wrong answer:
|
|
200
|
+
*
|
|
201
|
+
* **An UNMEASURED reading posts nothing and attempts nothing.** Not because
|
|
202
|
+
* posting would fail, but because there is no reading to post from, and a
|
|
203
|
+
* round that posted a guess would put the reviewer's absence on the diff as
|
|
204
|
+
* though it were a finding.
|
|
205
|
+
*
|
|
206
|
+
* **A 422 demotes before it fails.** `line` is documented as "Required
|
|
207
|
+
* unless using subject_type:file", and a line outside a hunk is refused with
|
|
208
|
+
* 422. Being in the changed-file set locates a finding to a FILE and never
|
|
209
|
+
* to a hunk, so this response is EXPECTED on a correct request and the
|
|
210
|
+
* retry is part of the contract rather than error recovery. Only a 422 that
|
|
211
|
+
* survives the demotion is a defect.
|
|
212
|
+
*
|
|
213
|
+
* **A 403 degrades and does not refuse — but only when it is the whole
|
|
214
|
+
* story.** For a `pull_request` from a forked repository GitHub downgrades
|
|
215
|
+
* write permissions to read regardless of what the workflow declares, unless
|
|
216
|
+
* an admin enabled *Send write tokens to workflows from pull requests*. So on
|
|
217
|
+
* a fork this producer cannot run, the adjudicator's annotation is still
|
|
218
|
+
* produced, and failing here would red every fork pull request for something
|
|
219
|
+
* the contributor cannot fix. What that must NOT do is become a blanket
|
|
220
|
+
* excuse: a round that saw a 403 AND some other failure is `FAILED`, because
|
|
221
|
+
* a broken producer hiding behind a fork is the fail-open this whole module
|
|
222
|
+
* exists to prevent.
|
|
223
|
+
*/
|
|
224
|
+
export async function postReviewThreads(
|
|
225
|
+
reading: AiReviewFindingsReading,
|
|
226
|
+
post: AiReviewThreadPoster,
|
|
227
|
+
keyFor?: (finding: Record<string, unknown>) => string | null,
|
|
228
|
+
): Promise<AiReviewThreadsResult> {
|
|
229
|
+
const detail: string[] = [];
|
|
230
|
+
|
|
231
|
+
if (reading.outcome === "UNMEASURED") {
|
|
232
|
+
detail.push(
|
|
233
|
+
"the review produced no usable reading, so there was nothing to post — this is not a review that found nothing",
|
|
234
|
+
);
|
|
235
|
+
return { outcome: "UNMEASURED", attempted: 0, posted: 0, demoted: 0, detail };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const requests = reviewThreadRequests(reading, keyFor);
|
|
239
|
+
if (requests.length === 0) {
|
|
240
|
+
detail.push("the reading located no finding that could be anchored to a path in this diff");
|
|
241
|
+
return { outcome: "NOTHING_TO_POST", attempted: 0, posted: 0, demoted: 0, detail };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
let posted = 0;
|
|
245
|
+
let demoted = 0;
|
|
246
|
+
let forbidden = 0;
|
|
247
|
+
let failed = 0;
|
|
248
|
+
|
|
249
|
+
for (const request of requests) {
|
|
250
|
+
const where = request.line === null ? request.path : `${request.path}:${request.line}`;
|
|
251
|
+
let answer = await post(request);
|
|
252
|
+
|
|
253
|
+
// The demotion, and only from a line. A file-level request that is
|
|
254
|
+
// refused has nothing left to drop.
|
|
255
|
+
if (!answer.ok && answer.status === UNPROCESSABLE && request.line !== null) {
|
|
256
|
+
detail.push(`${where}: line refused (422), retrying as a file-level comment`);
|
|
257
|
+
answer = await post({ ...request, line: null });
|
|
258
|
+
if (answer.ok) demoted += 1;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
if (answer.ok) {
|
|
262
|
+
posted += 1;
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
if (answer.status === FORBIDDEN) {
|
|
266
|
+
forbidden += 1;
|
|
267
|
+
detail.push(
|
|
268
|
+
`${where}: 403 — this token cannot write pull-request comments. A pull_request run from a ` +
|
|
269
|
+
`fork is handed a read-only token whatever the workflow declares.`,
|
|
270
|
+
);
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
failed += 1;
|
|
274
|
+
detail.push(`${where}: ${answer.status ?? "no status"} — ${answer.message}`);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// Order matters. A real failure outranks the fork excuse, and both outrank
|
|
278
|
+
// a partial success — a round that posted three of four comments has lost
|
|
279
|
+
// a finding and must say so.
|
|
280
|
+
if (failed > 0) {
|
|
281
|
+
return { outcome: "FAILED", attempted: requests.length, posted, demoted, detail };
|
|
282
|
+
}
|
|
283
|
+
if (forbidden > 0) {
|
|
284
|
+
return { outcome: "DEGRADED_NO_WRITE", attempted: requests.length, posted, demoted, detail };
|
|
285
|
+
}
|
|
286
|
+
detail.push(
|
|
287
|
+
`posted ${posted} review ${posted === 1 ? "comment" : "comments"}` +
|
|
288
|
+
(demoted === 0 ? "" : `, ${demoted} at file level after a line was refused`),
|
|
289
|
+
);
|
|
290
|
+
return { outcome: "POSTED", attempted: requests.length, posted, demoted, detail };
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/** The exit code a round's outcome earns. Only `FAILED` refuses. */
|
|
294
|
+
export function aiReviewThreadsExitCode(result: AiReviewThreadsResult): number {
|
|
295
|
+
return result.outcome === "FAILED" ? AI_REVIEW_THREADS_REFUSAL : 0;
|
|
296
|
+
}
|
package/lib/cli-commands.ts
CHANGED
|
@@ -166,6 +166,27 @@ export const CLI_COMMANDS: Record<string, CliCommand> = {
|
|
|
166
166
|
// project tree at all — it talks to the GitHub API about a repository it
|
|
167
167
|
// is told the name of, so an appended `--project-root <cwd>` would be
|
|
168
168
|
// refused by `parseArgs` as an unknown argument.
|
|
169
|
+
// Registered for the same reachability reason as the two around it
|
|
170
|
+
// (#1612): the generated `ai-review` job runs in a consumer's repository,
|
|
171
|
+
// which holds no copy of rungate's sources at any path a workflow could
|
|
172
|
+
// name. This registry is that path.
|
|
173
|
+
//
|
|
174
|
+
// SEPARATE from `ai-review-findings` on purpose. That command exits
|
|
175
|
+
// non-zero on a blocking finding — so a poster sharing its entry point
|
|
176
|
+
// could never run on the review that actually found something, which is
|
|
177
|
+
// the only review worth posting.
|
|
178
|
+
//
|
|
179
|
+
// No `projectRootFlag`: the payload lives outside the checkout in
|
|
180
|
+
// `RUNNER_TEMP`, everything else is a flag, and an appended
|
|
181
|
+
// `--project-root <cwd>` would arrive as a second positional, which
|
|
182
|
+
// `parse` refuses by name rather than silently reading as the payload.
|
|
183
|
+
"ai-review-threads": {
|
|
184
|
+
script: "scripts/ai-review-threads.ts",
|
|
185
|
+
summary: "Post an AI review's findings onto the pull request's diff as review threads",
|
|
186
|
+
usage:
|
|
187
|
+
"rungate ai-review-threads <path-to-findings.json> --repo <owner/name> " +
|
|
188
|
+
"--pr <n> --commit <sha> [--changed-files <path>]",
|
|
189
|
+
},
|
|
169
190
|
"record-ai-review-unmeasured": {
|
|
170
191
|
script: "scripts/record-ai-review-unmeasured.ts",
|
|
171
192
|
summary: "Record WHICH kind of unmeasured each unmeasured AI-review round was",
|
package/lib/github.ts
CHANGED
|
@@ -42,6 +42,31 @@ export interface GitHubClient {
|
|
|
42
42
|
get: (params: { owner: string; repo: string; pull_number: number }) => Promise<{ data: any }>;
|
|
43
43
|
update: (params: { owner: string; repo: string; pull_number: number; [key: string]: any }) => Promise<{ data: any }>;
|
|
44
44
|
list: (params: { owner: string; repo: string; [key: string]: any }) => Promise<{ data: any[] }>;
|
|
45
|
+
/**
|
|
46
|
+
* One INLINE review comment, anchored to the diff (#1612).
|
|
47
|
+
*
|
|
48
|
+
* Not `issues.createComment`, which is the other endpoint and the one
|
|
49
|
+
* the findings ledger uses. The difference is not cosmetic: only a
|
|
50
|
+
* comment created here opens a review THREAD, and a review thread is
|
|
51
|
+
* the only thing `required_conversation_resolution` can read. A
|
|
52
|
+
* repository that turns that protection on while its reviewer posts
|
|
53
|
+
* issue comments has a required gate over an always-empty population.
|
|
54
|
+
*
|
|
55
|
+
* `line` is omitted and `subject_type: "file"` sent in its place when
|
|
56
|
+
* the finding could not be anchored to a line — the API documents
|
|
57
|
+
* `line` as "Required unless using subject_type:file", and answers 422
|
|
58
|
+
* for a line outside a diff hunk.
|
|
59
|
+
*/
|
|
60
|
+
createReviewComment: (params: {
|
|
61
|
+
owner: string;
|
|
62
|
+
repo: string;
|
|
63
|
+
pull_number: number;
|
|
64
|
+
body: string;
|
|
65
|
+
commit_id: string;
|
|
66
|
+
path: string;
|
|
67
|
+
line?: number;
|
|
68
|
+
subject_type?: "line" | "file";
|
|
69
|
+
}) => Promise<{ data: any }>;
|
|
45
70
|
};
|
|
46
71
|
repos: {
|
|
47
72
|
/** Repository metadata. `owner.type` and `default_branch` are what PROTECT-1 reads (#909). */
|