wachiman 1.12.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -101
- package/package.json +1 -1
- package/skills/wachiman-loop/SKILL.md +91 -0
- package/src/call-message.ts +26 -3
- package/src/cli.ts +81 -5
- package/src/config.ts +19 -0
- package/src/connectors/local-git.ts +101 -13
- package/src/cost.ts +20 -0
- package/src/doctor/checks.ts +1 -1
- package/src/engine.ts +532 -55
- package/src/estimate.ts +64 -0
- package/src/fidelity/block.ts +263 -0
- package/src/fidelity/claims-call.ts +172 -0
- package/src/fidelity/claims.ts +69 -0
- package/src/fidelity/digest.ts +80 -0
- package/src/fidelity/grade.ts +242 -0
- package/src/fidelity/inert.ts +25 -0
- package/src/fidelity/rules.ts +166 -0
- package/src/fidelity/sink.ts +135 -0
- package/src/fidelity/summary.ts +16 -0
- package/src/fidelity/types.ts +93 -0
- package/src/formats.ts +16 -3
- package/src/history/fingerprint.ts +1 -1
- package/src/history/store.ts +39 -1
- package/src/init/options.ts +1 -1
- package/src/local.ts +20 -2
- package/src/loop/command.ts +441 -0
- package/src/loop/state.ts +88 -0
- package/src/loop/stop-rule.ts +98 -0
- package/src/providers/claude-code.ts +47 -12
- package/src/providers/codex.ts +3 -2
- package/src/providers/index.ts +5 -2
- package/src/providers/types.ts +10 -0
- package/src/sinks/json.ts +13 -2
- package/src/skills/install.ts +4 -4
- package/src/store/command.ts +109 -0
- package/src/store/figures.ts +153 -0
- package/src/store/html.ts +138 -0
- package/src/store/index.ts +298 -0
- package/templates/caller-workflow.yml +45 -15
- package/templates/wachiman-review-caller.yml +8 -0
package/src/engine.ts
CHANGED
|
@@ -27,11 +27,12 @@
|
|
|
27
27
|
* best-effort, not a sandbox: it relies on the consumer repository's trust assumption (only trusted people push).
|
|
28
28
|
* The command-line entry point is `src/cli.ts`.
|
|
29
29
|
*/
|
|
30
|
-
import { NO_USAGE, addUsage, costLine, costLogLine, type CallCost, type Operation, type TokenUsage } from "./cost";
|
|
30
|
+
import { ESTIMATE_NOTE, NO_USAGE, addUsage, costLine, costLogLine, isEstimate, type CallCost, type Operation, type TokenUsage } from "./cost";
|
|
31
31
|
import { spawnSync } from "node:child_process";
|
|
32
32
|
import { closeSync, mkdtempSync, openSync, readFileSync, readSync, realpathSync, rmSync, writeFileSync } from "node:fs";
|
|
33
33
|
import { tmpdir } from "node:os";
|
|
34
34
|
import { join } from "node:path";
|
|
35
|
+
import { estimateReview, sizeGate, suggestedBudget, type ReviewEstimate } from "./estimate";
|
|
35
36
|
import { isAutomatic, operatorProblem, type EngineConfig, type Trigger } from "./config";
|
|
36
37
|
import {
|
|
37
38
|
EVIDENCE_CAPS,
|
|
@@ -72,6 +73,24 @@ import {
|
|
|
72
73
|
type VerifierVerdict,
|
|
73
74
|
} from "./verify";
|
|
74
75
|
import { validate as validateSchema } from "./json-schema";
|
|
76
|
+
import {
|
|
77
|
+
ASSIGNMENTS_SCHEMA,
|
|
78
|
+
CLAIMS_BUDGET_USD,
|
|
79
|
+
CLAIMS_SCHEMA,
|
|
80
|
+
assembleClaims,
|
|
81
|
+
buildClaimsPrompt,
|
|
82
|
+
claimsParagraph,
|
|
83
|
+
claimsTextTruncated,
|
|
84
|
+
parseClaimList,
|
|
85
|
+
type ClaimAssignment,
|
|
86
|
+
type ProposedClaim,
|
|
87
|
+
} from "./fidelity/claims-call";
|
|
88
|
+
import { checkClaims } from "./fidelity/claims";
|
|
89
|
+
import { claimsDigest, claimsInput, stripBlock } from "./fidelity/digest";
|
|
90
|
+
import { buildFidelity } from "./fidelity/grade";
|
|
91
|
+
import { fidelitySummary } from "./fidelity/summary";
|
|
92
|
+
import { publishConfidence } from "./fidelity/sink";
|
|
93
|
+
import type { Fidelity } from "./fidelity/types";
|
|
75
94
|
import type { ProviderErrorClass } from "./providers/errors";
|
|
76
95
|
import type { Effort, ProviderAdapter } from "./providers/types";
|
|
77
96
|
import { renderDataPathQuestions, selectQuestions, type DataPathQuestion } from "./questions";
|
|
@@ -197,6 +216,8 @@ export interface ReviewOutput {
|
|
|
197
216
|
risk_facts?: RiskFact[];
|
|
198
217
|
/** The batch's judgement items (ADR 0031). Absent or empty when none apply. */
|
|
199
218
|
judgement?: JudgementItem[];
|
|
219
|
+
/** The batch's claim assignments (ADR 0033), asked for only when PR Confidence is on; checked by `assembleClaims`. */
|
|
220
|
+
claim_assignments?: ClaimAssignment[];
|
|
200
221
|
thread_replies: { comment_id: number; status: "addressed" | "still_open"; reply: string }[];
|
|
201
222
|
summary: string;
|
|
202
223
|
}
|
|
@@ -348,7 +369,7 @@ export const isBinaryAtHead = (headDir: string, filename: string) => headContent
|
|
|
348
369
|
/**
|
|
349
370
|
* Splits the kept files into those a model can review and those it cannot. The engine decides it
|
|
350
371
|
* from the file, never the model: a binary the model skipped as "binary" would leave the review
|
|
351
|
-
* partial, and a partial review
|
|
372
|
+
* partial, and a partial review has to carry the file to the next push as an unread one.
|
|
352
373
|
* A file with a patch is text by definition (GitHub sends none for a binary); one without is read
|
|
353
374
|
* at the head. One that is not there any more is a binary only when GitHub counted no lines for it
|
|
354
375
|
* either (`additions` and `deletions` both 0): a removed text file too large for a patch still has
|
|
@@ -662,6 +683,54 @@ export function annotateDiff(files: ReviewFile[]) {
|
|
|
662
683
|
|
|
663
684
|
export const marker = (sha: string) => `<!-- agent-review sha=${sha} -->`;
|
|
664
685
|
|
|
686
|
+
/**
|
|
687
|
+
* What the review at `sha` left for the next run (ADR 0036). A complete review leaves nothing
|
|
688
|
+
* (`rest` is empty). A partial, unscoped one lists the files it did not review (`rest`) and which
|
|
689
|
+
* of those a model was already sent once and did not review (`tried`, a subset of `rest`).
|
|
690
|
+
*/
|
|
691
|
+
export interface ReviewState {
|
|
692
|
+
sha: string;
|
|
693
|
+
rest: string[];
|
|
694
|
+
tried: string[];
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
const PARTIAL_SOURCE = "<!-- wachi:partial sha=([A-Za-z0-9]+) state=([A-Za-z0-9_-]+) -->";
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* The most characters the partial marker may take. The marker sits in the footer, which the
|
|
701
|
+
* fitting to `MAX_BODY_CHARS` never cuts, so it is capped at a sixth of the body: about 110 files
|
|
702
|
+
* of 65 characters. A longer list writes no partial marker (`partialMarker` returns null).
|
|
703
|
+
*/
|
|
704
|
+
export const MAX_PARTIAL_MARKER_CHARS = 10_000;
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* The marker a partial review writes: `<!-- wachi:partial sha=… state=… -->`. The state is the
|
|
708
|
+
* JSON `{"r": [files], "t": [indexes into r]}` as base64url, so no file name can end the comment
|
|
709
|
+
* (the alphabet has no `>`, and the comment's text is never read back as text) and any name GitHub
|
|
710
|
+
* allows survives (spaces, unicode, `-->`). Null when the marker would pass `MAX_PARTIAL_MARKER_CHARS`.
|
|
711
|
+
*/
|
|
712
|
+
export function partialMarker(state: ReviewState): string | null {
|
|
713
|
+
const index = new Map(state.rest.map((f, i) => [f, i]));
|
|
714
|
+
const tried = state.tried.flatMap((f) => (index.has(f) ? [index.get(f) as number] : []));
|
|
715
|
+
const encoded = Buffer.from(JSON.stringify({ r: state.rest, t: tried }), "utf8").toString("base64url");
|
|
716
|
+
const text = `<!-- wachi:partial sha=${state.sha} state=${encoded} -->`;
|
|
717
|
+
return text.length > MAX_PARTIAL_MARKER_CHARS ? null : text;
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/** The files and indexes inside a partial marker's state, or null when it is not what `partialMarker` writes. */
|
|
721
|
+
function decodePartialState(sha: string, encoded: string): ReviewState | null {
|
|
722
|
+
try {
|
|
723
|
+
const data = JSON.parse(Buffer.from(encoded, "base64url").toString("utf8")) as { r?: unknown; t?: unknown };
|
|
724
|
+
if (!Array.isArray(data.r) || !Array.isArray(data.t)) return null;
|
|
725
|
+
if (!data.r.every((f) => typeof f === "string")) return null;
|
|
726
|
+
const rest = data.r as string[];
|
|
727
|
+
if (!data.t.every((i) => Number.isInteger(i) && (i as number) >= 0 && (i as number) < rest.length)) return null;
|
|
728
|
+
return { sha, rest, tried: (data.t as number[]).map((i) => rest[i]) };
|
|
729
|
+
} catch {
|
|
730
|
+
return null;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
665
734
|
/**
|
|
666
735
|
* Hidden signatures the engine writes outside any model text. With `neutralise` on all model text,
|
|
667
736
|
* the model cannot produce one, so a signed item was written by this engine and not by another
|
|
@@ -709,12 +778,13 @@ export const neutralise = (text: string) =>
|
|
|
709
778
|
* <!-- agent-review sha=… -->
|
|
710
779
|
*
|
|
711
780
|
* and the marker is read only as the last line, right after the signature (`postReview` then adds
|
|
712
|
-
* its `<!-- wachi:attempt id=… -->` line after it, which is allowed).
|
|
781
|
+
* its `<!-- wachi:attempt id=… -->` line after it, which is allowed). A partial, unscoped review
|
|
782
|
+
* writes `<!-- wachi:partial sha=… state=… -->` in the marker's place (see `partialMarker`). Anything else
|
|
713
783
|
* (a marker in the middle of the body, in a file name, in a list) is not the engine's footer, even
|
|
714
784
|
* if some string slipped past `neutralise`.
|
|
715
785
|
*/
|
|
716
786
|
const SIGNED_FOOTER = new RegExp(
|
|
717
|
-
`\\n\\n${SIGNATURE_REVIEW_SOURCE}\\n\\n
|
|
787
|
+
`\\n\\n${SIGNATURE_REVIEW_SOURCE}\\n\\n(?:${MARKER_SOURCE}|${PARTIAL_SOURCE})(?:\\n\\n${ATTEMPT_SOURCE})?$`,
|
|
718
788
|
);
|
|
719
789
|
/** A review written before signatures existed ended with `</sub>`, a blank line and the marker. */
|
|
720
790
|
const LEGACY_FOOTER = new RegExp(`</sub>\\n\\n${MARKER_SOURCE}$`);
|
|
@@ -726,7 +796,7 @@ const LEGACY_FOOTER = new RegExp(`</sub>\\n\\n${MARKER_SOURCE}$`);
|
|
|
726
796
|
* is never read; and the flag is an HTML comment, which `neutralise` defuses in model text.
|
|
727
797
|
*/
|
|
728
798
|
const AUTO_FOOTER = new RegExp(
|
|
729
|
-
`(?:^|\\n\\n)<sub>[^\\n]*\\$(\\d+(?:\\.\\d+)?)</sub>\\n\\n${AUTO_SOURCE}\\n\\n(?:${SIGNATURE_REVIEW_SOURCE}(?:\\n\\n
|
|
799
|
+
`(?:^|\\n\\n)<sub>[^\\n]*\\$(\\d+(?:\\.\\d+)?)</sub>\\n\\n${AUTO_SOURCE}\\n\\n(?:${SIGNATURE_REVIEW_SOURCE}(?:\\n\\n(?:${MARKER_SOURCE}|${PARTIAL_SOURCE}))?(?:\\n\\n${ATTEMPT_SOURCE})?|${SIGNATURE_REPLY_SOURCE})$`,
|
|
730
800
|
);
|
|
731
801
|
|
|
732
802
|
/** The dollars an engine-signed body records as spent by an automatic run, or null when it records none. */
|
|
@@ -770,9 +840,21 @@ export function readCeilingNotice(body: string) {
|
|
|
770
840
|
return body.trimEnd().match(CEILING_NOTICE)?.[1] ?? null;
|
|
771
841
|
}
|
|
772
842
|
|
|
773
|
-
/**
|
|
843
|
+
/**
|
|
844
|
+
* What a signed review's footer records: a complete review's marker (nothing left over), or a
|
|
845
|
+
* partial one's list of the files not reviewed. Null when the body does not end with that footer,
|
|
846
|
+
* or its partial state is not one `partialMarker` writes.
|
|
847
|
+
*/
|
|
848
|
+
export function readSignedState(body: string): ReviewState | null {
|
|
849
|
+
const found = body.trimEnd().match(SIGNED_FOOTER);
|
|
850
|
+
if (!found) return null;
|
|
851
|
+
if (found[1] !== undefined) return { sha: found[1], rest: [], tried: [] };
|
|
852
|
+
return decodePartialState(found[2], found[3]);
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
/** The commit in a signed review's footer, complete or partial, or null when the body does not end with that footer. */
|
|
774
856
|
export function readSignedMarker(body: string) {
|
|
775
|
-
return body
|
|
857
|
+
return readSignedState(body)?.sha ?? null;
|
|
776
858
|
}
|
|
777
859
|
|
|
778
860
|
/** The marker in the footer a review had before signatures existed (see `lastEngineMarker`). */
|
|
@@ -990,6 +1072,12 @@ function commitsText(commits: string[] | undefined) {
|
|
|
990
1072
|
export const prSoFar = (prior: { spent: number; runs: number }, thisRun: number) =>
|
|
991
1073
|
`PR so far $${(prior.spent + thisRun).toFixed(2)} over ${countOf(prior.runs + 1, "automatic run")}`;
|
|
992
1074
|
|
|
1075
|
+
/** True when this run's dollars are API-price estimates: the engine is on a Claude subscription token (ADR 0037). */
|
|
1076
|
+
const onSubscription = (deps: { provider: ProviderAdapter }) => isEstimate(deps.provider.auth);
|
|
1077
|
+
|
|
1078
|
+
/** After a `<sub>` line of this run's cost and the pull request's total: the same wording as the cost line's. */
|
|
1079
|
+
const estimateTail = (estimate: boolean) => (estimate ? ` · ${ESTIMATE_NOTE}` : "");
|
|
1080
|
+
|
|
993
1081
|
export function buildReview(input: {
|
|
994
1082
|
/** A finding's `anchor`, when given (a `ReviewResult`'s), decides inline or body; else its file's patch does. */
|
|
995
1083
|
output: Omit<ReviewOutput, "findings"> & { findings: (Finding & Partial<Pick<ResultFinding, "anchor" | "origin" | "verification">>)[] };
|
|
@@ -1027,10 +1115,26 @@ export function buildReview(input: {
|
|
|
1027
1115
|
instructionChanges?: { files: string[]; baseSha: string };
|
|
1028
1116
|
/** The verifier's record (ADR 0027): its outcome goes in the footer's cost line. */
|
|
1029
1117
|
verification?: ResultVerificationV2;
|
|
1118
|
+
/** The run is on a Claude subscription: its dollar figures are API-price estimates (ADR 0037). */
|
|
1119
|
+
estimate?: boolean;
|
|
1030
1120
|
/** The risk map (ADR 0031), rendered right after the count line. */
|
|
1031
1121
|
riskMap?: RiskMap;
|
|
1122
|
+
/** PR Confidence (ADR 0033): the grade as one line after the count line; the description block holds the rest. */
|
|
1123
|
+
fidelity?: Fidelity;
|
|
1032
1124
|
/** A review of one push to the pull request: the map says it covers that push (ADR 0011). */
|
|
1033
1125
|
pushOnly?: boolean;
|
|
1126
|
+
/**
|
|
1127
|
+
* The files of `files` reviewed whole because the previous partial review did not reach them
|
|
1128
|
+
* (ADR 0036), named in the headline. Only with `since`.
|
|
1129
|
+
*/
|
|
1130
|
+
carried?: string[];
|
|
1131
|
+
/**
|
|
1132
|
+
* The files an earlier run already sent to a model and that came back not reviewed. One that
|
|
1133
|
+
* comes back not reviewed again is named in the notice and left out of the next push's list.
|
|
1134
|
+
*/
|
|
1135
|
+
tried?: string[];
|
|
1136
|
+
/** Where the engine's warnings go (a partial marker too long to write). */
|
|
1137
|
+
warn?: (line: string) => void;
|
|
1034
1138
|
}): Review {
|
|
1035
1139
|
const { output, files, headSha, since, scoped, costUsd, batches = [], modelLabel, effort } = input;
|
|
1036
1140
|
const mention = input.mention ?? DEFAULT_MENTION;
|
|
@@ -1068,9 +1172,14 @@ export function buildReview(input: {
|
|
|
1068
1172
|
const reviewed = new Set(output.files_reviewed);
|
|
1069
1173
|
const notReviewed = expected.filter((f) => !reviewed.has(f));
|
|
1070
1174
|
const reasons = new Map(output.files_skipped.map((s) => [s.path, s.reason]));
|
|
1071
|
-
const
|
|
1175
|
+
const carried = new Set((input.carried ?? []).filter((f) => expected.includes(f)));
|
|
1176
|
+
const sinceScope = since
|
|
1072
1177
|
? `the changes since \`${short(since)}\`${input.rebased ? " (after a rebase)" : commitsText(input.commits)}`
|
|
1073
1178
|
: "the whole pull request";
|
|
1179
|
+
const unread = `the ${countOf(carried.size, "file")} the previous review did not reach`;
|
|
1180
|
+
// A resumed review: what changed since, if anything did, and the files the last one left unread.
|
|
1181
|
+
const scope =
|
|
1182
|
+
since && carried.size > 0 ? (carried.size < expected.length ? `${sinceScope} and ${unread}` : unread) : sinceScope;
|
|
1074
1183
|
const unreviewable = input.unreviewable ?? [];
|
|
1075
1184
|
const notReviewable =
|
|
1076
1185
|
unreviewable.length > 0
|
|
@@ -1079,9 +1188,20 @@ export function buildReview(input: {
|
|
|
1079
1188
|
const parts: string[] = [];
|
|
1080
1189
|
const list = notReviewed.map((f) => `\`${neutralise(f)}\`${reasons.has(f) ? ` (${neutralise(reasons.get(f) ?? "")})` : ""}`);
|
|
1081
1190
|
const fullList = list.join(", ");
|
|
1191
|
+
// What the next push carries (ADR 0036). A file that was sent to a model before and came back
|
|
1192
|
+
// not reviewed again, with a reason other than "not reached" (the run never got to it), has had
|
|
1193
|
+
// its two tries: it is named here and no longer carried. A scoped review leaves the list alone.
|
|
1194
|
+
const notReached = (f: string) => (reasons.get(f) ?? "").startsWith("not reached");
|
|
1195
|
+
const priorTried = new Set(input.tried ?? []);
|
|
1196
|
+
const twice = scoped ? [] : notReviewed.filter((f) => priorTried.has(f) && !notReached(f));
|
|
1197
|
+
const rest = scoped ? [] : notReviewed.filter((f) => !twice.includes(f));
|
|
1198
|
+
const twiceNote =
|
|
1199
|
+
twice.length > 0
|
|
1200
|
+
? ` Not reviewed after two tries, so the next push will not carry ${twice.length === 1 ? "it" : "them"}: ${fitList(twice.map((f) => `\`${neutralise(f)}\``), 2_000)}.`
|
|
1201
|
+
: "";
|
|
1082
1202
|
if (notReviewed.length > 0) {
|
|
1083
1203
|
parts.push(
|
|
1084
|
-
branded(`⚠️ **Partial review**: the agent reviewed ${expected.length - notReviewed.length} of ${expected.length} files of ${scope}, at \`${short(headSha)}\`. Not reviewed: ${fullList}${notReviewable}
|
|
1204
|
+
branded(`⚠️ **Partial review**: the agent reviewed ${expected.length - notReviewed.length} of ${expected.length} files of ${scope}, at \`${short(headSha)}\`. Not reviewed: ${fullList}${notReviewable}.${twiceNote} Ask \`${mention} review <paths>\` for the rest.`),
|
|
1085
1205
|
);
|
|
1086
1206
|
} else {
|
|
1087
1207
|
parts.push(
|
|
@@ -1097,6 +1217,7 @@ export function buildReview(input: {
|
|
|
1097
1217
|
parts.push(
|
|
1098
1218
|
`${countLine(output.findings)}; ${comments.length} on their lines. Reply in a comment's thread to answer it.`,
|
|
1099
1219
|
);
|
|
1220
|
+
if (input.fidelity) parts.push(fidelitySummary(input.fidelity));
|
|
1100
1221
|
if (input.verification?.status === "failed") {
|
|
1101
1222
|
parts.push("⚠️ **Not verified**: the second reader that checks each finding did not finish, so these findings are posted as the review wrote them.");
|
|
1102
1223
|
} else if (input.verification?.status === "partial") {
|
|
@@ -1129,7 +1250,7 @@ export function buildReview(input: {
|
|
|
1129
1250
|
`| ${neutralise(b.label)} | ${b.files} | ${b.lines} | ${b.maxRisk} | ${b.status} | ${b.turns} | $${b.costUsd.toFixed(2)} |`,
|
|
1130
1251
|
);
|
|
1131
1252
|
parts.push(
|
|
1132
|
-
`<details><summary>Coverage by batch</summary>\n\n| Batch | Files | Lines | Highest risk | Status | Turns | Cost |\n| --- | --- | --- | --- | --- | --- | --- |\n${rows.join("\n")}\n\n</details>`,
|
|
1253
|
+
`<details><summary>Coverage by batch</summary>\n\n| Batch | Files | Lines | Highest risk | Status | Turns | Cost${input.estimate ? " (estimate)" : ""} |\n| --- | --- | --- | --- | --- | --- | --- |\n${rows.join("\n")}\n\n</details>`,
|
|
1133
1254
|
);
|
|
1134
1255
|
}
|
|
1135
1256
|
parts.push(
|
|
@@ -1144,14 +1265,26 @@ export function buildReview(input: {
|
|
|
1144
1265
|
...(input.automatic && input.priorSpend ? [prSoFar(input.priorSpend, costUsd)] : []),
|
|
1145
1266
|
],
|
|
1146
1267
|
usage: input.usage,
|
|
1268
|
+
estimate: input.estimate,
|
|
1147
1269
|
costUsd,
|
|
1148
1270
|
}),
|
|
1149
1271
|
);
|
|
1150
1272
|
if (input.automatic) parts.push(SIGNATURE.auto);
|
|
1151
1273
|
parts.push(SIGNATURE.review);
|
|
1152
|
-
// The next `review` request starts from this commit
|
|
1153
|
-
//
|
|
1154
|
-
|
|
1274
|
+
// The next `review` request starts from this commit. A review that covered everything moves it
|
|
1275
|
+
// (the complete marker). A partial one moves it too, and lists what it did not reach, so the next
|
|
1276
|
+
// push reviews those files (ADR 0036). A scoped review writes neither: it read only the paths
|
|
1277
|
+
// asked for, so the last unscoped review's commit and list stay as they were.
|
|
1278
|
+
if (!scoped) {
|
|
1279
|
+
const tried = rest.filter((f) => priorTried.has(f) || !notReached(f));
|
|
1280
|
+
const state = rest.length > 0 ? partialMarker({ sha: headSha, rest, tried }) : null;
|
|
1281
|
+
if (rest.length === 0) parts.push(marker(headSha));
|
|
1282
|
+
else if (state !== null) parts.push(state);
|
|
1283
|
+
else
|
|
1284
|
+
input.warn?.(
|
|
1285
|
+
`${rest.length} file(s) not reviewed do not fit in a partial marker (more than ${MAX_PARTIAL_MARKER_CHARS} characters): none is written, so the next push reviews the whole pull request again.`,
|
|
1286
|
+
);
|
|
1287
|
+
}
|
|
1155
1288
|
// A part another text replaces (or `null`, none) while the body is being fitted.
|
|
1156
1289
|
const replaced = new Map<number, string | null>();
|
|
1157
1290
|
const render = () =>
|
|
@@ -1702,7 +1835,8 @@ export function listFiles(gh: GhFn, repo: string, pr: number): ReviewFile[] {
|
|
|
1702
1835
|
}
|
|
1703
1836
|
|
|
1704
1837
|
/**
|
|
1705
|
-
* The commit the last Wachi review covered
|
|
1838
|
+
* The commit the last Wachi review covered, complete or partial (a partial review's marker also
|
|
1839
|
+
* lists the files it did not reach; `lastReviewState` returns them). A review is Wachi's when `github-actions[bot]` wrote
|
|
1706
1840
|
* it AND it carries the review signature: that login is shared by every workflow of the
|
|
1707
1841
|
* repository, so the login alone proves nothing. The marker counts only at the footer position
|
|
1708
1842
|
* (see `readSignedMarker`).
|
|
@@ -1713,7 +1847,16 @@ export function listFiles(gh: GhFn, repo: string, pr: number): ReviewFile[] {
|
|
|
1713
1847
|
* ones are another workflow's and are ignored. Drop the unsigned branch once the cutoff has passed.
|
|
1714
1848
|
*/
|
|
1715
1849
|
export function lastReviewedSha(gh: GhFn, repo: string, pr: number) {
|
|
1716
|
-
return
|
|
1850
|
+
return lastReviewState(gh, repo, pr)?.sha ?? null;
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
/**
|
|
1854
|
+
* What the last Wachi review left for the next run (see `lastReviewedSha` for which reviews count):
|
|
1855
|
+
* its commit and, for a partial review, the files it did not reach. The commit alone is enough for
|
|
1856
|
+
* a run that only answers threads. The push path reads the files too (`review`).
|
|
1857
|
+
*/
|
|
1858
|
+
export function lastReviewState(gh: GhFn, repo: string, pr: number) {
|
|
1859
|
+
return lastEngineState(
|
|
1717
1860
|
botReviews(gh, repo, pr).map((r) => ({ body: r.body, created_at: r.submitted_at })),
|
|
1718
1861
|
);
|
|
1719
1862
|
}
|
|
@@ -1728,15 +1871,21 @@ function botReviews(gh: GhFn, repo: string, pr: number) {
|
|
|
1728
1871
|
return reviews.map((r) => ({ body: r.body ?? "", submitted_at: r.submitted_at ?? undefined }));
|
|
1729
1872
|
}
|
|
1730
1873
|
|
|
1731
|
-
/** The
|
|
1732
|
-
export function
|
|
1874
|
+
/** The state of the last review the engine wrote, among `github-actions[bot]` reviews in order. */
|
|
1875
|
+
export function lastEngineState(reviews: { body: string; created_at?: string }[]): ReviewState | null {
|
|
1733
1876
|
const firstSigned = reviews.findIndex((r) => r.body.includes(SIGNATURE.review));
|
|
1734
|
-
const found = reviews.map((r, i) => {
|
|
1735
|
-
if (r.body.includes(SIGNATURE.review)) return
|
|
1877
|
+
const found = reviews.map((r, i): ReviewState | null => {
|
|
1878
|
+
if (r.body.includes(SIGNATURE.review)) return readSignedState(r.body);
|
|
1736
1879
|
const legacy = (firstSigned === -1 || i < firstSigned) && beforeLegacyCutoff(r.created_at);
|
|
1737
|
-
|
|
1880
|
+
const sha = legacy ? readLegacyMarker(r.body) : null;
|
|
1881
|
+
return sha === null ? null : { sha, rest: [], tried: [] };
|
|
1738
1882
|
});
|
|
1739
|
-
return found.filter((m): m is
|
|
1883
|
+
return found.filter((m): m is ReviewState => m !== null).at(-1) ?? null;
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1886
|
+
/** The commit of the last review the engine wrote, complete or partial, among `github-actions[bot]` reviews in order. */
|
|
1887
|
+
export function lastEngineMarker(reviews: { body: string; created_at?: string }[]) {
|
|
1888
|
+
return lastEngineState(reviews)?.sha ?? null;
|
|
1740
1889
|
}
|
|
1741
1890
|
|
|
1742
1891
|
export interface Comparison {
|
|
@@ -2214,7 +2363,7 @@ export function ceilingRoom(
|
|
|
2214
2363
|
postReview(deps, repo, pr, headSha, {
|
|
2215
2364
|
body: [
|
|
2216
2365
|
branded(
|
|
2217
|
-
`Automatic reviews paused: $${spent.toFixed(2)} of $${ceiling.toFixed(2)} spent on this pull request. Comment \`${config.mention} review\` to review now
|
|
2366
|
+
`Automatic reviews paused: $${spent.toFixed(2)} of $${ceiling.toFixed(2)} spent on this pull request. Comment \`${config.mention} review\` to review now.${onSubscription(deps) ? " These reviews run on a Claude subscription, so the dollars are API-equivalent estimates, not a bill." : ""}`,
|
|
2218
2367
|
),
|
|
2219
2368
|
SIGNATURE.review,
|
|
2220
2369
|
ceilingFooter(ceiling),
|
|
@@ -2224,6 +2373,81 @@ export function ceilingRoom(
|
|
|
2224
2373
|
return null;
|
|
2225
2374
|
}
|
|
2226
2375
|
|
|
2376
|
+
/** The estimate of reviewing `files` with this configuration: the batches `planBatches` makes, priced per batch. */
|
|
2377
|
+
export function estimateFor(files: ReviewFile[], config: EngineConfig): ReviewEstimate {
|
|
2378
|
+
const batches = planBatches(files, { maxLines: config.batchLines, maxChars: config.batchChars }).length;
|
|
2379
|
+
return estimateReview(files, { batches, batchBudgetUsd: config.batchBudgetUsd, model: config.model });
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
/** The hidden marker of the size-gate comment, so the next push updates it instead of adding another. */
|
|
2383
|
+
export const SIZE_GATE_MARKER = "<!-- wachi:size-gate -->";
|
|
2384
|
+
|
|
2385
|
+
const usd2 = (n: number) => `$${n.toFixed(2)}`;
|
|
2386
|
+
const count = (n: number) => n.toLocaleString("en-US");
|
|
2387
|
+
|
|
2388
|
+
/**
|
|
2389
|
+
* The comment that says a pull request was not reviewed because its estimate is above the size
|
|
2390
|
+
* gate (nothing was sent to a model), what the estimate rests on, and the inputs that lift it.
|
|
2391
|
+
*/
|
|
2392
|
+
export function sizeGateBody(o: {
|
|
2393
|
+
estimate: ReviewEstimate;
|
|
2394
|
+
gate: number;
|
|
2395
|
+
headSha: string;
|
|
2396
|
+
mention: string;
|
|
2397
|
+
model: string;
|
|
2398
|
+
}): string {
|
|
2399
|
+
const { estimate: e, gate, headSha, mention, model } = o;
|
|
2400
|
+
const budget = suggestedBudget(e.usd);
|
|
2401
|
+
return [
|
|
2402
|
+
branded(
|
|
2403
|
+
`**Not reviewed: this pull request is above the size gate.** Reviewing all of it would cost about **${usd2(e.usd)}**, and the gate is **${usd2(gate)}**, so nothing was sent to a model and nothing was spent.`,
|
|
2404
|
+
),
|
|
2405
|
+
`The estimate: ${count(e.batches)} batches × ${usd2(e.perBatchUsd)} a batch for \`${model}\` (${count(e.files)} files, ${count(e.lines)} changed lines), from the diff alone, before any model call. It can be off by about 40% either way.`,
|
|
2406
|
+
`To review the whole pull request, set the run budget in the workflow that calls Wachiman, then comment \`${mention} review\`:`,
|
|
2407
|
+
["```yaml", " with:", ` budget_usd: ${budget}`, "```"].join("\n"),
|
|
2408
|
+
`A \`budget_usd\` at or above the estimate lifts the gate for that run. To have pushes reviewed automatically as well, also set \`auto_budget_usd: ${budget}\`. To review part of it now, comment \`${mention} review <path> [path…]\`.`,
|
|
2409
|
+
`<sub>Estimated at \`${short(headSha)}\`. This comment is updated on each push instead of a new one.</sub>`,
|
|
2410
|
+
SIZE_GATE_MARKER,
|
|
2411
|
+
].join("\n\n");
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
/**
|
|
2415
|
+
* Posts the size-gate comment, or updates the one already there. Only the bot's own comments are
|
|
2416
|
+
* looked at, so a person cannot plant the marker. The comment has no cost line, so
|
|
2417
|
+
* `automaticSpend` does not count it.
|
|
2418
|
+
*/
|
|
2419
|
+
export function postSizeGateNotice(
|
|
2420
|
+
deps: Deps,
|
|
2421
|
+
repo: string,
|
|
2422
|
+
pr: number,
|
|
2423
|
+
headSha: string,
|
|
2424
|
+
estimate: ReviewEstimate,
|
|
2425
|
+
gate: number,
|
|
2426
|
+
) {
|
|
2427
|
+
const { config } = deps;
|
|
2428
|
+
const body = sizeGateBody({
|
|
2429
|
+
estimate,
|
|
2430
|
+
gate,
|
|
2431
|
+
headSha,
|
|
2432
|
+
mention: config.mention,
|
|
2433
|
+
model: config.model,
|
|
2434
|
+
});
|
|
2435
|
+
if (config.dryRun) {
|
|
2436
|
+
deps.log("--- size gate comment ---\n" + body);
|
|
2437
|
+
return;
|
|
2438
|
+
}
|
|
2439
|
+
const jq = '.[] | select(.user.login == "github-actions[bot]") | {id, body} | @json';
|
|
2440
|
+
const existing = jsonLines<{ id: number; body: string | null }>(
|
|
2441
|
+
deps.gh(["api", "--paginate", `repos/${repo}/issues/${pr}/comments`, "--jq", jq]),
|
|
2442
|
+
).filter((c) => (c.body ?? "").includes(SIZE_GATE_MARKER));
|
|
2443
|
+
const last = existing[existing.length - 1];
|
|
2444
|
+
if (last) {
|
|
2445
|
+
deps.gh(["api", "-X", "PATCH", `repos/${repo}/issues/comments/${last.id}`, "-f", `body=${body}`]);
|
|
2446
|
+
return;
|
|
2447
|
+
}
|
|
2448
|
+
deps.gh(["pr", "comment", String(pr), "-R", repo, "--body-file", "-"], body);
|
|
2449
|
+
}
|
|
2450
|
+
|
|
2227
2451
|
/** Waits before the 1st and 2nd retry of a review POST GitHub answered with a transient error. */
|
|
2228
2452
|
export const REVIEW_RETRY_DELAYS_MS = [1_000, 3_000];
|
|
2229
2453
|
|
|
@@ -2257,18 +2481,30 @@ function reviewLanded(deps: Deps, repo: string, pr: number, attemptId: string) {
|
|
|
2257
2481
|
return reviews.some((r) => (r.body ?? "").trimEnd().endsWith(attemptFooter(attemptId)));
|
|
2258
2482
|
}
|
|
2259
2483
|
|
|
2260
|
-
|
|
2484
|
+
/** The address of the review GitHub created, from the answer to the POST; undefined when it said none. */
|
|
2485
|
+
function reviewUrlOf(answer: string): string | undefined {
|
|
2486
|
+
try {
|
|
2487
|
+
const url = (JSON.parse(answer) as { html_url?: unknown }).html_url;
|
|
2488
|
+
return typeof url === "string" && url.startsWith("https://") ? url : undefined;
|
|
2489
|
+
} catch {
|
|
2490
|
+
return undefined;
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
|
|
2494
|
+
/** Posts the review. Returns its address when GitHub gave one (the description block links to it). */
|
|
2495
|
+
export function postReview(deps: Deps, repo: string, pr: number, headSha: string, review: Review): string | undefined {
|
|
2261
2496
|
if (deps.config.dryRun) {
|
|
2262
2497
|
deps.log(`--- review at ${short(headSha)} ---\n${review.body}`);
|
|
2263
2498
|
for (const c of review.comments) deps.log(`--- ${c.path}:${c.line} ---\n${c.body}`);
|
|
2264
|
-
return;
|
|
2499
|
+
return undefined;
|
|
2265
2500
|
}
|
|
2501
|
+
let url: string | undefined;
|
|
2266
2502
|
const sleep = deps.sleep ?? blockingSleep;
|
|
2267
2503
|
// One id for this call: the retries and the fallback are the same attempt, and an older review
|
|
2268
2504
|
// of the same head does not carry it.
|
|
2269
2505
|
const attemptId = crypto.randomUUID();
|
|
2270
|
-
const send = (r: Review) =>
|
|
2271
|
-
deps.gh(
|
|
2506
|
+
const send = (r: Review) => {
|
|
2507
|
+
const answer = deps.gh(
|
|
2272
2508
|
["api", "-X", "POST", `repos/${repo}/pulls/${pr}/reviews`, "--input", "-"],
|
|
2273
2509
|
JSON.stringify({
|
|
2274
2510
|
commit_id: headSha,
|
|
@@ -2277,6 +2513,8 @@ export function postReview(deps: Deps, repo: string, pr: number, headSha: string
|
|
|
2277
2513
|
comments: r.comments,
|
|
2278
2514
|
}),
|
|
2279
2515
|
);
|
|
2516
|
+
url = reviewUrlOf(answer);
|
|
2517
|
+
};
|
|
2280
2518
|
// A transient error may hide a POST that did land, so look before each retry.
|
|
2281
2519
|
const sendWithRetry = (r: Review) => {
|
|
2282
2520
|
for (let retry = 0; ; retry++) {
|
|
@@ -2306,6 +2544,7 @@ export function postReview(deps: Deps, repo: string, pr: number, headSha: string
|
|
|
2306
2544
|
deps.warn(`GitHub refused the inline comments; posting them in the body. ${error}`);
|
|
2307
2545
|
sendWithRetry(moveCommentsToBody(review));
|
|
2308
2546
|
}
|
|
2547
|
+
return url;
|
|
2309
2548
|
}
|
|
2310
2549
|
|
|
2311
2550
|
/**
|
|
@@ -2348,6 +2587,13 @@ export interface ReviewOptions {
|
|
|
2348
2587
|
* finding above `info`. Its money is reserved out of `totalBudgetUsd` before the batches run.
|
|
2349
2588
|
*/
|
|
2350
2589
|
verify?: { model: string; budgetUsd: number };
|
|
2590
|
+
/**
|
|
2591
|
+
* PR Confidence (ADR 0033): for a pull request, one small call first lists the claims in its
|
|
2592
|
+
* title and body, every batch says which claim each of its files serves, and the engine computes
|
|
2593
|
+
* the grade into `result.fidelity`. Set by the pull request review for a full review only, never
|
|
2594
|
+
* by `reviewOptionsOf`: a local review and an incremental or scoped one have no grade.
|
|
2595
|
+
*/
|
|
2596
|
+
confidence?: { budgetUsd: number };
|
|
2351
2597
|
}
|
|
2352
2598
|
|
|
2353
2599
|
/** Today's settings as the core's options. */
|
|
@@ -2406,6 +2652,9 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2406
2652
|
// About 40k tokens at 3.5 characters a token.
|
|
2407
2653
|
maxChars: options.batchChars,
|
|
2408
2654
|
});
|
|
2655
|
+
// The files this run reviews whole because the previous review did not reach them.
|
|
2656
|
+
const keptNames = new Set(kept.map((f) => f.filename));
|
|
2657
|
+
const carriedHere = (input.since?.carried ?? []).filter((f) => keptNames.has(f));
|
|
2409
2658
|
const threads = input.conversation ?? [];
|
|
2410
2659
|
const threadsByBatch = assignThreads(threads, batches);
|
|
2411
2660
|
const everyFile = kept.map((f) => `- ${f.filename} +${f.additions} −${f.deletions}`).join("\n");
|
|
@@ -2417,6 +2666,7 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2417
2666
|
headDir: input.headDir,
|
|
2418
2667
|
since: input.since?.sha ?? null,
|
|
2419
2668
|
rebased: input.since?.rebased,
|
|
2669
|
+
...(carriedHere.length > 0 ? { carried: carriedHere, changedSince: kept.length - carriedHere.length } : {}),
|
|
2420
2670
|
paths: input.scope,
|
|
2421
2671
|
dropped: dropped.length,
|
|
2422
2672
|
everyFile,
|
|
@@ -2430,14 +2680,32 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2430
2680
|
const notRun: string[] = [];
|
|
2431
2681
|
let spent = 0;
|
|
2432
2682
|
let usage = NO_USAGE;
|
|
2683
|
+
// PR Confidence (ADR 0033): only a pull request, only when the caller asked. The claims call
|
|
2684
|
+
// comes first so that every batch can say which claim each of its files serves; what it spends
|
|
2685
|
+
// is spent before the batches, so they cannot overspend the pull request's budget.
|
|
2686
|
+
const confidence = change.kind === "pull-request" ? options.confidence : undefined;
|
|
2687
|
+
const claimsStep = confidence
|
|
2688
|
+
? listClaims(core, options, context, Math.min(confidence.budgetUsd, batchTotal / 4))
|
|
2689
|
+
: undefined;
|
|
2690
|
+
if (claimsStep) {
|
|
2691
|
+
spent += claimsStep.costUsd;
|
|
2692
|
+
usage = addUsage(usage, claimsStep.usage);
|
|
2693
|
+
if (claimsStep.claims && claimsStep.claims.length > 0) context.claims = claimsStep.claims.map((c) => c.text);
|
|
2694
|
+
}
|
|
2433
2695
|
let budgetError = false;
|
|
2434
2696
|
let limitError: "turns" | "timeout" | undefined;
|
|
2697
|
+
// Any call, a retry included, that ended on a turn or time limit or on its budget: a batch that
|
|
2698
|
+
// still gave usable output is not a finished review, so PR Confidence cannot grade it.
|
|
2699
|
+
let anyLimit = false;
|
|
2435
2700
|
// The first call the provider refused because the reviewer's account cannot make it (no credit,
|
|
2436
2701
|
// bad key, rate limit that outlasted its retry): every later call would fail the same way.
|
|
2437
2702
|
let rejection: { kind: Exclude<ProviderErrorClass, "other">; text: string } | undefined;
|
|
2438
2703
|
// The first unclassified error, kept for the message of the final throw.
|
|
2439
2704
|
let otherError: string | undefined;
|
|
2440
2705
|
const rejectedFiles: string[] = [];
|
|
2706
|
+
// Files of a batch that was started and then stopped on its budget, so no model refused them:
|
|
2707
|
+
// like the batches never started, they were not reached (ADR 0036).
|
|
2708
|
+
const budgetCut: string[] = [];
|
|
2441
2709
|
const noteRejection = (r: BatchResult["rejection"]) => {
|
|
2442
2710
|
if (!r) return;
|
|
2443
2711
|
if (r.kind === "other") otherError ??= r.text;
|
|
@@ -2488,6 +2756,7 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2488
2756
|
if (first.output) parts.push({ label, output: first.output });
|
|
2489
2757
|
if (first.budget) budgetError = true;
|
|
2490
2758
|
if (first.stopped) limitError = first.stopped;
|
|
2759
|
+
if (first.budget || first.stopped) anyLimit = true;
|
|
2491
2760
|
let status = first.output
|
|
2492
2761
|
? "reviewed"
|
|
2493
2762
|
: first.budget
|
|
@@ -2524,8 +2793,11 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2524
2793
|
turns += retry.turns;
|
|
2525
2794
|
cost += retry.costUsd;
|
|
2526
2795
|
noteRejection(retry.rejection);
|
|
2796
|
+
if (retry.budget || retry.stopped) anyLimit = true;
|
|
2527
2797
|
if (retry.output) parts.push({ label: `${label}, retry`, output: retry.output });
|
|
2528
2798
|
const left = batchGaps(gaps, retry.output);
|
|
2799
|
+
if (retry.rejection && retry.rejection.kind !== "other") rejectedFiles.push(...left.map((f) => f.filename));
|
|
2800
|
+
else if (retry.budget) budgetCut.push(...left.map((f) => f.filename));
|
|
2529
2801
|
status =
|
|
2530
2802
|
left.length === 0 ? "reviewed after a retry" : `${left.length} file(s) left after a retry`;
|
|
2531
2803
|
} else if (gaps.length > 0) {
|
|
@@ -2534,6 +2806,9 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2534
2806
|
: first.stopped === "timeout"
|
|
2535
2807
|
? `${gaps.length} file(s) left: the call timed out`
|
|
2536
2808
|
: `${gaps.length} file(s) left: budget spent`;
|
|
2809
|
+
// A refused call or a spent budget is not the files' doing; a timeout may be, so it stays unexplained.
|
|
2810
|
+
if (rejection) rejectedFiles.push(...gaps.map((f) => f.filename));
|
|
2811
|
+
else if (first.stopped !== "timeout") budgetCut.push(...gaps.map((f) => f.filename));
|
|
2537
2812
|
}
|
|
2538
2813
|
reports.push({
|
|
2539
2814
|
label,
|
|
@@ -2576,8 +2851,16 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2576
2851
|
} else checked = annotateUnverified(merged.findings);
|
|
2577
2852
|
for (const path of notRun)
|
|
2578
2853
|
merged.files_skipped.push({ path, reason: "not reached: the pull request's budget was spent" });
|
|
2579
|
-
|
|
2580
|
-
|
|
2854
|
+
// A file the model listed itself, with a reason of its own, is replaced: the run's own cause comes first.
|
|
2855
|
+
for (const [paths, reason] of [
|
|
2856
|
+
[budgetCut, "not reached: the call ran out of its budget"],
|
|
2857
|
+
[rejectedFiles, "not reached: the provider rejected the call"],
|
|
2858
|
+
] as const) {
|
|
2859
|
+
for (const path of paths) {
|
|
2860
|
+
merged.files_skipped = merged.files_skipped.filter((s) => s.path !== path);
|
|
2861
|
+
merged.files_skipped.push({ path, reason });
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2581
2864
|
|
|
2582
2865
|
// The anchor is decided here, so a sink needs no patch to know which findings sit on a diff line.
|
|
2583
2866
|
const commentable = new Map(kept.map((f) => [f.filename, commentableLines(f.patch ?? "")]));
|
|
@@ -2625,6 +2908,41 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2625
2908
|
core.log(
|
|
2626
2909
|
`Risk map: ${riskMap.droppedFacts} risk fact(s) or judgement item(s) dropped: their file is not in the review, or their line does not exist at the head.`,
|
|
2627
2910
|
);
|
|
2911
|
+
let fidelity: Fidelity | undefined;
|
|
2912
|
+
if (claimsStep) {
|
|
2913
|
+
const proposed = claimsStep.claims;
|
|
2914
|
+
const entries = parts.flatMap((p) => (Array.isArray(p.output.claim_assignments) ? p.output.claim_assignments : []));
|
|
2915
|
+
const assembled = proposed ? assembleClaims(proposed, entries) : { claims: [], assignments: [] };
|
|
2916
|
+
const sha = change.head.sha;
|
|
2917
|
+
const digest = claimsDigest(change.title ?? "", change.description);
|
|
2918
|
+
fidelity = buildFidelity({
|
|
2919
|
+
claims: checkClaims(assembled.claims, (path, line) => {
|
|
2920
|
+
const count = lineCount(path);
|
|
2921
|
+
return count !== null && line >= 1 && line <= count;
|
|
2922
|
+
}),
|
|
2923
|
+
assignments: assembled.assignments,
|
|
2924
|
+
// Every file of the pull request, not only the ones the review read: a binary, an oversize
|
|
2925
|
+
// diff or a path the owner excluded is explained only by a rule, or it is named unexplained.
|
|
2926
|
+
changedFiles: input.files.map((f) => ({ path: f.filename, patch: f.patch })),
|
|
2927
|
+
// Not this change's doing: a pre-existing finding does not grade the pull request.
|
|
2928
|
+
openFindings: findings
|
|
2929
|
+
.filter((f) => f.origin !== "pre-existing")
|
|
2930
|
+
.map((f) => ({ severity: f.severity, path: f.path, line: f.line, title: f.title })),
|
|
2931
|
+
riskMap,
|
|
2932
|
+
run: {
|
|
2933
|
+
// No claims list means no claim was read: that is an unknown, not a pull request that claims nothing.
|
|
2934
|
+
finished: proposed !== undefined && stop === "complete" && !anyLimit,
|
|
2935
|
+
...(notRun.length > 0 || budgetError || claimsStep.budget ? { stoppedBy: "spend-ceiling" as const } : {}),
|
|
2936
|
+
...(claimsStep.truncated ? { textTruncated: true } : {}),
|
|
2937
|
+
unreadFiles: expected.filter((f) => !reviewedSet.has(f)).length,
|
|
2938
|
+
reviewedCommit: sha,
|
|
2939
|
+
// The connector checks again at the moment it writes: here the review and the pull request agree.
|
|
2940
|
+
headCommit: sha,
|
|
2941
|
+
digestAtRead: digest,
|
|
2942
|
+
digestAtWrite: digest,
|
|
2943
|
+
},
|
|
2944
|
+
});
|
|
2945
|
+
}
|
|
2628
2946
|
return {
|
|
2629
2947
|
format: REVIEW_RESULT_V2,
|
|
2630
2948
|
change,
|
|
@@ -2660,9 +2978,47 @@ export function reviewChange(input: ReviewInput, options: ReviewOptions, deps: C
|
|
|
2660
2978
|
notices,
|
|
2661
2979
|
},
|
|
2662
2980
|
...(verification ? { verification } : {}),
|
|
2981
|
+
...(fidelity ? { fidelity } : {}),
|
|
2663
2982
|
};
|
|
2664
2983
|
}
|
|
2665
2984
|
|
|
2985
|
+
/**
|
|
2986
|
+
* The claims call (ADR 0033 decision 8): one small call that lists what the title and body claim,
|
|
2987
|
+
* from the author's text alone (Wachiman's own block stripped). It fails closed: with no list the
|
|
2988
|
+
* review still runs, and the grade is `cannot say`. A call that cannot be afforded is not made.
|
|
2989
|
+
*/
|
|
2990
|
+
function listClaims(
|
|
2991
|
+
deps: CoreDeps,
|
|
2992
|
+
options: ReviewOptions,
|
|
2993
|
+
ctx: ReviewContext,
|
|
2994
|
+
budgetUsd: number,
|
|
2995
|
+
): { claims?: ProposedClaim[]; costUsd: number; usage: TokenUsage; budget: boolean; truncated: boolean } {
|
|
2996
|
+
const input = claimsInput(ctx.meta.title ?? "", ctx.meta.body);
|
|
2997
|
+
const truncated = claimsTextTruncated(input);
|
|
2998
|
+
const none = { costUsd: 0, usage: NO_USAGE, budget: false, truncated };
|
|
2999
|
+
if (budgetUsd < MIN_BATCH_USD) {
|
|
3000
|
+
deps.warn("PR Confidence: no budget left for the claims call; the pull request gets no grade.");
|
|
3001
|
+
return { ...none, budget: true };
|
|
3002
|
+
}
|
|
3003
|
+
const prompt = buildClaimsPrompt(input);
|
|
3004
|
+
const call = attempt(
|
|
3005
|
+
() => runProvider(deps, options, ctx.headDir, { operation: "review", label: "claims", ...ctx.costTags }, prompt, ctx.headDir, budgetUsd, CLAIMS_SCHEMA),
|
|
3006
|
+
budgetUsd,
|
|
3007
|
+
);
|
|
3008
|
+
if (!call.ok) {
|
|
3009
|
+
deps.warn(`PR Confidence: the claims call failed (${call.error}); the pull request gets no grade.`);
|
|
3010
|
+
return { ...none, costUsd: call.costUsd };
|
|
3011
|
+
}
|
|
3012
|
+
const { value: result } = call;
|
|
3013
|
+
// A call that ended on its turn or time limit is not trusted even when its list parses: the list
|
|
3014
|
+
// may be the part it got to, and nothing is salvaged from an unfinished call.
|
|
3015
|
+
const limited = result.terminalReason === "turns" || result.terminalReason === "timeout";
|
|
3016
|
+
const claims = result.isError || limited ? undefined : parseClaimList(result.structuredOutput);
|
|
3017
|
+
if (limited) deps.warn(`PR Confidence: the claims call ended on its ${result.terminalReason} limit; the pull request gets no grade.`);
|
|
3018
|
+
else if (!claims) deps.warn("PR Confidence: the claims call returned no usable list; the pull request gets no grade.");
|
|
3019
|
+
return { claims, costUsd: result.costUsd, usage: result.usage ?? NO_USAGE, budget: result.terminalReason === "budget", truncated };
|
|
3020
|
+
}
|
|
3021
|
+
|
|
2666
3022
|
/** Why a verifier call ended without answering everything, from how it ended; undefined for a clean finish. */
|
|
2667
3023
|
function stopOf(result: { terminalReason: string; isError: boolean; providerError?: ProviderErrorClass; errorText?: string }): VerifierStop | undefined {
|
|
2668
3024
|
const { terminalReason: reason } = result;
|
|
@@ -2802,6 +3158,8 @@ export function githubReviewInput(p: {
|
|
|
2802
3158
|
since: string | null;
|
|
2803
3159
|
rebased: boolean;
|
|
2804
3160
|
commits?: string[];
|
|
3161
|
+
/** The files of `files` the previous partial review did not reach (they carry their whole diff). */
|
|
3162
|
+
carried?: string[];
|
|
2805
3163
|
threads: Thread[];
|
|
2806
3164
|
}): ReviewInput {
|
|
2807
3165
|
return {
|
|
@@ -2813,7 +3171,8 @@ export function githubReviewInput(p: {
|
|
|
2813
3171
|
base: { sha: p.baseSha },
|
|
2814
3172
|
head: { sha: p.headSha },
|
|
2815
3173
|
title: p.meta.title,
|
|
2816
|
-
description
|
|
3174
|
+
// Wachiman's own description block is never the author's text (ADR 0033 decision 14).
|
|
3175
|
+
description: p.meta.body === undefined ? undefined : stripBlock(p.meta.body),
|
|
2817
3176
|
author: p.meta.author?.login ?? undefined,
|
|
2818
3177
|
identifier: `#${p.pr}`,
|
|
2819
3178
|
},
|
|
@@ -2822,7 +3181,14 @@ export function githubReviewInput(p: {
|
|
|
2822
3181
|
headDir: p.headDir,
|
|
2823
3182
|
scope: p.scope,
|
|
2824
3183
|
...(p.since
|
|
2825
|
-
? {
|
|
3184
|
+
? {
|
|
3185
|
+
since: {
|
|
3186
|
+
sha: p.since,
|
|
3187
|
+
rebased: p.rebased,
|
|
3188
|
+
...(p.commits?.length ? { commits: p.commits } : {}),
|
|
3189
|
+
...(p.carried?.length ? { carried: p.carried } : {}),
|
|
3190
|
+
},
|
|
3191
|
+
}
|
|
2826
3192
|
: {}),
|
|
2827
3193
|
...(p.threads.length > 0 ? { conversation: p.threads } : {}),
|
|
2828
3194
|
};
|
|
@@ -2836,6 +3202,12 @@ export function githubReview(
|
|
|
2836
3202
|
rebased: boolean;
|
|
2837
3203
|
/** The commits since `since`, for the headline of a re-review. */
|
|
2838
3204
|
commits?: string[];
|
|
3205
|
+
/** The files the previous partial review did not reach, reviewed whole in this run. */
|
|
3206
|
+
carried?: string[];
|
|
3207
|
+
/** The files an earlier run sent to a model that came back not reviewed (ADR 0036). */
|
|
3208
|
+
tried?: string[];
|
|
3209
|
+
/** Where a warning goes (a partial marker too long to write). */
|
|
3210
|
+
warn?: (line: string) => void;
|
|
2839
3211
|
/** What the automatic runs before this one spent, for the footer's running total. */
|
|
2840
3212
|
priorSpend?: { spent: number; runs: number };
|
|
2841
3213
|
scoped: boolean;
|
|
@@ -2843,6 +3215,8 @@ export function githubReview(
|
|
|
2843
3215
|
automatic: boolean;
|
|
2844
3216
|
/** Computed by the connector from every file of the pull request, not only the reviewed ones. */
|
|
2845
3217
|
instructionChanges: { files: string[]; baseSha: string };
|
|
3218
|
+
/** The run is on a Claude subscription: the footer says its dollars are estimates. */
|
|
3219
|
+
estimate?: boolean;
|
|
2846
3220
|
},
|
|
2847
3221
|
): Review {
|
|
2848
3222
|
return buildReview({
|
|
@@ -2860,6 +3234,9 @@ export function githubReview(
|
|
|
2860
3234
|
since: github.since,
|
|
2861
3235
|
rebased: github.rebased,
|
|
2862
3236
|
commits: github.commits,
|
|
3237
|
+
carried: github.carried,
|
|
3238
|
+
tried: github.tried,
|
|
3239
|
+
warn: github.warn,
|
|
2863
3240
|
priorSpend: github.priorSpend,
|
|
2864
3241
|
unreviewable: result.coverage.unreviewable,
|
|
2865
3242
|
scoped: github.scoped,
|
|
@@ -2873,8 +3250,10 @@ export function githubReview(
|
|
|
2873
3250
|
mention: github.mention,
|
|
2874
3251
|
instructionChanges: github.instructionChanges,
|
|
2875
3252
|
riskMap: result.riskMap,
|
|
3253
|
+
...(result.fidelity ? { fidelity: result.fidelity } : {}),
|
|
2876
3254
|
pushOnly: github.since !== null,
|
|
2877
3255
|
...(result.verification ? { verification: result.verification } : {}),
|
|
3256
|
+
...(github.estimate ? { estimate: true } : {}),
|
|
2878
3257
|
});
|
|
2879
3258
|
}
|
|
2880
3259
|
|
|
@@ -2895,8 +3274,18 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
2895
3274
|
if (ignored.length > 0)
|
|
2896
3275
|
deps.log(`Not paths in this pull request, ignored: ${ignored.join(" ")}`);
|
|
2897
3276
|
// A dry run may ask for the whole pull request again (AGENT_REVIEW_FULL_REVIEW): no marker read.
|
|
2898
|
-
const
|
|
2899
|
-
|
|
3277
|
+
const state = config.fullReview ? null : lastReviewState(gh, repo, pr);
|
|
3278
|
+
const last = state?.sha ?? null;
|
|
3279
|
+
// What the last review did not reach (ADR 0036), by name. Only an unscoped run carries it over: a
|
|
3280
|
+
// request for paths reviews those paths. The files that left the pull request, or are excluded
|
|
3281
|
+
// now, are dropped here without a word; the rest are reviewed whole, with the patch the pull
|
|
3282
|
+
// request has now, even when they also changed since: nothing of them was ever reviewed.
|
|
3283
|
+
const unread = new Set(paths.length === 0 ? (state?.rest ?? []) : []);
|
|
3284
|
+
const carried: ReviewFile[] =
|
|
3285
|
+
unread.size === 0
|
|
3286
|
+
? []
|
|
3287
|
+
: filterFiles(prFiles.filter((f) => unread.has(f.filename)), [...DEFAULT_EXCLUDE, ...config.exclude], []).kept;
|
|
3288
|
+
if (last === headSha && paths.length === 0 && carried.length === 0) {
|
|
2900
3289
|
bodyOnly(
|
|
2901
3290
|
`Already reviewed at \`${short(headSha)}\`. Push changes, or ask \`${config.mention} review <paths>\`.`,
|
|
2902
3291
|
);
|
|
@@ -2907,7 +3296,12 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
2907
3296
|
// The commits since `since`, for the headline; only when compare lists every one of them.
|
|
2908
3297
|
let commits: string[] | undefined;
|
|
2909
3298
|
let files = prFiles;
|
|
2910
|
-
|
|
3299
|
+
// The head did not move but files were never reached: only those are reviewed.
|
|
3300
|
+
if (last === headSha && paths.length === 0) {
|
|
3301
|
+
deps.log(`Resuming at ${short(headSha)}: ${carried.length} file(s) the previous review did not reach: ${carried.map((f) => f.filename).join(", ")}.`);
|
|
3302
|
+
since = last;
|
|
3303
|
+
files = [];
|
|
3304
|
+
} else if (last && paths.length === 0) {
|
|
2911
3305
|
let status = "unavailable";
|
|
2912
3306
|
let cmp: ReturnType<typeof compareFiles> | null = null;
|
|
2913
3307
|
try {
|
|
@@ -2945,14 +3339,15 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
2945
3339
|
comments: [],
|
|
2946
3340
|
});
|
|
2947
3341
|
};
|
|
2948
|
-
if (delta.kind === "unchanged") {
|
|
3342
|
+
if (delta.kind === "unchanged" && carried.length === 0) {
|
|
2949
3343
|
upToDate(
|
|
2950
3344
|
`The ${cmp.files.length} file(s) changed since ${short(last)} are not part of the pull request's own change (a merge of the base branch), or their change is the one already reviewed; nothing to review.`,
|
|
2951
3345
|
);
|
|
2952
3346
|
return;
|
|
2953
3347
|
}
|
|
2954
3348
|
const keep = delta.kind === "changed" ? new Set(delta.files.map((f) => f.filename)) : null;
|
|
2955
|
-
|
|
3349
|
+
// "unchanged" reaches here only to resume: nothing changed since, the unread files remain.
|
|
3350
|
+
const candidates = delta.kind === "unchanged" ? [] : keep ? own.filter((f) => keep.has(f.filename)) : own;
|
|
2956
3351
|
// The file's patch at `last`: "" when the pull request did not change it then, undefined when unknown.
|
|
2957
3352
|
const before = delta.kind === "changed" ? new Map(delta.before.map((f) => [f.filename, f])) : null;
|
|
2958
3353
|
const thenOf = (f: ReviewFile) => patchAtReview(before, f, inPr.get(f.filename));
|
|
@@ -2974,7 +3369,7 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
2974
3369
|
deps.log(
|
|
2975
3370
|
`${foreignOnly.length} file(s) changed since ${short(last)} hold only lines that came from the merged branch, not reviewed: ${foreignOnly.join(", ")}.`,
|
|
2976
3371
|
);
|
|
2977
|
-
if (files.length === 0) {
|
|
3372
|
+
if (files.length === 0 && carried.length === 0) {
|
|
2978
3373
|
upToDate(`Every change since ${short(last)} in the pull request's files came from the merged branch; nothing to review.`);
|
|
2979
3374
|
return;
|
|
2980
3375
|
}
|
|
@@ -2982,12 +3377,20 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
2982
3377
|
deps.log(
|
|
2983
3378
|
`${cmp.files.length - files.length} of the ${cmp.files.length} file(s) changed since ${short(last)} are not reviewed: not in the pull request's own change, or that change is the one already reviewed.`,
|
|
2984
3379
|
);
|
|
2985
|
-
deps.log(`Reviewed since ${short(last)}: ${files.map((f) => f.filename).join(", ")}.`);
|
|
3380
|
+
if (files.length > 0) deps.log(`Reviewed since ${short(last)}: ${files.map((f) => f.filename).join(", ")}.`);
|
|
2986
3381
|
} else {
|
|
2987
3382
|
// The branch was rewritten (a rebase, a squash, a force-push). The content decides, not the SHAs.
|
|
2988
3383
|
const base = baseSha(gh, repo, pr);
|
|
2989
3384
|
const delta = rebaseDelta(gh, repo, base, last, prFiles);
|
|
2990
|
-
if (delta.kind === "unchanged") {
|
|
3385
|
+
if (delta.kind === "unchanged" && carried.length > 0) {
|
|
3386
|
+
// The same change over a new head, but files were never reached: only those are reviewed.
|
|
3387
|
+
deps.log(
|
|
3388
|
+
`The last reviewed commit ${short(last)} is not behind the head (${status}), but the pull request's changes are the same; rebased with no content change. The ${carried.length} file(s) the previous review did not reach are reviewed.`,
|
|
3389
|
+
);
|
|
3390
|
+
since = last;
|
|
3391
|
+
rebased = true;
|
|
3392
|
+
files = [];
|
|
3393
|
+
} else if (delta.kind === "unchanged") {
|
|
2991
3394
|
deps.log(
|
|
2992
3395
|
`The last reviewed commit ${short(last)} is not behind the head (${status}), but the pull request's changes are the same${delta.removed.length > 0 ? ` (no longer in it: ${delta.removed.join(", ")})` : ""}; rebased with no content change, nothing to review.`,
|
|
2993
3396
|
);
|
|
@@ -3001,8 +3404,7 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
3001
3404
|
comments: [],
|
|
3002
3405
|
});
|
|
3003
3406
|
return;
|
|
3004
|
-
}
|
|
3005
|
-
if (delta.kind === "changed") {
|
|
3407
|
+
} else if (delta.kind === "changed") {
|
|
3006
3408
|
deps.log(
|
|
3007
3409
|
`The last reviewed commit ${short(last)} is not behind the head (${status}); the branch was rewritten. ${delta.files.length} file(s) differ from what was reviewed, ${delta.unchanged} are the same and are not reviewed again.`,
|
|
3008
3410
|
);
|
|
@@ -3017,6 +3419,13 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
3017
3419
|
}
|
|
3018
3420
|
}
|
|
3019
3421
|
|
|
3422
|
+
// An incremental run adds the unread files to what changed since, each once and with its whole
|
|
3423
|
+
// patch. A run that fell back to the whole pull request (`since` is null) already holds them.
|
|
3424
|
+
const resumed = since !== null ? carried.map((f) => f.filename) : [];
|
|
3425
|
+
if (resumed.length > 0) {
|
|
3426
|
+
const unreadNames = new Set(resumed);
|
|
3427
|
+
files = [...files.filter((f) => !unreadNames.has(f.filename)), ...carried];
|
|
3428
|
+
}
|
|
3020
3429
|
const { kept, dropped } = filterFiles(files, [...DEFAULT_EXCLUDE, ...config.exclude], paths);
|
|
3021
3430
|
if (kept.length === 0) {
|
|
3022
3431
|
bodyOnly(
|
|
@@ -3042,6 +3451,15 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
3042
3451
|
});
|
|
3043
3452
|
return;
|
|
3044
3453
|
}
|
|
3454
|
+
// The size gate: what this run would send, priced before any model call. Nothing is posted as a
|
|
3455
|
+
// review and no marker is written, so the next push sees no marker and gates again.
|
|
3456
|
+
const estimate = estimateFor(allBinary.reviewable, config);
|
|
3457
|
+
const gate = sizeGate(config, isAutomatic(config.trigger));
|
|
3458
|
+
if (estimate.usd > gate) {
|
|
3459
|
+
deps.log(`Estimated $${estimate.usd.toFixed(2)} is above the $${gate.toFixed(2)} size gate; nothing sent to a model.`);
|
|
3460
|
+
postSizeGateNotice(deps, repo, pr, headSha, estimate, gate);
|
|
3461
|
+
return;
|
|
3462
|
+
}
|
|
3045
3463
|
// After the rebase exits above (they cost nothing and move the marker) and before any paid step.
|
|
3046
3464
|
const ceiling = ceilingRoom(deps, repo, pr, headSha);
|
|
3047
3465
|
if (ceiling === null) return;
|
|
@@ -3068,9 +3486,17 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
3068
3486
|
since,
|
|
3069
3487
|
rebased,
|
|
3070
3488
|
commits,
|
|
3489
|
+
carried: resumed,
|
|
3071
3490
|
threads,
|
|
3072
3491
|
});
|
|
3073
|
-
|
|
3492
|
+
// PR Confidence grades a full review only (ADR 0033): an incremental or scoped review reads a
|
|
3493
|
+
// part of the change, and "nothing else changed" cannot be said of a part.
|
|
3494
|
+
const fullReview = since === null && paths.length === 0;
|
|
3495
|
+
const options: ReviewOptions = {
|
|
3496
|
+
...reviewOptionsOf(config, totalBudget),
|
|
3497
|
+
...(config.confidence && fullReview ? { confidence: { budgetUsd: CLAIMS_BUDGET_USD } } : {}),
|
|
3498
|
+
};
|
|
3499
|
+
const result = reviewChange(input, options, deps);
|
|
3074
3500
|
const { usage, costUsdEstimate: spent } = result.execution;
|
|
3075
3501
|
const operation: Operation = since ? "re-review" : "review";
|
|
3076
3502
|
|
|
@@ -3087,6 +3513,7 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
3087
3513
|
operation,
|
|
3088
3514
|
extra: prior ? [prSoFar(prior, spent)] : [],
|
|
3089
3515
|
usage,
|
|
3516
|
+
estimate: onSubscription(deps),
|
|
3090
3517
|
costUsd: spent,
|
|
3091
3518
|
}),
|
|
3092
3519
|
SIGNATURE.auto,
|
|
@@ -3098,11 +3525,20 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
3098
3525
|
const { rejection, stop } = result.execution;
|
|
3099
3526
|
const refused = rejection && rejection.kind !== "other" ? { kind: rejection.kind, text: rejection.text } : undefined;
|
|
3100
3527
|
if (refused) {
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
auth
|
|
3104
|
-
|
|
3105
|
-
|
|
3528
|
+
// On a subscription token the key is not the credential: say what fits it, and that nothing falls back to a key.
|
|
3529
|
+
const headline = (
|
|
3530
|
+
deps.provider.auth === "oauth-token"
|
|
3531
|
+
? {
|
|
3532
|
+
billing: "**Not reviewed: the reviewer's Claude subscription refused the call (usage limit or plan access).** Re-run after the limit resets. Wachiman does not fall back to an API key.",
|
|
3533
|
+
auth: "**Not reviewed: the reviewer's Claude subscription token was rejected.** Check the `CLAUDE_CODE_OAUTH_TOKEN` secret (a token lasts a year; make a new one with `claude setup-token`), then re-run.",
|
|
3534
|
+
rate_limit: "**Not reviewed: Claude kept rate-limiting the reviewer.** Re-run later.",
|
|
3535
|
+
}
|
|
3536
|
+
: {
|
|
3537
|
+
billing: "**Not reviewed: the reviewer's Anthropic account has no credit.** Top up, then re-run this job.",
|
|
3538
|
+
auth: "**Not reviewed: the reviewer's API key was rejected.** Check the `ANTHROPIC_API_KEY` secret, then re-run.",
|
|
3539
|
+
rate_limit: "**Not reviewed: the Anthropic API kept rate-limiting the reviewer.** Re-run later.",
|
|
3540
|
+
}
|
|
3541
|
+
)[refused.kind];
|
|
3106
3542
|
// `oneLine` neutralises the text and cuts it at 300 characters.
|
|
3107
3543
|
const detail = oneLine(refused.text, 300);
|
|
3108
3544
|
spentNotice(detail ? `${headline}\n\n> ${detail}` : headline);
|
|
@@ -3136,6 +3572,9 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
3136
3572
|
since,
|
|
3137
3573
|
rebased,
|
|
3138
3574
|
commits,
|
|
3575
|
+
carried: resumed,
|
|
3576
|
+
tried: paths.length === 0 ? state?.tried : undefined,
|
|
3577
|
+
warn: deps.warn,
|
|
3139
3578
|
priorSpend: prior,
|
|
3140
3579
|
scoped: paths.length > 0,
|
|
3141
3580
|
mention: config.mention,
|
|
@@ -3144,16 +3583,27 @@ export function review(deps: Deps, repo: string, pr: number, words: string[]) {
|
|
|
3144
3583
|
files: changedInstructionFiles(prFiles.map((f) => f.filename)),
|
|
3145
3584
|
baseSha: instructions.baseSha,
|
|
3146
3585
|
},
|
|
3586
|
+
estimate: onSubscription(deps),
|
|
3147
3587
|
});
|
|
3148
3588
|
refuseLeak(deps, [
|
|
3149
3589
|
built.body,
|
|
3150
3590
|
...built.comments.map((c) => c.body),
|
|
3151
3591
|
...result.threadReplies.map((r) => r.reply),
|
|
3152
3592
|
]);
|
|
3153
|
-
postReview(deps, repo, pr, headSha, built);
|
|
3593
|
+
const reviewUrl = postReview(deps, repo, pr, headSha, built);
|
|
3594
|
+
// After the review, best-effort: the review is the record and is already posted.
|
|
3595
|
+
if (config.confidence) {
|
|
3596
|
+
publishConfidence({ gh, dryRun: config.dryRun, log: deps.log, warn: deps.warn, now: deps.now }, repo, pr, {
|
|
3597
|
+
headSha,
|
|
3598
|
+
fidelity: result.fidelity,
|
|
3599
|
+
partialReason: "This review covered only part of the pull request, so it cannot grade the whole change",
|
|
3600
|
+
reviewUrl,
|
|
3601
|
+
permalinkBase: `https://github.com/${repo}`,
|
|
3602
|
+
});
|
|
3603
|
+
}
|
|
3154
3604
|
// This run's cost and the pull request's total, under each reply. Not flagged automatic: the
|
|
3155
3605
|
// review body above carries the flagged line, so the ceiling would count the run twice.
|
|
3156
|
-
const replyCost = prior ? `\n\n<sub>this run $${spent.toFixed(2)} · ${prSoFar(prior, spent)}</sub>` : "";
|
|
3606
|
+
const replyCost = prior ? `\n\n<sub>this run $${spent.toFixed(2)} · ${prSoFar(prior, spent)}${estimateTail(onSubscription(deps))}</sub>` : "";
|
|
3157
3607
|
for (const r of result.threadReplies) {
|
|
3158
3608
|
const label = r.status === "addressed" ? "Looks addressed" : "Still open";
|
|
3159
3609
|
if (config.dryRun) {
|
|
@@ -3361,8 +3811,8 @@ export function threadReplies(deps: Deps, repo: string, pr: number, author?: str
|
|
|
3361
3811
|
const prior = { spent: ceiling.spent, runs: ceiling.runs };
|
|
3362
3812
|
const cost =
|
|
3363
3813
|
i === 0
|
|
3364
|
-
? `\n\n${costLine({ modelLabel: deps.provider.modelLabel(), effort: config.effort, operation: "reply", extra: [`answers ${toPost.length} thread(s)`, prSoFar(prior, call.costUsd)], usage: call.usage, costUsd: call.costUsd })}\n\n${SIGNATURE.auto}`
|
|
3365
|
-
: `\n\n<sub>this run $${call.costUsd.toFixed(2)} · ${prSoFar(prior, call.costUsd)}</sub>`;
|
|
3814
|
+
? `\n\n${costLine({ modelLabel: deps.provider.modelLabel(), effort: config.effort, operation: "reply", extra: [`answers ${toPost.length} thread(s)`, prSoFar(prior, call.costUsd)], usage: call.usage, estimate: onSubscription(deps), costUsd: call.costUsd })}\n\n${SIGNATURE.auto}`
|
|
3815
|
+
: `\n\n<sub>this run $${call.costUsd.toFixed(2)} · ${prSoFar(prior, call.costUsd)}${estimateTail(onSubscription(deps))}</sub>`;
|
|
3366
3816
|
const body = `${branded(`**${label}**`)} at \`${short(headSha)}\`: ${neutralise(reply.reply)}${cost}\n\n${SIGNATURE.reply}`;
|
|
3367
3817
|
if (config.dryRun) {
|
|
3368
3818
|
deps.log(`--- reply to ${reply.comment_id}: ${label} ---\n${body}`);
|
|
@@ -3381,6 +3831,9 @@ export interface ReviewContext {
|
|
|
3381
3831
|
since: string | null;
|
|
3382
3832
|
/** The branch was rewritten since `since`: the files are those whose changed lines differ. */
|
|
3383
3833
|
rebased?: boolean;
|
|
3834
|
+
/** The files reviewed whole because the previous review did not reach them, and how many files changed since. */
|
|
3835
|
+
carried?: readonly string[];
|
|
3836
|
+
changedSince?: number;
|
|
3384
3837
|
paths: string[];
|
|
3385
3838
|
dropped: number;
|
|
3386
3839
|
everyFile: string;
|
|
@@ -3388,6 +3841,8 @@ export interface ReviewContext {
|
|
|
3388
3841
|
meta: { title?: string; body?: string; author?: string };
|
|
3389
3842
|
/** `renderInstructions` of the base commit's rules. */
|
|
3390
3843
|
instructions: string;
|
|
3844
|
+
/** PR Confidence on: the claims the batches assign their files to, as the claims call listed them. */
|
|
3845
|
+
claims?: readonly string[];
|
|
3391
3846
|
/** How the cost log line names the change; absent for a change with no repository or number. */
|
|
3392
3847
|
costTags?: Pick<CallCost, "repo" | "pr">;
|
|
3393
3848
|
}
|
|
@@ -3431,12 +3886,20 @@ export function buildBatchPrompt(
|
|
|
3431
3886
|
evidence = "",
|
|
3432
3887
|
questions: readonly DataPathQuestion[] = selectQuestions(files),
|
|
3433
3888
|
) {
|
|
3434
|
-
const
|
|
3435
|
-
const
|
|
3889
|
+
const carried = new Set(ctx.carried ?? []);
|
|
3890
|
+
const list = files
|
|
3891
|
+
.map((f) => `- ${f.filename} +${f.additions} −${f.deletions}${carried.has(f.filename) ? " (not reached by the previous review: whole diff)" : ""}`)
|
|
3892
|
+
.join("\n");
|
|
3893
|
+
const sinceScope = ctx.since
|
|
3436
3894
|
? ctx.rebased
|
|
3437
3895
|
? `only the files whose changed lines differ from what was reviewed at commit ${ctx.since}, the last reviewed one (the branch was rebased since; a file's diff below is still its whole diff against the base)`
|
|
3438
3896
|
: `only the changes since commit ${ctx.since}, the last reviewed one`
|
|
3439
3897
|
: "the whole pull request";
|
|
3898
|
+
// A resumed review: the previous one stopped before these files, so they were never reviewed.
|
|
3899
|
+
// They come with their whole diff against the base, changed since or not.
|
|
3900
|
+
const unreached = `${carried.size} file(s) the previous review, at commit ${ctx.since}, did not reach: those files are marked in the lists below and are reviewed whole, because nothing of them was reviewed before, with their whole diff against the base`;
|
|
3901
|
+
const scope =
|
|
3902
|
+
ctx.since && carried.size > 0 ? ((ctx.changedSince ?? 0) > 0 ? `${sinceScope}, and ${unreached}` : `only ${unreached}`) : sinceScope;
|
|
3440
3903
|
return `You are reviewing ${ctx.describe}, checked out at its head commit ${ctx.headSha}.
|
|
3441
3904
|
The repository's rules are in the "Repository instructions" section below; the ADRs in docs/adr/ hold its decisions.
|
|
3442
3905
|
|
|
@@ -3476,7 +3939,7 @@ finding that already has an open thread.
|
|
|
3476
3939
|
`
|
|
3477
3940
|
: ""
|
|
3478
3941
|
}
|
|
3479
|
-
${RISK_PARAGRAPH}
|
|
3942
|
+
${RISK_PARAGRAPH}${ctx.claims ? `\n\n${claimsParagraph(ctx.claims)}` : ""}
|
|
3480
3943
|
|
|
3481
3944
|
${FLOW_QUESTIONS}${renderDataPathQuestions(questions)}
|
|
3482
3945
|
|
|
@@ -3505,8 +3968,18 @@ export function relativisePaths(output: ReviewOutput, repoDir: string): ReviewOu
|
|
|
3505
3968
|
// `wellFormed` drops it later, and it must not throw here.
|
|
3506
3969
|
const stripItemPath = <T extends { path: string }>(item: T): T =>
|
|
3507
3970
|
item !== null && typeof item === "object" && typeof item.path === "string" ? { ...item, path: strip(item.path) } : item;
|
|
3971
|
+
// Same rule for a claim assignment: a path that is not a string is left for `assembleClaims` to drop.
|
|
3972
|
+
const stripAssignment = (a: ClaimAssignment): ClaimAssignment =>
|
|
3973
|
+
a !== null && typeof a === "object"
|
|
3974
|
+
? {
|
|
3975
|
+
...a,
|
|
3976
|
+
path: typeof a.path === "string" ? strip(a.path) : a.path,
|
|
3977
|
+
evidence: Array.isArray(a.evidence) ? a.evidence.map(stripItemPath) : a.evidence,
|
|
3978
|
+
}
|
|
3979
|
+
: a;
|
|
3508
3980
|
return {
|
|
3509
3981
|
...output,
|
|
3982
|
+
...(Array.isArray(output.claim_assignments) ? { claim_assignments: output.claim_assignments.map(stripAssignment) } : {}),
|
|
3510
3983
|
files_reviewed: output.files_reviewed.map(strip),
|
|
3511
3984
|
files_skipped: output.files_skipped.map((s) => ({ ...s, path: strip(s.path) })),
|
|
3512
3985
|
findings: output.findings.map((f) => ({ ...f, path: strip(f.path) })),
|
|
@@ -3649,7 +4122,7 @@ function reviewBatch(
|
|
|
3649
4122
|
prompt,
|
|
3650
4123
|
dir,
|
|
3651
4124
|
budgetUsd,
|
|
3652
|
-
SCHEMA,
|
|
4125
|
+
ctx.claims ? { ...SCHEMA, properties: { ...SCHEMA.properties, claim_assignments: ASSIGNMENTS_SCHEMA } } : SCHEMA,
|
|
3653
4126
|
),
|
|
3654
4127
|
budgetUsd,
|
|
3655
4128
|
);
|
|
@@ -3741,6 +4214,7 @@ export function runProvider(
|
|
|
3741
4214
|
// (`attempt`); record it that way here, still as a table row and a log line, then fail as before.
|
|
3742
4215
|
const dead: CallCost = {
|
|
3743
4216
|
...what,
|
|
4217
|
+
...(deps.provider.auth ? { auth: deps.provider.auth } : {}),
|
|
3744
4218
|
model: limits.model,
|
|
3745
4219
|
turns: 0,
|
|
3746
4220
|
durationMs: Date.now() - started,
|
|
@@ -3778,10 +4252,12 @@ export function runProvider(
|
|
|
3778
4252
|
durationMs: result.durationMs,
|
|
3779
4253
|
providerError: result.providerError,
|
|
3780
4254
|
errorText: result.errorText,
|
|
4255
|
+
credential: deps.provider.auth,
|
|
3781
4256
|
}),
|
|
3782
4257
|
);
|
|
3783
4258
|
const cost: CallCost = {
|
|
3784
4259
|
...what,
|
|
4260
|
+
...(deps.provider.auth ? { auth: deps.provider.auth } : {}),
|
|
3785
4261
|
model: result.model ?? limits.model,
|
|
3786
4262
|
turns: result.turns,
|
|
3787
4263
|
durationMs: result.durationMs,
|
|
@@ -3954,6 +4430,7 @@ export function answer(deps: Deps, repo: string, pr: number, question: string, o
|
|
|
3954
4430
|
effort: deps.config.effort,
|
|
3955
4431
|
operation: "on-demand",
|
|
3956
4432
|
usage: result?.usage,
|
|
4433
|
+
estimate: onSubscription(deps),
|
|
3957
4434
|
costUsd,
|
|
3958
4435
|
});
|
|
3959
4436
|
const signed = asker === undefined ? SIGNATURE.answer : `${SIGNATURE.ask}\n\n${SIGNATURE.answer}`;
|
|
@@ -3995,7 +4472,7 @@ export function answer(deps: Deps, repo: string, pr: number, question: string, o
|
|
|
3995
4472
|
function postAskLimit(deps: Deps, repo: string, pr: number, spent: number) {
|
|
3996
4473
|
const body = [
|
|
3997
4474
|
branded(
|
|
3998
|
-
`The question budget for this pull request is used up: questions from collaborators have spent $${spent.toFixed(2)} of the $${deps.config.askCeilingUsd.toFixed(2)} ceiling, so this question is not answered. The repository operator can still ask
|
|
4475
|
+
`The question budget for this pull request is used up: questions from collaborators have spent $${spent.toFixed(2)} of the $${deps.config.askCeilingUsd.toFixed(2)} ceiling, so this question is not answered. The repository operator can still ask.${onSubscription(deps) ? " These are API-equivalent estimates: the reviews run on a Claude subscription, not a bill." : ""}`,
|
|
3999
4476
|
),
|
|
4000
4477
|
SIGNATURE.answer,
|
|
4001
4478
|
SIGNATURE.askLimit,
|