ticket-runner 0.6.1 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,13 @@
1
1
  import { branchName, worktreePath } from "./branch.js";
2
2
  import type { Config } from "./config.js";
3
+ import {
4
+ CREDENTIAL_WORDS,
5
+ type Credential,
6
+ type StrayStage,
7
+ expectedKeySource,
8
+ } from "./credential.js";
3
9
  import { tickMetCriteria } from "./criteria.js";
10
+ import type { BaseBranchChecks, FailingBase } from "./failing-base.js";
4
11
  import { type SkipReason, isGuardReason, skipReason } from "./guards.js";
5
12
  import { markHandoffsTaken } from "./handoff.js";
6
13
  import type { HostKind } from "./host.js";
@@ -23,6 +30,7 @@ import type { Issue, Tracker } from "./ports/tracker.js";
23
30
  import type {
24
31
  KeptTranscripts,
25
32
  ReachedState,
33
+ RebaseOutcome,
26
34
  TicketState,
27
35
  Workspace,
28
36
  WorktreeFromRemote,
@@ -38,7 +46,11 @@ import {
38
46
  import { retryLogDir, stageLogDir } from "./run-log.js";
39
47
  import { holdsClaim } from "./stranded.js";
40
48
  import {
49
+ RELEASE_WORDS,
50
+ releaseWords,
41
51
  draftPullRequestBody,
52
+ failingBaseNote,
53
+ failingBaseSummary,
42
54
  guardComment,
43
55
  handoffComment,
44
56
  hasGuardWarning,
@@ -81,8 +93,17 @@ export type TicketOutcome =
81
93
  ticket: number;
82
94
  title: string;
83
95
  branch: string;
84
- /** Where the rate limit landed, which is all the summary says about it. */
96
+ /** Where the Stage was stopped, which is most of what the summary says about it. */
85
97
  stage: FailurePoint;
98
+ /** What stopped it, which is the rest. */
99
+ cause: ReleaseCause;
100
+ /**
101
+ * When the rate limit resets, ISO-8601, where the Stage it stopped was
102
+ * told. Absent for an API failure, which says nothing of when it ends.
103
+ */
104
+ limitResetsAt?: string;
105
+ /** The Check the Base branch fails, and where, for a Release over it. */
106
+ failingBase?: { commit: string; command: string };
86
107
  notes: RoutedNote[];
87
108
  }
88
109
  | {
@@ -112,6 +133,17 @@ export interface Pipeline {
112
133
  * can point at the Run's directory: a cloud Host's is gone with its session.
113
134
  */
114
135
  host: HostKind;
136
+ /**
137
+ * What every Stage of this Run is billed to, chosen once when it started:
138
+ * the subscription, or the Host's API key for a Run started with `--api-key`.
139
+ */
140
+ credential: Credential;
141
+ /**
142
+ * Every Stage of this Run that said it ran on something other than
143
+ * {@link credential}, in the order they came back, shared by every Ticket so
144
+ * the Run summary can name them all.
145
+ */
146
+ strayStages: StrayStage[];
115
147
  /**
116
148
  * The branch every Ticket of this Run branches from, rebases onto, merges
117
149
  * into and pulls, resolved once before the Run started
@@ -130,6 +162,12 @@ export interface Pipeline {
130
162
  * them are told which one it is.
131
163
  */
132
164
  standingNotes: StandingNotes;
165
+ /**
166
+ * The Checks the Run ran at merge-bases with the Base branch, made once and
167
+ * shared by every Ticket: the Tickets that fail together over one broken
168
+ * Base branch run the Check there once between them, and post one Note.
169
+ */
170
+ baseChecks: BaseBranchChecks;
133
171
  /**
134
172
  * Whether the Target is the pipeline's own repository, decided once before
135
173
  * the Run started ({@link import("./self-hosting.js").isPipelineRepository}).
@@ -157,21 +195,45 @@ class TicketFailure extends Error {
157
195
  }
158
196
 
159
197
  /**
160
- * A Stage the subscription rate limit stopped.
198
+ * What stops a Ticket that says nothing about it: the rate limit, a transient
199
+ * API failure — the API overloaded, broken or silent on its own side after the
200
+ * CLI had already retried it — the API key of an `--api-key` Run out of credit
201
+ * or past its workspace's spend limit, and a Base branch that fails the Check
202
+ * the Ticket's branch failed.
203
+ */
204
+ export type ReleaseCause = StageReleaseCause | "failing-base";
205
+
206
+ /** The Stage failures among them. */
207
+ type StageReleaseCause = Extract<StageFailure, "rate-limited" | "api-error" | "out-of-budget">;
208
+
209
+ function releaseCause(result: StageResult): StageReleaseCause | undefined {
210
+ return result.failure === "rate-limited" ||
211
+ result.failure === "api-error" ||
212
+ result.failure === "out-of-budget"
213
+ ? result.failure
214
+ : undefined;
215
+ }
216
+
217
+ /**
218
+ * A Stage something other than the Ticket stopped, on its way to a Release.
161
219
  *
162
- * The one Stage failure that says nothing about the Ticket, so it is the one
163
- * that neither ends it nor spends its Fix budget: the Claim is released and a
164
- * State file is left for the next Run to resume from (ADR-0004). It is not a
220
+ * The Stage failures that say nothing about the Ticket are the ones that
221
+ * neither end it nor spend its Fix budget: the Claim is released and a State
222
+ * file is left for the next Run to resume from (ADR-0004). It is not a
165
223
  * {@link TicketFailure} because every path that handles one of those would
166
224
  * either hand the Ticket off or buy it a fix Stage, and this is neither.
167
225
  */
168
- class RateLimited extends Error {
226
+ class Releasing extends Error {
169
227
  constructor(
170
228
  readonly point: FailurePoint,
171
229
  /** The state the Ticket reached, which is where a later Run picks it up. */
172
230
  readonly state: ReachedState,
231
+ readonly reason: ReleaseCause,
232
+ readonly limitResetsAt?: string,
233
+ /** The Check the Base branch fails too, for a Release over it. */
234
+ readonly failingBase?: FailingBase,
173
235
  ) {
174
- super(`the ${point} Stage hit the subscription rate limit`);
236
+ super(`the ${point} Stage was stopped · ${RELEASE_WORDS[reason]}`);
175
237
  }
176
238
  }
177
239
 
@@ -194,9 +256,10 @@ function asTicketFailure(error: unknown, point: FailurePoint): TicketFailure {
194
256
  * Nothing ends in a merge that has not been through a green pass of the whole
195
257
  * gauntlet.
196
258
  *
197
- * A Stage the subscription rate limit stopped ends none of that: the Ticket is
198
- * released, and a Run started once the limit has reset resumes it from the state
199
- * it had reached rather than from the top. A hand-off keeps that state too, so a
259
+ * A Stage the subscription rate limit or a transient API failure stopped ends
260
+ * none of that: the Ticket is released, and a Run started once the limit has
261
+ * reset, or the API recovered, resumes it from the state it had reached rather
262
+ * than from the top. A hand-off keeps that state too, so a
200
263
  * human who relabels the Ticket `ready-for-agent` hands it back and the next Run
201
264
  * carries on from what it reached rather than paying for the implement Stage a
202
265
  * second time.
@@ -289,6 +352,7 @@ async function takeTicket(
289
352
  fixUsed: resume?.fixUsed ?? false,
290
353
  ...(resume?.pullRequest === undefined ? {} : { pullRequest: resume.pullRequest }),
291
354
  ...(resume?.title === undefined ? {} : { title: resume.title }),
355
+ ...(resume?.rebaseFirst === true ? { rebaseFirst: true } : {}),
292
356
  });
293
357
 
294
358
  // The claim is the first write on the board: it shows what the pipeline holds
@@ -327,6 +391,10 @@ async function takeTicket(
327
391
  let fixUsed = resume?.fixUsed ?? false;
328
392
  // The pass behind a fix Stage logs beside the first one rather than over it.
329
393
  let logDir = stageLogDir(repoRoot, runId, ticket);
394
+ // The unmet criteria the fix Stage was sent in for, and their evidence, which
395
+ // the verify behind it is held to. Only this Run's: the State file keeps no
396
+ // Verdict, so a retry a later Run resumes is graded without them.
397
+ let unmetBefore: string | undefined;
330
398
 
331
399
  // Declared out here because ticking the Acceptance Criteria it proved happens
332
400
  // after the merge, where a failure may no longer hand the Ticket off.
@@ -388,18 +456,28 @@ async function takeTicket(
388
456
  let commits: string[];
389
457
  let coAuthors: string[];
390
458
  let title: string;
459
+ // Owed once, by a Ticket released over a failing Base branch, and inside
460
+ // the passes rather than before them: a conflict it meets spends the Fix
461
+ // budget as one at the Landing does.
462
+ let rebaseOwed = resume?.rebaseFirst === true;
391
463
 
392
464
  // Checks through CI, run again from the top when the fix budget buys a
393
465
  // second pass: a fix earns no shortcut, so every gate grades it afresh.
394
466
  for (;;) {
395
467
  try {
468
+ if (rebaseOwed) {
469
+ rebaseOwed = false;
470
+ point = "rebase";
471
+ await rebaseFirst(pipeline, issue, worktree, branch, logDir, landing, progress);
472
+ await record.advance({ rebaseFirst: false });
473
+ }
396
474
  point = "checks";
397
475
  // Before anything grades the worktree, and so before verify discards
398
476
  // what a Stage left uncommitted.
399
477
  await requireCommitted(pipeline, worktree, progress);
400
478
  await runChecks(pipeline, worktree, progress);
401
479
  point = "verify";
402
- verdict = await verify(pipeline, issue, worktree, logDir, progress, notes);
480
+ verdict = await verify(pipeline, issue, worktree, logDir, progress, notes, unmetBefore);
403
481
 
404
482
  point = "rebase";
405
483
  // The Landing, from here until the Base branch has been pulled: the
@@ -448,12 +526,12 @@ async function takeTicket(
448
526
  );
449
527
  await record.advance({ pullRequest });
450
528
  point = "ci";
451
- await requireGreenCi(pipeline, pullRequest, progress);
529
+ await requireGreenCi(pipeline, worktree, pullRequest, progress, landing);
452
530
  break;
453
531
  } catch (error) {
454
- // A rate limit is nobody's defect, so the fix budget does not answer
455
- // for it: the Ticket is released, whole, further up.
456
- if (error instanceof RateLimited) throw error;
532
+ // A rate limit or an API failure is nobody's defect, so the fix budget
533
+ // does not answer for it: the Ticket is released, whole, further up.
534
+ if (error instanceof Releasing) throw error;
457
535
  const failure = asTicketFailure(error, point);
458
536
  if (fixUsed || failure.kind === undefined) throw failure;
459
537
  fixUsed = true;
@@ -463,6 +541,7 @@ async function takeTicket(
463
541
  // the queue at the back when it reaches the rebase again.
464
542
  landing.leave();
465
543
  logDir = retryLogDir(repoRoot, runId, ticket);
544
+ if (failure.kind === "unmet-criteria") unmetBefore = failure.evidence;
466
545
  await fix(
467
546
  pipeline,
468
547
  issue,
@@ -475,8 +554,8 @@ async function takeTicket(
475
554
  record,
476
555
  );
477
556
  // Recorded once the Stage has come back, not when the budget was
478
- // committed: a fix Stage the rate limit stopped before it ran spends
479
- // nothing, and the release below says so.
557
+ // committed: a fix Stage the rate limit or an API failure stopped
558
+ // before it ran spends nothing, and the release below says so.
480
559
  await record.advance({ fixUsed: true });
481
560
  }
482
561
  }
@@ -492,15 +571,15 @@ async function takeTicket(
492
571
  // pull request and a Release's labels are nothing for the next Ticket to
493
572
  // wait behind, and a Ticket that never reached the rebase gives up nothing.
494
573
  landing.leave();
495
- if (error instanceof RateLimited) {
574
+ if (error instanceof Releasing) {
496
575
  return release(pipeline, {
497
576
  issue,
498
577
  user,
499
578
  branch,
500
- limit: error,
579
+ stopped: error,
501
580
  record,
502
- // A fix Stage the limit stopped before it ran spends nothing: the
503
- // budget is still there for the Run that resumes the Ticket.
581
+ // A fix Stage stopped before it ran spends nothing: the budget is
582
+ // still there for the Run that resumes the Ticket.
504
583
  fixUsed: fixUsed && error.point !== "fix",
505
584
  notes,
506
585
  });
@@ -703,7 +782,9 @@ function howItWasTaken(resume: TicketState | undefined, stranded: boolean): stri
703
782
  type Reached = Omit<TicketState, "runId" | "updatedAt">;
704
783
 
705
784
  /** What one write moves that on by. */
706
- type Advance = Partial<Pick<Reached, "state" | "fixUsed" | "pullRequest" | "title">>;
785
+ type Advance = Partial<
786
+ Pick<Reached, "state" | "fixUsed" | "pullRequest" | "title" | "rebaseFirst">
787
+ >;
707
788
 
708
789
  /**
709
790
  * The State file a Ticket keeps, and the one place a Ticket writes it.
@@ -776,8 +857,12 @@ class ResumeRecord {
776
857
  * read one can still name what wrote it (ADR-0007).
777
858
  */
778
859
  private file(): TicketState {
860
+ // Written only while it is owed, so a State file says nothing of a rebase
861
+ // nobody has to do.
862
+ const { rebaseFirst, ...reached } = this.reached;
779
863
  return {
780
- ...this.reached,
864
+ ...reached,
865
+ ...(rebaseFirst === true ? { rebaseFirst } : {}),
781
866
  runId: this.pipeline.runId,
782
867
  version: this.pipeline.version,
783
868
  updatedAt: new Date().toISOString(),
@@ -819,11 +904,16 @@ async function passOver(
819
904
  * Start one Stage under the model, effort, turn and wall-clock limits its own
820
905
  * config names. Everything a Stage differs in is the prompt, where it runs and
821
906
  * where it logs; the limits come from one place so no Stage can quietly skip them.
907
+ *
908
+ * The credential is the Run's too, and what the Stage says it actually ran on
909
+ * is checked here for the same reason: a Stage billed somewhere the Run did not
910
+ * choose is named whichever Stage it was.
822
911
  */
823
- function runStage(
912
+ async function runStage(
824
913
  pipeline: Pipeline,
825
914
  stage: StageName,
826
915
  request: {
916
+ ticket: number;
827
917
  prompt: string;
828
918
  cwd: string;
829
919
  logDir: string;
@@ -832,12 +922,13 @@ function runStage(
832
922
  },
833
923
  ): Promise<StageResult> {
834
924
  const limits = pipeline.config.stages[stage];
835
- return pipeline.runner.run({
925
+ const result = await pipeline.runner.run({
836
926
  stage,
837
927
  prompt: request.prompt,
838
928
  cwd: request.cwd,
839
929
  logDir: request.logDir,
840
930
  permissionMode: pipeline.config.permissionMode,
931
+ credential: pipeline.credential,
841
932
  model: limits.model,
842
933
  effort: limits.effort,
843
934
  maxTurns: limits.maxTurns,
@@ -847,10 +938,35 @@ function runStage(
847
938
  ? {}
848
939
  : { resultRequired: request.resultRequired }),
849
940
  });
941
+ noteStrayCredential(pipeline, request.ticket, stage, result);
942
+ return result;
850
943
  }
851
944
 
852
945
  /**
853
- * Report a Stage the rate limit stopped, and raise the Ticket's release.
946
+ * Name a Stage whose `system/init` event reported another `apiKeySource` than
947
+ * the Run's credential expects, in the log now and in the summary at the end.
948
+ * A Stage that reported none, one that never started, is not named: there is
949
+ * nothing it said to set against the Run's choice.
950
+ */
951
+ function noteStrayCredential(
952
+ pipeline: Pipeline,
953
+ ticket: number,
954
+ stage: StageName,
955
+ result: StageResult,
956
+ ): void {
957
+ const { apiKeySource } = result;
958
+ const expected = expectedKeySource(pipeline.credential);
959
+ if (apiKeySource === undefined || apiKeySource === expected) return;
960
+ pipeline.strayStages.push({ ticket, stage, apiKeySource });
961
+ pipeline.log?.(
962
+ `#${ticket} ${stage} ran on apiKeySource ${apiKeySource}, ` +
963
+ `not the ${CREDENTIAL_WORDS[pipeline.credential]} (${expected})`,
964
+ );
965
+ }
966
+
967
+ /**
968
+ * Report a Stage the rate limit or an API failure stopped, and raise the
969
+ * Ticket's release.
854
970
  *
855
971
  * The one place the pause is composed, because the conflict Stage reaches it by
856
972
  * another road: the row reads as a pause rather than a cross, and the error
@@ -859,36 +975,49 @@ function runStage(
859
975
  * work that is already on the branch.
860
976
  */
861
977
  async function releasedStage(
978
+ pipeline: Pipeline,
862
979
  progress: Progress,
863
980
  stage: StageName,
864
981
  point: FailurePoint,
865
982
  result: StageResult,
866
- ): Promise<RateLimited> {
867
- await progress.record(stageRow(stage, result, "⏸ rate limited"));
868
- return new RateLimited(point, stage === "implement" ? "claimed" : "implemented");
983
+ cause: StageReleaseCause,
984
+ ): Promise<Releasing> {
985
+ await progress.record(
986
+ stageRow(stage, result, `⏸ ${stageFailureCell(result, pipeline.credential)}`),
987
+ );
988
+ return new Releasing(
989
+ point,
990
+ stage === "implement" ? "claimed" : "implemented",
991
+ cause,
992
+ result.limitResetsAt,
993
+ );
869
994
  }
870
995
 
871
996
  /**
872
997
  * Report a Stage that did not come back, and say what it costs the Ticket.
873
998
  *
874
- * The rate limit is the one Stage failure the Ticket did nothing to earn, so it
875
- * releases the Ticket instead of ending it. Every other failure is the Stage's
876
- * own, and its caller decides whether the fix budget can buy another.
999
+ * The rate limit and a transient API failure are the Stage failures the Ticket
1000
+ * did nothing to earn, so they release the Ticket instead of ending it. Every
1001
+ * other failure is the Stage's own, and its caller decides whether the fix
1002
+ * budget can buy another.
877
1003
  */
878
1004
  async function stageDidNotFinish(
879
1005
  pipeline: Pipeline,
880
1006
  progress: Progress,
881
1007
  stage: "implement" | "verify" | "fix",
882
1008
  result: StageResult,
883
- ): Promise<TicketFailure | RateLimited> {
884
- if (result.failure === "rate-limited") {
885
- return await releasedStage(progress, stage, stage, result);
1009
+ ): Promise<TicketFailure | Releasing> {
1010
+ const cause = releaseCause(result);
1011
+ if (cause !== undefined) {
1012
+ return await releasedStage(pipeline, progress, stage, stage, result, cause);
886
1013
  }
887
1014
 
888
- await progress.record(stageRow(stage, result, `❌ ${stageFailureCell(result)}`));
1015
+ await progress.record(
1016
+ stageRow(stage, result, `❌ ${stageFailureCell(result, pipeline.credential)}`),
1017
+ );
889
1018
  return new TicketFailure(
890
1019
  stage,
891
- describeStageFailure(stage, pipeline.config.stages[stage], result),
1020
+ describeStageFailure(stage, pipeline.config.stages[stage], result, pipeline.credential),
892
1021
  );
893
1022
  }
894
1023
 
@@ -996,11 +1125,13 @@ async function implement(
996
1125
  ): Promise<void> {
997
1126
  const stage = pipeline.config.stages.implement;
998
1127
  const result = await runStage(pipeline, "implement", {
1128
+ ticket: issue.number,
999
1129
  prompt: implementPrompt(
1000
1130
  issue.url,
1001
1131
  pipeline.baseBranch,
1002
1132
  stage.extraPrompt,
1003
1133
  pipeline.selfHosted,
1134
+ pipeline.config.checks,
1004
1135
  await standingNotesNumber(pipeline),
1005
1136
  ),
1006
1137
  cwd: worktree,
@@ -1015,15 +1146,15 @@ async function implement(
1015
1146
  await recordTitle(result, record);
1016
1147
 
1017
1148
  // Pushed whatever became of the Stage, like its Notes: a session that
1018
- // committed and then ran into the rate limit still committed, and the Host
1019
- // that resumes the Ticket may be another one.
1149
+ // committed and then was stopped by the rate limit or an API failure still
1150
+ // committed, and the Host that resumes the Ticket may be another one.
1020
1151
  const commits = await commitCount(pipeline, branch);
1021
1152
  if (commits > 0) await pushCommitted(pipeline, issue.number, worktree, branch);
1022
1153
  if (!result.ok) throw await stageDidNotFinish(pipeline, progress, "implement", result);
1023
1154
 
1024
1155
  // An agent that gave up silently leaves a clean branch behind. That is a
1025
1156
  // failure, not something to verify. A resumed Ticket's branch is not clean —
1026
- // it carries what the Stage the rate limit stopped had committed — so this
1157
+ // it carries what the Stage a Release stopped had committed — so this
1027
1158
  // asks the same question there: is there anything at all to grade. Whether it
1028
1159
  // is enough is the Verdict's business, not this guard's.
1029
1160
  if (commits === 0) {
@@ -1037,13 +1168,53 @@ async function implement(
1037
1168
  await progress.record(stageRow("implement", result, "✅ committed"));
1038
1169
  }
1039
1170
 
1171
+ /**
1172
+ * Bring a Ticket released over a failing Base branch up to the Base branch
1173
+ * before its Checks grade it, so a fix a human has landed there meanwhile
1174
+ * reaches the branch. Without it the Ticket would meet the same failure with
1175
+ * its budget unspent, and spend it.
1176
+ *
1177
+ * The pull and the rebase go through the Landing, which is where the Base
1178
+ * branch is brought up to the remote's and rebased onto (ADR-0005), and out of
1179
+ * it again at once: nothing is merged behind them. A conflict goes to the
1180
+ * Conflict Stage as one at the Landing does, outside it, since the rebase
1181
+ * there runs again before the pull request anyway. Left unresolved, the
1182
+ * branch would keep the merge-base that fails and be released again by every
1183
+ * Run that took it up.
1184
+ */
1185
+ async function rebaseFirst(
1186
+ pipeline: Pipeline,
1187
+ issue: Issue,
1188
+ worktree: string,
1189
+ branch: string,
1190
+ logDir: string,
1191
+ landing: LandingTurn,
1192
+ progress: Progress,
1193
+ ): Promise<void> {
1194
+ const { workspace, baseBranch } = pipeline;
1195
+ await landing.enter();
1196
+ let rebase: RebaseOutcome;
1197
+ try {
1198
+ await workspace.pullBase(baseBranch);
1199
+ rebase = await workspace.rebase(worktree, baseBranch);
1200
+ } finally {
1201
+ landing.leave();
1202
+ }
1203
+ if (!rebase.ok) {
1204
+ await resolveConflict(pipeline, issue, worktree, branch, logDir, rebase.conflict, progress);
1205
+ return;
1206
+ }
1207
+ await pushCommitted(pipeline, issue.number, worktree, branch);
1208
+ pipeline.log?.(`#${issue.number} rebased onto ${baseBranch} before its Checks`);
1209
+ }
1210
+
1040
1211
  /**
1041
1212
  * Refuse a worktree holding changes no commit carries, naming every path.
1042
1213
  *
1043
1214
  * Nothing is discarded here: a fix Stage sent in for it decides what belongs
1044
1215
  * to the Ticket, and a hand-off leaves the changes where a human can find them.
1045
1216
  * The summary names no Stage, because a resumed Ticket cannot say which one
1046
- * left them: a fix Stage the rate limit stopped spent no budget.
1217
+ * left them: a fix Stage a Release stopped spent no budget.
1047
1218
  */
1048
1219
  async function requireCommitted(
1049
1220
  pipeline: Pipeline,
@@ -1074,13 +1245,23 @@ async function runChecks(
1074
1245
  for (const command of pipeline.config.checks) {
1075
1246
  const result = await pipeline.workspace.runCheck(command, worktree, minutes * 60_000);
1076
1247
  if (!result.ok) {
1248
+ const durationMs = Date.now() - startedAt;
1249
+ // Asked before the row is written, because the row is what differs: a
1250
+ // Check the Base branch fails too is nothing the Ticket did.
1251
+ await releaseIfTheBaseFails(pipeline, {
1252
+ point: "checks",
1253
+ worktree,
1254
+ commands: [command],
1255
+ progress,
1256
+ durationMs,
1257
+ });
1077
1258
  // A Check that hung and one that failed are the same failure to the fix
1078
1259
  // budget and the hand-off; they are told apart only in what they say, so
1079
1260
  // that a fix Stage knows whether it is mending a hang or an assertion.
1080
1261
  await progress.record({
1081
1262
  point: "checks",
1082
1263
  outcome: `❌ \`${command}\` ${result.timedOut ? "timed out" : "failed"}`,
1083
- durationMs: Date.now() - startedAt,
1264
+ durationMs,
1084
1265
  });
1085
1266
  throw new TicketFailure(
1086
1267
  "checks",
@@ -1097,6 +1278,75 @@ async function runChecks(
1097
1278
  });
1098
1279
  }
1099
1280
 
1281
+ /**
1282
+ * The first of `commands` the Base branch fails, at the commit the branch at
1283
+ * `worktree` inherited from it, or nothing when it passes them all.
1284
+ *
1285
+ * Nothing, too, when the merge-base could not be graded at all: the Ticket is
1286
+ * then charged for the failure as it always was, which costs a fix Stage at
1287
+ * worst, where releasing on a guess would leave a Ticket's own defect unmended
1288
+ * and stop the Run.
1289
+ */
1290
+ async function inheritedFailure(
1291
+ pipeline: Pipeline,
1292
+ worktree: string,
1293
+ commands: readonly string[],
1294
+ ): Promise<FailingBase | undefined> {
1295
+ try {
1296
+ const commit = await pipeline.workspace.mergeBase(worktree, pipeline.baseBranch);
1297
+ return await pipeline.baseChecks.firstFailing(
1298
+ {
1299
+ workspace: pipeline.workspace,
1300
+ repoRoot: pipeline.repoRoot,
1301
+ timeoutMs: pipeline.config.checkTimeoutMinutes * 60_000,
1302
+ ...(pipeline.log === undefined ? {} : { log: pipeline.log }),
1303
+ },
1304
+ commit,
1305
+ commands,
1306
+ );
1307
+ } catch (error) {
1308
+ pipeline.log?.(
1309
+ `could not run the Checks at the merge-base with ${pipeline.baseBranch}: ${(error as Error).message}`,
1310
+ );
1311
+ return undefined;
1312
+ }
1313
+ }
1314
+
1315
+ /**
1316
+ * Release the Ticket when the Base branch fails one of `commands` too, with the
1317
+ * row that says so; return when it does not, for the caller to charge the
1318
+ * failure to the Ticket as it always did.
1319
+ *
1320
+ * Only a Ticket that reached its Checks gets here, so the implement Stage's
1321
+ * work is always on its branch.
1322
+ */
1323
+ async function releaseIfTheBaseFails(
1324
+ pipeline: Pipeline,
1325
+ {
1326
+ point,
1327
+ worktree,
1328
+ commands,
1329
+ progress,
1330
+ durationMs,
1331
+ }: {
1332
+ point: "checks" | "ci";
1333
+ worktree: string;
1334
+ commands: readonly string[];
1335
+ progress: Progress;
1336
+ /** How long the failure took on the branch, which the row reports. */
1337
+ durationMs: number;
1338
+ },
1339
+ ): Promise<void> {
1340
+ const inherited = await inheritedFailure(pipeline, worktree, commands);
1341
+ if (inherited === undefined) return;
1342
+ await progress.record({
1343
+ point,
1344
+ outcome: `⏸ \`${inherited.command}\` fails on the Base branch`,
1345
+ durationMs,
1346
+ });
1347
+ throw new Releasing(point, "implemented", "failing-base", undefined, inherited);
1348
+ }
1349
+
1100
1350
  /**
1101
1351
  * What the Check printed before the kill, and then a line saying that a kill is
1102
1352
  * what ended it. A fix Stage is given the evidence and little else, so the
@@ -1116,14 +1366,19 @@ async function verify(
1116
1366
  logDir: string,
1117
1367
  progress: Progress,
1118
1368
  notes: RoutedNote[],
1369
+ /** The unmet criteria and evidence of the Verdict a fix Stage was sent in for. */
1370
+ unmetBefore: string | undefined,
1119
1371
  ): Promise<Verdict> {
1120
1372
  const stage = pipeline.config.stages.verify;
1121
1373
  const result = await runStage(pipeline, "verify", {
1374
+ ticket: issue.number,
1122
1375
  prompt: verifyPrompt(
1123
1376
  issue.url,
1124
1377
  stage.extraPrompt,
1125
1378
  pipeline.selfHosted,
1379
+ pipeline.config.checks,
1126
1380
  await standingNotesNumber(pipeline),
1381
+ unmetBefore,
1127
1382
  ),
1128
1383
  cwd: worktree,
1129
1384
  logDir,
@@ -1209,12 +1464,14 @@ async function fix(
1209
1464
  const commitsBefore = await commitCount(pipeline, branch);
1210
1465
 
1211
1466
  const result = await runStage(pipeline, "fix", {
1467
+ ticket: issue.number,
1212
1468
  prompt: fixPrompt(
1213
1469
  issue.url,
1214
1470
  failure,
1215
1471
  pipeline.baseBranch,
1216
1472
  stage.extraPrompt,
1217
1473
  pipeline.selfHosted,
1474
+ pipeline.config.checks,
1218
1475
  await standingNotesNumber(pipeline),
1219
1476
  ),
1220
1477
  cwd: worktree,
@@ -1239,9 +1496,17 @@ async function fix(
1239
1496
  // A branch that grew is the whole signal, which is not the same as one that
1240
1497
  // changed: a session that squashed the branch shorter, or amended in place,
1241
1498
  // is read here as having committed nothing.
1499
+ //
1500
+ // The hand-off carries the evidence the Stage was sent in with. Committing
1501
+ // nothing is also how a fix Stage hands back an unmet criterion only a human
1502
+ // can settle, and the human then reads the question where it was asked.
1242
1503
  if (!grew) {
1243
1504
  await progress.record(stageRow("fix", result, "❌ no commits"));
1244
- throw new TicketFailure("fix", "the fix Stage left no new commits on the branch");
1505
+ throw new TicketFailure(
1506
+ "fix",
1507
+ "the fix Stage left no new commits on the branch",
1508
+ failure.evidence,
1509
+ );
1245
1510
  }
1246
1511
 
1247
1512
  await progress.record(stageRow("fix", result, "✅ committed"));
@@ -1260,9 +1525,10 @@ async function fix(
1260
1525
  * came back clean because it quietly abandoned the rebase has not, and only the
1261
1526
  * worktree can tell the difference.
1262
1527
  *
1263
- * A Stage the rate limit stopped is read the same way, and only then released:
1264
- * one that had already finished the rebase leaves nothing that needs an agent
1265
- * session, so the Ticket carries on to the Checks, the pull request and CI.
1528
+ * A Stage the rate limit or an API failure stopped is read the same way, and
1529
+ * only then released: one that had already finished the rebase leaves nothing
1530
+ * that needs an agent session, so the Ticket carries on to the Checks, the pull
1531
+ * request and CI.
1266
1532
  */
1267
1533
  async function resolveConflict(
1268
1534
  pipeline: Pipeline,
@@ -1309,16 +1575,18 @@ async function conflictStage(
1309
1575
  logDir: string,
1310
1576
  conflict: string,
1311
1577
  progress: Progress,
1312
- ): Promise<TicketFailure | RateLimited | undefined> {
1578
+ ): Promise<TicketFailure | Releasing | undefined> {
1313
1579
  const stage = pipeline.config.stages.conflict;
1314
1580
  try {
1315
1581
  const result = await runStage(pipeline, "conflict", {
1582
+ ticket: issue.number,
1316
1583
  prompt: conflictPrompt(
1317
1584
  issue.url,
1318
1585
  conflict,
1319
1586
  pipeline.baseBranch,
1320
1587
  stage.extraPrompt,
1321
1588
  pipeline.selfHosted,
1589
+ pipeline.config.checks,
1322
1590
  ),
1323
1591
  cwd: worktree,
1324
1592
  logDir,
@@ -1327,8 +1595,8 @@ async function conflictStage(
1327
1595
  const state = await pipeline.workspace.rebaseState(worktree, pipeline.baseBranch);
1328
1596
  if (state.resolved) {
1329
1597
  // The worktree decides the row, as it decides the outcome: a Stage that
1330
- // ran out of turns, or into the rate limit, after finishing the rebase
1331
- // did the job it was sent to do.
1598
+ // ran out of turns, or into the rate limit or an API failure, after
1599
+ // finishing the rebase did the job it was sent to do.
1332
1600
  await progress.record(stageRow("conflict", result, "✅ rebased"));
1333
1601
  // The rebase rewrote the branch, and the resolution is a commit of the
1334
1602
  // Stage's like any other.
@@ -1336,11 +1604,13 @@ async function conflictStage(
1336
1604
  return undefined;
1337
1605
  }
1338
1606
 
1339
- // A rate limit left the conflict exactly as it found it: nothing for a fix
1340
- // Stage to mend and nothing to blame the Ticket for. The caller still owes
1341
- // the worktree its abort, so the branch the resumed Run rebases is clean.
1342
- if (result.failure === "rate-limited") {
1343
- return await releasedStage(progress, "conflict", "rebase", result);
1607
+ // A rate limit or an API failure left the conflict exactly as it found it:
1608
+ // nothing for a fix Stage to mend and nothing to blame the Ticket for. The
1609
+ // caller still owes the worktree its abort, so the branch the resumed Run
1610
+ // rebases is clean.
1611
+ const cause = releaseCause(result);
1612
+ if (cause !== undefined) {
1613
+ return await releasedStage(pipeline, progress, "conflict", "rebase", result, cause);
1344
1614
  }
1345
1615
 
1346
1616
  await progress.record(stageRow("conflict", result, "❌ unresolved"));
@@ -1348,7 +1618,7 @@ async function conflictStage(
1348
1618
  "rebase",
1349
1619
  result.ok
1350
1620
  ? `the conflict Stage did not finish the rebase onto ${pipeline.baseBranch}`
1351
- : describeStageFailure("conflict", stage, result),
1621
+ : describeStageFailure("conflict", stage, result, pipeline.credential),
1352
1622
  [conflict, state.unresolved].join("\n\n"),
1353
1623
  "unresolved-conflict",
1354
1624
  );
@@ -1422,8 +1692,10 @@ async function publishPullRequest(
1422
1692
  */
1423
1693
  async function requireGreenCi(
1424
1694
  pipeline: Pipeline,
1695
+ worktree: string,
1425
1696
  pullRequest: number,
1426
1697
  progress: Progress,
1698
+ landing: LandingTurn,
1427
1699
  ): Promise<void> {
1428
1700
  const startedAt = Date.now();
1429
1701
  const outcome = await pipeline.tracker.waitForCi(
@@ -1437,11 +1709,27 @@ async function requireGreenCi(
1437
1709
  case "passed":
1438
1710
  await record("✅ passed");
1439
1711
  return;
1440
- case "failed":
1712
+ case "failed": {
1713
+ // Out of the Landing first: whether the Ticket is released or sent to a
1714
+ // fix Stage, it leaves, and the Checks below take minutes nobody else
1715
+ // should wait behind.
1716
+ landing.leave();
1717
+ // CI's own runs on the Base branch cannot say whether it fails there: a
1718
+ // workflow that runs only on pull requests leaves a squash commit with
1719
+ // none, and a failure the clock brings starts after its last green run.
1720
+ // So the merge-base is graded with the Checks, which CI runs too.
1721
+ await releaseIfTheBaseFails(pipeline, {
1722
+ point: "ci",
1723
+ worktree,
1724
+ commands: pipeline.config.checks,
1725
+ progress,
1726
+ durationMs: Date.now() - startedAt,
1727
+ });
1441
1728
  await record("❌ failed");
1442
1729
  // The names are the line a human reads; the log excerpt is what a fix
1443
1730
  // Stage works from, and it is empty whenever none could be fetched.
1444
1731
  throw new TicketFailure("ci", outcome.summary, outcome.excerpt, "failed-ci");
1732
+ }
1445
1733
  case "timed-out":
1446
1734
  await record("❌ timed out");
1447
1735
  throw new TicketFailure(
@@ -1473,7 +1761,7 @@ interface Release {
1473
1761
  issue: Issue;
1474
1762
  user: string;
1475
1763
  branch: string;
1476
- limit: RateLimited;
1764
+ stopped: Releasing;
1477
1765
  /** The Ticket's State file, which the release brings up to date and leaves. */
1478
1766
  record: ResumeRecord;
1479
1767
  /** Whether the Fix budget was spent by a failure of the Ticket's own. */
@@ -1492,23 +1780,106 @@ interface Release {
1492
1780
  */
1493
1781
  async function release(
1494
1782
  pipeline: Pipeline,
1495
- { issue, user, branch, limit, record, fixUsed, notes }: Release,
1783
+ { issue, user, branch, stopped, record, fixUsed, notes }: Release,
1496
1784
  ): Promise<TicketOutcome> {
1497
1785
  const { tracker, config } = pipeline;
1498
1786
  const ticket = issue.number;
1499
1787
 
1500
1788
  // Brought up to date before the Claim comes off: a Ticket back on the Frontier
1501
- // whose state is out of date is one the next Run would redo Stages for.
1502
- await record.advance({ state: limit.state, fixUsed });
1789
+ // whose state is out of date is one the next Run would redo Stages for. One
1790
+ // released over a failing Base branch is rebased first when it comes back,
1791
+ // or it would meet the same failure.
1792
+ await record.advance({
1793
+ state: stopped.state,
1794
+ fixUsed,
1795
+ ...(stopped.failingBase === undefined ? {} : { rebaseFirst: true }),
1796
+ });
1797
+ if (stopped.failingBase !== undefined) {
1798
+ await noteFailingBase(pipeline, { issue, branch, stopped, failing: stopped.failingBase, notes });
1799
+ }
1503
1800
 
1504
1801
  // The claim, undone in the order it was made, so the assignee — which is what
1505
1802
  // another Run reads to tell a taken Ticket from a free one — comes off last.
1506
1803
  await tracker.addLabel(ticket, config.labels.readyForAgent);
1507
1804
  await tracker.removeLabel(ticket, config.labels.inProgress);
1508
1805
  await tracker.unassign(ticket, user);
1509
- pipeline.log?.(`#${ticket} released at ${limit.point} · rate limit · ${branch}`);
1806
+ pipeline.log?.(
1807
+ `#${ticket} released at ${stopped.point} · ${releaseWords(stopped.reason, pipeline.credential)} · ${branch}`,
1808
+ );
1510
1809
 
1511
- return { outcome: "released", ticket, title: issue.title, branch, stage: limit.point, notes };
1810
+ return {
1811
+ outcome: "released",
1812
+ ticket,
1813
+ title: issue.title,
1814
+ branch,
1815
+ stage: stopped.point,
1816
+ cause: stopped.reason,
1817
+ ...(stopped.limitResetsAt === undefined ? {} : { limitResetsAt: stopped.limitResetsAt }),
1818
+ ...(stopped.failingBase === undefined
1819
+ ? {}
1820
+ : {
1821
+ failingBase: { commit: stopped.failingBase.commit, command: stopped.failingBase.command },
1822
+ }),
1823
+ notes,
1824
+ };
1825
+ }
1826
+
1827
+ /**
1828
+ * Tell triage the Base branch fails a Check, once this Run per merge-base
1829
+ * commit however many Tickets it releases: the pipeline's own Note, on the
1830
+ * standing Notes issue, since the defect is the Base branch's and no Ticket's.
1831
+ *
1832
+ * Logged and nothing more when it cannot be posted, as a Stage's Note is: the
1833
+ * Release is what the Ticket is owed, and the next Ticket released over the
1834
+ * same commit tries the Note again.
1835
+ */
1836
+ async function noteFailingBase(
1837
+ pipeline: Pipeline,
1838
+ {
1839
+ issue,
1840
+ branch,
1841
+ stopped,
1842
+ failing,
1843
+ notes,
1844
+ }: {
1845
+ issue: Issue;
1846
+ branch: string;
1847
+ stopped: Releasing;
1848
+ failing: FailingBase;
1849
+ notes: RoutedNote[];
1850
+ },
1851
+ ): Promise<void> {
1852
+ const origin = issue.number;
1853
+ try {
1854
+ const note = await pipeline.baseChecks.noteOnce(failing.commit, async () => {
1855
+ const { number, opened } = await pipeline.standingNotes.resolve(
1856
+ standingNotesLookup(pipeline),
1857
+ );
1858
+ await pipeline.tracker.comment(
1859
+ number,
1860
+ failingBaseNote({
1861
+ origin,
1862
+ point: stopped.point === "ci" ? "ci" : "checks",
1863
+ baseBranch: pipeline.baseBranch,
1864
+ branch,
1865
+ ...failing,
1866
+ }),
1867
+ );
1868
+ pipeline.log?.(`#${origin} noted on #${number} that ${pipeline.baseBranch} fails \`${failing.command}\``);
1869
+ return {
1870
+ origin,
1871
+ stage: "pipeline",
1872
+ issue: number,
1873
+ opened,
1874
+ summary: failingBaseSummary(pipeline.baseBranch, failing.command),
1875
+ };
1876
+ });
1877
+ if (note !== undefined) notes.push(note);
1878
+ } catch (error) {
1879
+ pipeline.log?.(
1880
+ `#${origin} could not note that ${pipeline.baseBranch} fails a Check: ${(error as Error).message}`,
1881
+ );
1882
+ }
1512
1883
  }
1513
1884
 
1514
1885
  /** The worktree a hand-off has to hand, which is not always one of this Run's. */
@@ -1623,8 +1994,11 @@ async function handOff(
1623
1994
  const pr = await tracker.createPullRequest({
1624
1995
  base: pipeline.baseBranch,
1625
1996
  head: branch,
1626
- // Nothing here is merged, so there is no commit subject worth deriving.
1627
- title: issue.title,
1997
+ // A human can still merge the draft, and its squash commit takes this
1998
+ // title, so it is the one a Stage answered for the branch where there is
1999
+ // one. Without it, the Ticket title rather than a commit subject: the
2000
+ // pipeline merges nothing here, so it derives nothing for a merge.
2001
+ title: record.title ?? issue.title,
1628
2002
  body: draftPullRequestBody(draftBody),
1629
2003
  draft: true,
1630
2004
  });
@@ -1780,6 +2154,16 @@ const STAGE_FAILURES: Record<
1780
2154
  // entry per failure and the next path to need it must not have to invent it.
1781
2155
  sentence: () => "hit the subscription rate limit",
1782
2156
  },
2157
+ // Released like the rate limit, so its sentence is kept for the same reason.
2158
+ "api-error": {
2159
+ cell: "API failure",
2160
+ sentence: () => "was stopped by an API failure the CLI had given up retrying",
2161
+ },
2162
+ // And so is this one.
2163
+ "out-of-budget": {
2164
+ cell: "out of API budget",
2165
+ sentence: () => "ran out of API budget: the key's credit or its workspace's spend limit",
2166
+ },
1783
2167
  "timed-out": {
1784
2168
  cell: "timed out",
1785
2169
  sentence: (limits) => `ran past its ${limits.maxMinutes} minute limit`,
@@ -1798,7 +2182,14 @@ const STAGE_FAILURES: Record<
1798
2182
  /** A Stage that came back without saying why gets the one word that is true. */
1799
2183
  const UNEXPLAINED = { cell: "failed", sentence: () => "failed" } as const;
1800
2184
 
1801
- function stageFailure(result: StageResult) {
2185
+ /**
2186
+ * The rate limit of an `--api-key` Run, which is the API's own rather than the
2187
+ * subscription's, and is named so wherever the subscription's would be.
2188
+ */
2189
+ const API_RATE_LIMIT = { cell: "API rate limited", sentence: () => "hit the API rate limit" } as const;
2190
+
2191
+ function stageFailure(result: StageResult, credential: Credential) {
2192
+ if (result.failure === "rate-limited" && credential === "api-key") return API_RATE_LIMIT;
1802
2193
  return result.failure ? STAGE_FAILURES[result.failure] : UNEXPLAINED;
1803
2194
  }
1804
2195
 
@@ -1806,11 +2197,12 @@ function describeStageFailure(
1806
2197
  stage: string,
1807
2198
  limits: StageLimits,
1808
2199
  result: StageResult,
2200
+ credential: Credential,
1809
2201
  ): string {
1810
- return `the ${stage} Stage ${stageFailure(result).sentence(limits)}`;
2202
+ return `the ${stage} Stage ${stageFailure(result, credential).sentence(limits)}`;
1811
2203
  }
1812
2204
 
1813
2205
  /** The same failure, short enough for a table cell. */
1814
- function stageFailureCell(result: StageResult): string {
1815
- return stageFailure(result).cell;
2206
+ function stageFailureCell(result: StageResult, credential: Credential): string {
2207
+ return stageFailure(result, credential).cell;
1816
2208
  }