ticket-runner 0.6.1 → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,7 @@
1
1
  import { branchName, worktreePath } from "./branch.js";
2
2
  import type { Config } from "./config.js";
3
3
  import { tickMetCriteria } from "./criteria.js";
4
+ import type { BaseBranchChecks, FailingBase } from "./failing-base.js";
4
5
  import { type SkipReason, isGuardReason, skipReason } from "./guards.js";
5
6
  import { markHandoffsTaken } from "./handoff.js";
6
7
  import type { HostKind } from "./host.js";
@@ -23,6 +24,7 @@ import type { Issue, Tracker } from "./ports/tracker.js";
23
24
  import type {
24
25
  KeptTranscripts,
25
26
  ReachedState,
27
+ RebaseOutcome,
26
28
  TicketState,
27
29
  Workspace,
28
30
  WorktreeFromRemote,
@@ -38,7 +40,10 @@ import {
38
40
  import { retryLogDir, stageLogDir } from "./run-log.js";
39
41
  import { holdsClaim } from "./stranded.js";
40
42
  import {
43
+ RELEASE_WORDS,
41
44
  draftPullRequestBody,
45
+ failingBaseNote,
46
+ failingBaseSummary,
42
47
  guardComment,
43
48
  handoffComment,
44
49
  hasGuardWarning,
@@ -81,8 +86,17 @@ export type TicketOutcome =
81
86
  ticket: number;
82
87
  title: string;
83
88
  branch: string;
84
- /** Where the rate limit landed, which is all the summary says about it. */
89
+ /** Where the Stage was stopped, which is most of what the summary says about it. */
85
90
  stage: FailurePoint;
91
+ /** What stopped it, which is the rest. */
92
+ cause: ReleaseCause;
93
+ /**
94
+ * When the rate limit resets, ISO-8601, where the Stage it stopped was
95
+ * told. Absent for an API failure, which says nothing of when it ends.
96
+ */
97
+ limitResetsAt?: string;
98
+ /** The Check the Base branch fails, and where, for a Release over it. */
99
+ failingBase?: { commit: string; command: string };
86
100
  notes: RoutedNote[];
87
101
  }
88
102
  | {
@@ -130,6 +144,12 @@ export interface Pipeline {
130
144
  * them are told which one it is.
131
145
  */
132
146
  standingNotes: StandingNotes;
147
+ /**
148
+ * The Checks the Run ran at merge-bases with the Base branch, made once and
149
+ * shared by every Ticket: the Tickets that fail together over one broken
150
+ * Base branch run the Check there once between them, and post one Note.
151
+ */
152
+ baseChecks: BaseBranchChecks;
133
153
  /**
134
154
  * Whether the Target is the pipeline's own repository, decided once before
135
155
  * the Run started ({@link import("./self-hosting.js").isPipelineRepository}).
@@ -157,21 +177,42 @@ class TicketFailure extends Error {
157
177
  }
158
178
 
159
179
  /**
160
- * A Stage the subscription rate limit stopped.
180
+ * What stops a Ticket that says nothing about it: the subscription rate limit,
181
+ * a transient API failure — the API overloaded, broken or silent on its own
182
+ * side after the CLI had already retried it — and a Base branch that fails the
183
+ * Check the Ticket's branch failed.
184
+ */
185
+ export type ReleaseCause = StageReleaseCause | "failing-base";
186
+
187
+ /** The Stage failures among them. */
188
+ type StageReleaseCause = Extract<StageFailure, "rate-limited" | "api-error">;
189
+
190
+ function releaseCause(result: StageResult): StageReleaseCause | undefined {
191
+ return result.failure === "rate-limited" || result.failure === "api-error"
192
+ ? result.failure
193
+ : undefined;
194
+ }
195
+
196
+ /**
197
+ * A Stage something other than the Ticket stopped, on its way to a Release.
161
198
  *
162
- * The one Stage failure that says nothing about the Ticket, so it is the one
163
- * that neither ends it nor spends its Fix budget: the Claim is released and a
164
- * State file is left for the next Run to resume from (ADR-0004). It is not a
199
+ * The Stage failures that say nothing about the Ticket are the ones that
200
+ * neither end it nor spend its Fix budget: the Claim is released and a State
201
+ * file is left for the next Run to resume from (ADR-0004). It is not a
165
202
  * {@link TicketFailure} because every path that handles one of those would
166
203
  * either hand the Ticket off or buy it a fix Stage, and this is neither.
167
204
  */
168
- class RateLimited extends Error {
205
+ class Releasing extends Error {
169
206
  constructor(
170
207
  readonly point: FailurePoint,
171
208
  /** The state the Ticket reached, which is where a later Run picks it up. */
172
209
  readonly state: ReachedState,
210
+ readonly reason: ReleaseCause,
211
+ readonly limitResetsAt?: string,
212
+ /** The Check the Base branch fails too, for a Release over it. */
213
+ readonly failingBase?: FailingBase,
173
214
  ) {
174
- super(`the ${point} Stage hit the subscription rate limit`);
215
+ super(`the ${point} Stage was stopped · ${RELEASE_WORDS[reason]}`);
175
216
  }
176
217
  }
177
218
 
@@ -194,9 +235,10 @@ function asTicketFailure(error: unknown, point: FailurePoint): TicketFailure {
194
235
  * Nothing ends in a merge that has not been through a green pass of the whole
195
236
  * gauntlet.
196
237
  *
197
- * A Stage the subscription rate limit stopped ends none of that: the Ticket is
198
- * released, and a Run started once the limit has reset resumes it from the state
199
- * it had reached rather than from the top. A hand-off keeps that state too, so a
238
+ * A Stage the subscription rate limit or a transient API failure stopped ends
239
+ * none of that: the Ticket is released, and a Run started once the limit has
240
+ * reset, or the API recovered, resumes it from the state it had reached rather
241
+ * than from the top. A hand-off keeps that state too, so a
200
242
  * human who relabels the Ticket `ready-for-agent` hands it back and the next Run
201
243
  * carries on from what it reached rather than paying for the implement Stage a
202
244
  * second time.
@@ -289,6 +331,7 @@ async function takeTicket(
289
331
  fixUsed: resume?.fixUsed ?? false,
290
332
  ...(resume?.pullRequest === undefined ? {} : { pullRequest: resume.pullRequest }),
291
333
  ...(resume?.title === undefined ? {} : { title: resume.title }),
334
+ ...(resume?.rebaseFirst === true ? { rebaseFirst: true } : {}),
292
335
  });
293
336
 
294
337
  // The claim is the first write on the board: it shows what the pipeline holds
@@ -327,6 +370,10 @@ async function takeTicket(
327
370
  let fixUsed = resume?.fixUsed ?? false;
328
371
  // The pass behind a fix Stage logs beside the first one rather than over it.
329
372
  let logDir = stageLogDir(repoRoot, runId, ticket);
373
+ // The unmet criteria the fix Stage was sent in for, and their evidence, which
374
+ // the verify behind it is held to. Only this Run's: the State file keeps no
375
+ // Verdict, so a retry a later Run resumes is graded without them.
376
+ let unmetBefore: string | undefined;
330
377
 
331
378
  // Declared out here because ticking the Acceptance Criteria it proved happens
332
379
  // after the merge, where a failure may no longer hand the Ticket off.
@@ -388,18 +435,28 @@ async function takeTicket(
388
435
  let commits: string[];
389
436
  let coAuthors: string[];
390
437
  let title: string;
438
+ // Owed once, by a Ticket released over a failing Base branch, and inside
439
+ // the passes rather than before them: a conflict it meets spends the Fix
440
+ // budget as one at the Landing does.
441
+ let rebaseOwed = resume?.rebaseFirst === true;
391
442
 
392
443
  // Checks through CI, run again from the top when the fix budget buys a
393
444
  // second pass: a fix earns no shortcut, so every gate grades it afresh.
394
445
  for (;;) {
395
446
  try {
447
+ if (rebaseOwed) {
448
+ rebaseOwed = false;
449
+ point = "rebase";
450
+ await rebaseFirst(pipeline, issue, worktree, branch, logDir, landing, progress);
451
+ await record.advance({ rebaseFirst: false });
452
+ }
396
453
  point = "checks";
397
454
  // Before anything grades the worktree, and so before verify discards
398
455
  // what a Stage left uncommitted.
399
456
  await requireCommitted(pipeline, worktree, progress);
400
457
  await runChecks(pipeline, worktree, progress);
401
458
  point = "verify";
402
- verdict = await verify(pipeline, issue, worktree, logDir, progress, notes);
459
+ verdict = await verify(pipeline, issue, worktree, logDir, progress, notes, unmetBefore);
403
460
 
404
461
  point = "rebase";
405
462
  // The Landing, from here until the Base branch has been pulled: the
@@ -448,12 +505,12 @@ async function takeTicket(
448
505
  );
449
506
  await record.advance({ pullRequest });
450
507
  point = "ci";
451
- await requireGreenCi(pipeline, pullRequest, progress);
508
+ await requireGreenCi(pipeline, worktree, pullRequest, progress, landing);
452
509
  break;
453
510
  } catch (error) {
454
- // A rate limit is nobody's defect, so the fix budget does not answer
455
- // for it: the Ticket is released, whole, further up.
456
- if (error instanceof RateLimited) throw error;
511
+ // A rate limit or an API failure is nobody's defect, so the fix budget
512
+ // does not answer for it: the Ticket is released, whole, further up.
513
+ if (error instanceof Releasing) throw error;
457
514
  const failure = asTicketFailure(error, point);
458
515
  if (fixUsed || failure.kind === undefined) throw failure;
459
516
  fixUsed = true;
@@ -463,6 +520,7 @@ async function takeTicket(
463
520
  // the queue at the back when it reaches the rebase again.
464
521
  landing.leave();
465
522
  logDir = retryLogDir(repoRoot, runId, ticket);
523
+ if (failure.kind === "unmet-criteria") unmetBefore = failure.evidence;
466
524
  await fix(
467
525
  pipeline,
468
526
  issue,
@@ -475,8 +533,8 @@ async function takeTicket(
475
533
  record,
476
534
  );
477
535
  // Recorded once the Stage has come back, not when the budget was
478
- // committed: a fix Stage the rate limit stopped before it ran spends
479
- // nothing, and the release below says so.
536
+ // committed: a fix Stage the rate limit or an API failure stopped
537
+ // before it ran spends nothing, and the release below says so.
480
538
  await record.advance({ fixUsed: true });
481
539
  }
482
540
  }
@@ -492,15 +550,15 @@ async function takeTicket(
492
550
  // pull request and a Release's labels are nothing for the next Ticket to
493
551
  // wait behind, and a Ticket that never reached the rebase gives up nothing.
494
552
  landing.leave();
495
- if (error instanceof RateLimited) {
553
+ if (error instanceof Releasing) {
496
554
  return release(pipeline, {
497
555
  issue,
498
556
  user,
499
557
  branch,
500
- limit: error,
558
+ stopped: error,
501
559
  record,
502
- // A fix Stage the limit stopped before it ran spends nothing: the
503
- // budget is still there for the Run that resumes the Ticket.
560
+ // A fix Stage stopped before it ran spends nothing: the budget is
561
+ // still there for the Run that resumes the Ticket.
504
562
  fixUsed: fixUsed && error.point !== "fix",
505
563
  notes,
506
564
  });
@@ -703,7 +761,9 @@ function howItWasTaken(resume: TicketState | undefined, stranded: boolean): stri
703
761
  type Reached = Omit<TicketState, "runId" | "updatedAt">;
704
762
 
705
763
  /** What one write moves that on by. */
706
- type Advance = Partial<Pick<Reached, "state" | "fixUsed" | "pullRequest" | "title">>;
764
+ type Advance = Partial<
765
+ Pick<Reached, "state" | "fixUsed" | "pullRequest" | "title" | "rebaseFirst">
766
+ >;
707
767
 
708
768
  /**
709
769
  * The State file a Ticket keeps, and the one place a Ticket writes it.
@@ -776,8 +836,12 @@ class ResumeRecord {
776
836
  * read one can still name what wrote it (ADR-0007).
777
837
  */
778
838
  private file(): TicketState {
839
+ // Written only while it is owed, so a State file says nothing of a rebase
840
+ // nobody has to do.
841
+ const { rebaseFirst, ...reached } = this.reached;
779
842
  return {
780
- ...this.reached,
843
+ ...reached,
844
+ ...(rebaseFirst === true ? { rebaseFirst } : {}),
781
845
  runId: this.pipeline.runId,
782
846
  version: this.pipeline.version,
783
847
  updatedAt: new Date().toISOString(),
@@ -850,7 +914,8 @@ function runStage(
850
914
  }
851
915
 
852
916
  /**
853
- * Report a Stage the rate limit stopped, and raise the Ticket's release.
917
+ * Report a Stage the rate limit or an API failure stopped, and raise the
918
+ * Ticket's release.
854
919
  *
855
920
  * The one place the pause is composed, because the conflict Stage reaches it by
856
921
  * another road: the row reads as a pause rather than a cross, and the error
@@ -863,26 +928,34 @@ async function releasedStage(
863
928
  stage: StageName,
864
929
  point: FailurePoint,
865
930
  result: StageResult,
866
- ): Promise<RateLimited> {
867
- await progress.record(stageRow(stage, result, "⏸ rate limited"));
868
- return new RateLimited(point, stage === "implement" ? "claimed" : "implemented");
931
+ cause: StageReleaseCause,
932
+ ): Promise<Releasing> {
933
+ await progress.record(stageRow(stage, result, `⏸ ${STAGE_FAILURES[cause].cell}`));
934
+ return new Releasing(
935
+ point,
936
+ stage === "implement" ? "claimed" : "implemented",
937
+ cause,
938
+ result.limitResetsAt,
939
+ );
869
940
  }
870
941
 
871
942
  /**
872
943
  * Report a Stage that did not come back, and say what it costs the Ticket.
873
944
  *
874
- * The rate limit is the one Stage failure the Ticket did nothing to earn, so it
875
- * releases the Ticket instead of ending it. Every other failure is the Stage's
876
- * own, and its caller decides whether the fix budget can buy another.
945
+ * The rate limit and a transient API failure are the Stage failures the Ticket
946
+ * did nothing to earn, so they release the Ticket instead of ending it. Every
947
+ * other failure is the Stage's own, and its caller decides whether the fix
948
+ * budget can buy another.
877
949
  */
878
950
  async function stageDidNotFinish(
879
951
  pipeline: Pipeline,
880
952
  progress: Progress,
881
953
  stage: "implement" | "verify" | "fix",
882
954
  result: StageResult,
883
- ): Promise<TicketFailure | RateLimited> {
884
- if (result.failure === "rate-limited") {
885
- return await releasedStage(progress, stage, stage, result);
955
+ ): Promise<TicketFailure | Releasing> {
956
+ const cause = releaseCause(result);
957
+ if (cause !== undefined) {
958
+ return await releasedStage(progress, stage, stage, result, cause);
886
959
  }
887
960
 
888
961
  await progress.record(stageRow(stage, result, `❌ ${stageFailureCell(result)}`));
@@ -1001,6 +1074,7 @@ async function implement(
1001
1074
  pipeline.baseBranch,
1002
1075
  stage.extraPrompt,
1003
1076
  pipeline.selfHosted,
1077
+ pipeline.config.checks,
1004
1078
  await standingNotesNumber(pipeline),
1005
1079
  ),
1006
1080
  cwd: worktree,
@@ -1015,15 +1089,15 @@ async function implement(
1015
1089
  await recordTitle(result, record);
1016
1090
 
1017
1091
  // Pushed whatever became of the Stage, like its Notes: a session that
1018
- // committed and then ran into the rate limit still committed, and the Host
1019
- // that resumes the Ticket may be another one.
1092
+ // committed and then was stopped by the rate limit or an API failure still
1093
+ // committed, and the Host that resumes the Ticket may be another one.
1020
1094
  const commits = await commitCount(pipeline, branch);
1021
1095
  if (commits > 0) await pushCommitted(pipeline, issue.number, worktree, branch);
1022
1096
  if (!result.ok) throw await stageDidNotFinish(pipeline, progress, "implement", result);
1023
1097
 
1024
1098
  // An agent that gave up silently leaves a clean branch behind. That is a
1025
1099
  // failure, not something to verify. A resumed Ticket's branch is not clean —
1026
- // it carries what the Stage the rate limit stopped had committed — so this
1100
+ // it carries what the Stage a Release stopped had committed — so this
1027
1101
  // asks the same question there: is there anything at all to grade. Whether it
1028
1102
  // is enough is the Verdict's business, not this guard's.
1029
1103
  if (commits === 0) {
@@ -1037,13 +1111,53 @@ async function implement(
1037
1111
  await progress.record(stageRow("implement", result, "✅ committed"));
1038
1112
  }
1039
1113
 
1114
+ /**
1115
+ * Bring a Ticket released over a failing Base branch up to the Base branch
1116
+ * before its Checks grade it, so a fix a human has landed there meanwhile
1117
+ * reaches the branch. Without it the Ticket would meet the same failure with
1118
+ * its budget unspent, and spend it.
1119
+ *
1120
+ * The pull and the rebase go through the Landing, which is where the Base
1121
+ * branch is brought up to the remote's and rebased onto (ADR-0005), and out of
1122
+ * it again at once: nothing is merged behind them. A conflict goes to the
1123
+ * Conflict Stage as one at the Landing does, outside it, since the rebase
1124
+ * there runs again before the pull request anyway. Left unresolved, the
1125
+ * branch would keep the merge-base that fails and be released again by every
1126
+ * Run that took it up.
1127
+ */
1128
+ async function rebaseFirst(
1129
+ pipeline: Pipeline,
1130
+ issue: Issue,
1131
+ worktree: string,
1132
+ branch: string,
1133
+ logDir: string,
1134
+ landing: LandingTurn,
1135
+ progress: Progress,
1136
+ ): Promise<void> {
1137
+ const { workspace, baseBranch } = pipeline;
1138
+ await landing.enter();
1139
+ let rebase: RebaseOutcome;
1140
+ try {
1141
+ await workspace.pullBase(baseBranch);
1142
+ rebase = await workspace.rebase(worktree, baseBranch);
1143
+ } finally {
1144
+ landing.leave();
1145
+ }
1146
+ if (!rebase.ok) {
1147
+ await resolveConflict(pipeline, issue, worktree, branch, logDir, rebase.conflict, progress);
1148
+ return;
1149
+ }
1150
+ await pushCommitted(pipeline, issue.number, worktree, branch);
1151
+ pipeline.log?.(`#${issue.number} rebased onto ${baseBranch} before its Checks`);
1152
+ }
1153
+
1040
1154
  /**
1041
1155
  * Refuse a worktree holding changes no commit carries, naming every path.
1042
1156
  *
1043
1157
  * Nothing is discarded here: a fix Stage sent in for it decides what belongs
1044
1158
  * to the Ticket, and a hand-off leaves the changes where a human can find them.
1045
1159
  * The summary names no Stage, because a resumed Ticket cannot say which one
1046
- * left them: a fix Stage the rate limit stopped spent no budget.
1160
+ * left them: a fix Stage a Release stopped spent no budget.
1047
1161
  */
1048
1162
  async function requireCommitted(
1049
1163
  pipeline: Pipeline,
@@ -1074,13 +1188,23 @@ async function runChecks(
1074
1188
  for (const command of pipeline.config.checks) {
1075
1189
  const result = await pipeline.workspace.runCheck(command, worktree, minutes * 60_000);
1076
1190
  if (!result.ok) {
1191
+ const durationMs = Date.now() - startedAt;
1192
+ // Asked before the row is written, because the row is what differs: a
1193
+ // Check the Base branch fails too is nothing the Ticket did.
1194
+ await releaseIfTheBaseFails(pipeline, {
1195
+ point: "checks",
1196
+ worktree,
1197
+ commands: [command],
1198
+ progress,
1199
+ durationMs,
1200
+ });
1077
1201
  // A Check that hung and one that failed are the same failure to the fix
1078
1202
  // budget and the hand-off; they are told apart only in what they say, so
1079
1203
  // that a fix Stage knows whether it is mending a hang or an assertion.
1080
1204
  await progress.record({
1081
1205
  point: "checks",
1082
1206
  outcome: `❌ \`${command}\` ${result.timedOut ? "timed out" : "failed"}`,
1083
- durationMs: Date.now() - startedAt,
1207
+ durationMs,
1084
1208
  });
1085
1209
  throw new TicketFailure(
1086
1210
  "checks",
@@ -1097,6 +1221,75 @@ async function runChecks(
1097
1221
  });
1098
1222
  }
1099
1223
 
1224
+ /**
1225
+ * The first of `commands` the Base branch fails, at the commit the branch at
1226
+ * `worktree` inherited from it, or nothing when it passes them all.
1227
+ *
1228
+ * Nothing, too, when the merge-base could not be graded at all: the Ticket is
1229
+ * then charged for the failure as it always was, which costs a fix Stage at
1230
+ * worst, where releasing on a guess would leave a Ticket's own defect unmended
1231
+ * and stop the Run.
1232
+ */
1233
+ async function inheritedFailure(
1234
+ pipeline: Pipeline,
1235
+ worktree: string,
1236
+ commands: readonly string[],
1237
+ ): Promise<FailingBase | undefined> {
1238
+ try {
1239
+ const commit = await pipeline.workspace.mergeBase(worktree, pipeline.baseBranch);
1240
+ return await pipeline.baseChecks.firstFailing(
1241
+ {
1242
+ workspace: pipeline.workspace,
1243
+ repoRoot: pipeline.repoRoot,
1244
+ timeoutMs: pipeline.config.checkTimeoutMinutes * 60_000,
1245
+ ...(pipeline.log === undefined ? {} : { log: pipeline.log }),
1246
+ },
1247
+ commit,
1248
+ commands,
1249
+ );
1250
+ } catch (error) {
1251
+ pipeline.log?.(
1252
+ `could not run the Checks at the merge-base with ${pipeline.baseBranch}: ${(error as Error).message}`,
1253
+ );
1254
+ return undefined;
1255
+ }
1256
+ }
1257
+
1258
+ /**
1259
+ * Release the Ticket when the Base branch fails one of `commands` too, with the
1260
+ * row that says so; return when it does not, for the caller to charge the
1261
+ * failure to the Ticket as it always did.
1262
+ *
1263
+ * Only a Ticket that reached its Checks gets here, so the implement Stage's
1264
+ * work is always on its branch.
1265
+ */
1266
+ async function releaseIfTheBaseFails(
1267
+ pipeline: Pipeline,
1268
+ {
1269
+ point,
1270
+ worktree,
1271
+ commands,
1272
+ progress,
1273
+ durationMs,
1274
+ }: {
1275
+ point: "checks" | "ci";
1276
+ worktree: string;
1277
+ commands: readonly string[];
1278
+ progress: Progress;
1279
+ /** How long the failure took on the branch, which the row reports. */
1280
+ durationMs: number;
1281
+ },
1282
+ ): Promise<void> {
1283
+ const inherited = await inheritedFailure(pipeline, worktree, commands);
1284
+ if (inherited === undefined) return;
1285
+ await progress.record({
1286
+ point,
1287
+ outcome: `⏸ \`${inherited.command}\` fails on the Base branch`,
1288
+ durationMs,
1289
+ });
1290
+ throw new Releasing(point, "implemented", "failing-base", undefined, inherited);
1291
+ }
1292
+
1100
1293
  /**
1101
1294
  * What the Check printed before the kill, and then a line saying that a kill is
1102
1295
  * what ended it. A fix Stage is given the evidence and little else, so the
@@ -1116,6 +1309,8 @@ async function verify(
1116
1309
  logDir: string,
1117
1310
  progress: Progress,
1118
1311
  notes: RoutedNote[],
1312
+ /** The unmet criteria and evidence of the Verdict a fix Stage was sent in for. */
1313
+ unmetBefore: string | undefined,
1119
1314
  ): Promise<Verdict> {
1120
1315
  const stage = pipeline.config.stages.verify;
1121
1316
  const result = await runStage(pipeline, "verify", {
@@ -1123,7 +1318,9 @@ async function verify(
1123
1318
  issue.url,
1124
1319
  stage.extraPrompt,
1125
1320
  pipeline.selfHosted,
1321
+ pipeline.config.checks,
1126
1322
  await standingNotesNumber(pipeline),
1323
+ unmetBefore,
1127
1324
  ),
1128
1325
  cwd: worktree,
1129
1326
  logDir,
@@ -1215,6 +1412,7 @@ async function fix(
1215
1412
  pipeline.baseBranch,
1216
1413
  stage.extraPrompt,
1217
1414
  pipeline.selfHosted,
1415
+ pipeline.config.checks,
1218
1416
  await standingNotesNumber(pipeline),
1219
1417
  ),
1220
1418
  cwd: worktree,
@@ -1239,9 +1437,17 @@ async function fix(
1239
1437
  // A branch that grew is the whole signal, which is not the same as one that
1240
1438
  // changed: a session that squashed the branch shorter, or amended in place,
1241
1439
  // is read here as having committed nothing.
1440
+ //
1441
+ // The hand-off carries the evidence the Stage was sent in with. Committing
1442
+ // nothing is also how a fix Stage hands back an unmet criterion only a human
1443
+ // can settle, and the human then reads the question where it was asked.
1242
1444
  if (!grew) {
1243
1445
  await progress.record(stageRow("fix", result, "❌ no commits"));
1244
- throw new TicketFailure("fix", "the fix Stage left no new commits on the branch");
1446
+ throw new TicketFailure(
1447
+ "fix",
1448
+ "the fix Stage left no new commits on the branch",
1449
+ failure.evidence,
1450
+ );
1245
1451
  }
1246
1452
 
1247
1453
  await progress.record(stageRow("fix", result, "✅ committed"));
@@ -1260,9 +1466,10 @@ async function fix(
1260
1466
  * came back clean because it quietly abandoned the rebase has not, and only the
1261
1467
  * worktree can tell the difference.
1262
1468
  *
1263
- * A Stage the rate limit stopped is read the same way, and only then released:
1264
- * one that had already finished the rebase leaves nothing that needs an agent
1265
- * session, so the Ticket carries on to the Checks, the pull request and CI.
1469
+ * A Stage the rate limit or an API failure stopped is read the same way, and
1470
+ * only then released: one that had already finished the rebase leaves nothing
1471
+ * that needs an agent session, so the Ticket carries on to the Checks, the pull
1472
+ * request and CI.
1266
1473
  */
1267
1474
  async function resolveConflict(
1268
1475
  pipeline: Pipeline,
@@ -1309,7 +1516,7 @@ async function conflictStage(
1309
1516
  logDir: string,
1310
1517
  conflict: string,
1311
1518
  progress: Progress,
1312
- ): Promise<TicketFailure | RateLimited | undefined> {
1519
+ ): Promise<TicketFailure | Releasing | undefined> {
1313
1520
  const stage = pipeline.config.stages.conflict;
1314
1521
  try {
1315
1522
  const result = await runStage(pipeline, "conflict", {
@@ -1319,6 +1526,7 @@ async function conflictStage(
1319
1526
  pipeline.baseBranch,
1320
1527
  stage.extraPrompt,
1321
1528
  pipeline.selfHosted,
1529
+ pipeline.config.checks,
1322
1530
  ),
1323
1531
  cwd: worktree,
1324
1532
  logDir,
@@ -1327,8 +1535,8 @@ async function conflictStage(
1327
1535
  const state = await pipeline.workspace.rebaseState(worktree, pipeline.baseBranch);
1328
1536
  if (state.resolved) {
1329
1537
  // The worktree decides the row, as it decides the outcome: a Stage that
1330
- // ran out of turns, or into the rate limit, after finishing the rebase
1331
- // did the job it was sent to do.
1538
+ // ran out of turns, or into the rate limit or an API failure, after
1539
+ // finishing the rebase did the job it was sent to do.
1332
1540
  await progress.record(stageRow("conflict", result, "✅ rebased"));
1333
1541
  // The rebase rewrote the branch, and the resolution is a commit of the
1334
1542
  // Stage's like any other.
@@ -1336,11 +1544,13 @@ async function conflictStage(
1336
1544
  return undefined;
1337
1545
  }
1338
1546
 
1339
- // A rate limit left the conflict exactly as it found it: nothing for a fix
1340
- // Stage to mend and nothing to blame the Ticket for. The caller still owes
1341
- // the worktree its abort, so the branch the resumed Run rebases is clean.
1342
- if (result.failure === "rate-limited") {
1343
- return await releasedStage(progress, "conflict", "rebase", result);
1547
+ // A rate limit or an API failure left the conflict exactly as it found it:
1548
+ // nothing for a fix Stage to mend and nothing to blame the Ticket for. The
1549
+ // caller still owes the worktree its abort, so the branch the resumed Run
1550
+ // rebases is clean.
1551
+ const cause = releaseCause(result);
1552
+ if (cause !== undefined) {
1553
+ return await releasedStage(progress, "conflict", "rebase", result, cause);
1344
1554
  }
1345
1555
 
1346
1556
  await progress.record(stageRow("conflict", result, "❌ unresolved"));
@@ -1422,8 +1632,10 @@ async function publishPullRequest(
1422
1632
  */
1423
1633
  async function requireGreenCi(
1424
1634
  pipeline: Pipeline,
1635
+ worktree: string,
1425
1636
  pullRequest: number,
1426
1637
  progress: Progress,
1638
+ landing: LandingTurn,
1427
1639
  ): Promise<void> {
1428
1640
  const startedAt = Date.now();
1429
1641
  const outcome = await pipeline.tracker.waitForCi(
@@ -1437,11 +1649,27 @@ async function requireGreenCi(
1437
1649
  case "passed":
1438
1650
  await record("✅ passed");
1439
1651
  return;
1440
- case "failed":
1652
+ case "failed": {
1653
+ // Out of the Landing first: whether the Ticket is released or sent to a
1654
+ // fix Stage, it leaves, and the Checks below take minutes nobody else
1655
+ // should wait behind.
1656
+ landing.leave();
1657
+ // CI's own runs on the Base branch cannot say whether it fails there: a
1658
+ // workflow that runs only on pull requests leaves a squash commit with
1659
+ // none, and a failure the clock brings starts after its last green run.
1660
+ // So the merge-base is graded with the Checks, which CI runs too.
1661
+ await releaseIfTheBaseFails(pipeline, {
1662
+ point: "ci",
1663
+ worktree,
1664
+ commands: pipeline.config.checks,
1665
+ progress,
1666
+ durationMs: Date.now() - startedAt,
1667
+ });
1441
1668
  await record("❌ failed");
1442
1669
  // The names are the line a human reads; the log excerpt is what a fix
1443
1670
  // Stage works from, and it is empty whenever none could be fetched.
1444
1671
  throw new TicketFailure("ci", outcome.summary, outcome.excerpt, "failed-ci");
1672
+ }
1445
1673
  case "timed-out":
1446
1674
  await record("❌ timed out");
1447
1675
  throw new TicketFailure(
@@ -1473,7 +1701,7 @@ interface Release {
1473
1701
  issue: Issue;
1474
1702
  user: string;
1475
1703
  branch: string;
1476
- limit: RateLimited;
1704
+ stopped: Releasing;
1477
1705
  /** The Ticket's State file, which the release brings up to date and leaves. */
1478
1706
  record: ResumeRecord;
1479
1707
  /** Whether the Fix budget was spent by a failure of the Ticket's own. */
@@ -1492,23 +1720,106 @@ interface Release {
1492
1720
  */
1493
1721
  async function release(
1494
1722
  pipeline: Pipeline,
1495
- { issue, user, branch, limit, record, fixUsed, notes }: Release,
1723
+ { issue, user, branch, stopped, record, fixUsed, notes }: Release,
1496
1724
  ): Promise<TicketOutcome> {
1497
1725
  const { tracker, config } = pipeline;
1498
1726
  const ticket = issue.number;
1499
1727
 
1500
1728
  // Brought up to date before the Claim comes off: a Ticket back on the Frontier
1501
- // whose state is out of date is one the next Run would redo Stages for.
1502
- await record.advance({ state: limit.state, fixUsed });
1729
+ // whose state is out of date is one the next Run would redo Stages for. One
1730
+ // released over a failing Base branch is rebased first when it comes back,
1731
+ // or it would meet the same failure.
1732
+ await record.advance({
1733
+ state: stopped.state,
1734
+ fixUsed,
1735
+ ...(stopped.failingBase === undefined ? {} : { rebaseFirst: true }),
1736
+ });
1737
+ if (stopped.failingBase !== undefined) {
1738
+ await noteFailingBase(pipeline, { issue, branch, stopped, failing: stopped.failingBase, notes });
1739
+ }
1503
1740
 
1504
1741
  // The claim, undone in the order it was made, so the assignee — which is what
1505
1742
  // another Run reads to tell a taken Ticket from a free one — comes off last.
1506
1743
  await tracker.addLabel(ticket, config.labels.readyForAgent);
1507
1744
  await tracker.removeLabel(ticket, config.labels.inProgress);
1508
1745
  await tracker.unassign(ticket, user);
1509
- pipeline.log?.(`#${ticket} released at ${limit.point} · rate limit · ${branch}`);
1746
+ pipeline.log?.(
1747
+ `#${ticket} released at ${stopped.point} · ${RELEASE_WORDS[stopped.reason]} · ${branch}`,
1748
+ );
1749
+
1750
+ return {
1751
+ outcome: "released",
1752
+ ticket,
1753
+ title: issue.title,
1754
+ branch,
1755
+ stage: stopped.point,
1756
+ cause: stopped.reason,
1757
+ ...(stopped.limitResetsAt === undefined ? {} : { limitResetsAt: stopped.limitResetsAt }),
1758
+ ...(stopped.failingBase === undefined
1759
+ ? {}
1760
+ : {
1761
+ failingBase: { commit: stopped.failingBase.commit, command: stopped.failingBase.command },
1762
+ }),
1763
+ notes,
1764
+ };
1765
+ }
1510
1766
 
1511
- return { outcome: "released", ticket, title: issue.title, branch, stage: limit.point, notes };
1767
+ /**
1768
+ * Tell triage the Base branch fails a Check, once this Run per merge-base
1769
+ * commit however many Tickets it releases: the pipeline's own Note, on the
1770
+ * standing Notes issue, since the defect is the Base branch's and no Ticket's.
1771
+ *
1772
+ * Logged and nothing more when it cannot be posted, as a Stage's Note is: the
1773
+ * Release is what the Ticket is owed, and the next Ticket released over the
1774
+ * same commit tries the Note again.
1775
+ */
1776
+ async function noteFailingBase(
1777
+ pipeline: Pipeline,
1778
+ {
1779
+ issue,
1780
+ branch,
1781
+ stopped,
1782
+ failing,
1783
+ notes,
1784
+ }: {
1785
+ issue: Issue;
1786
+ branch: string;
1787
+ stopped: Releasing;
1788
+ failing: FailingBase;
1789
+ notes: RoutedNote[];
1790
+ },
1791
+ ): Promise<void> {
1792
+ const origin = issue.number;
1793
+ try {
1794
+ const note = await pipeline.baseChecks.noteOnce(failing.commit, async () => {
1795
+ const { number, opened } = await pipeline.standingNotes.resolve(
1796
+ standingNotesLookup(pipeline),
1797
+ );
1798
+ await pipeline.tracker.comment(
1799
+ number,
1800
+ failingBaseNote({
1801
+ origin,
1802
+ point: stopped.point === "ci" ? "ci" : "checks",
1803
+ baseBranch: pipeline.baseBranch,
1804
+ branch,
1805
+ ...failing,
1806
+ }),
1807
+ );
1808
+ pipeline.log?.(`#${origin} noted on #${number} that ${pipeline.baseBranch} fails \`${failing.command}\``);
1809
+ return {
1810
+ origin,
1811
+ stage: "pipeline",
1812
+ issue: number,
1813
+ opened,
1814
+ summary: failingBaseSummary(pipeline.baseBranch, failing.command),
1815
+ };
1816
+ });
1817
+ if (note !== undefined) notes.push(note);
1818
+ } catch (error) {
1819
+ pipeline.log?.(
1820
+ `#${origin} could not note that ${pipeline.baseBranch} fails a Check: ${(error as Error).message}`,
1821
+ );
1822
+ }
1512
1823
  }
1513
1824
 
1514
1825
  /** The worktree a hand-off has to hand, which is not always one of this Run's. */
@@ -1623,8 +1934,11 @@ async function handOff(
1623
1934
  const pr = await tracker.createPullRequest({
1624
1935
  base: pipeline.baseBranch,
1625
1936
  head: branch,
1626
- // Nothing here is merged, so there is no commit subject worth deriving.
1627
- title: issue.title,
1937
+ // A human can still merge the draft, and its squash commit takes this
1938
+ // title, so it is the one a Stage answered for the branch where there is
1939
+ // one. Without it, the Ticket title rather than a commit subject: the
1940
+ // pipeline merges nothing here, so it derives nothing for a merge.
1941
+ title: record.title ?? issue.title,
1628
1942
  body: draftPullRequestBody(draftBody),
1629
1943
  draft: true,
1630
1944
  });
@@ -1780,6 +2094,11 @@ const STAGE_FAILURES: Record<
1780
2094
  // entry per failure and the next path to need it must not have to invent it.
1781
2095
  sentence: () => "hit the subscription rate limit",
1782
2096
  },
2097
+ // Released like the rate limit, so its sentence is kept for the same reason.
2098
+ "api-error": {
2099
+ cell: "API failure",
2100
+ sentence: () => "was stopped by an API failure the CLI had given up retrying",
2101
+ },
1783
2102
  "timed-out": {
1784
2103
  cell: "timed out",
1785
2104
  sentence: (limits) => `ran past its ${limits.maxMinutes} minute limit`,