ticket-runner 0.6.2 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,11 @@
1
1
  import { branchName, worktreePath } from "./branch.js";
2
2
  import type { Config } from "./config.js";
3
+ import {
4
+ CREDENTIAL_WORDS,
5
+ type Credential,
6
+ type StrayStage,
7
+ expectedKeySource,
8
+ } from "./credential.js";
3
9
  import { tickMetCriteria } from "./criteria.js";
4
10
  import type { BaseBranchChecks, FailingBase } from "./failing-base.js";
5
11
  import { type SkipReason, isGuardReason, skipReason } from "./guards.js";
@@ -41,6 +47,7 @@ import { retryLogDir, stageLogDir } from "./run-log.js";
41
47
  import { holdsClaim } from "./stranded.js";
42
48
  import {
43
49
  RELEASE_WORDS,
50
+ releaseWords,
44
51
  draftPullRequestBody,
45
52
  failingBaseNote,
46
53
  failingBaseSummary,
@@ -126,6 +133,17 @@ export interface Pipeline {
126
133
  * can point at the Run's directory: a cloud Host's is gone with its session.
127
134
  */
128
135
  host: HostKind;
136
+ /**
137
+ * What every Stage of this Run is billed to, chosen once when it started:
138
+ * the subscription, or the Host's API key for a Run started with `--api-key`.
139
+ */
140
+ credential: Credential;
141
+ /**
142
+ * Every Stage of this Run that said it ran on something other than
143
+ * {@link credential}, in the order they came back, shared by every Ticket so
144
+ * the Run summary can name them all.
145
+ */
146
+ strayStages: StrayStage[];
129
147
  /**
130
148
  * The branch every Ticket of this Run branches from, rebases onto, merges
131
149
  * into and pulls, resolved once before the Run started
@@ -177,18 +195,21 @@ class TicketFailure extends Error {
177
195
  }
178
196
 
179
197
  /**
180
- * What stops a Ticket that says nothing about it: the subscription rate limit,
181
- * a transient API failure — the API overloaded, broken or silent on its own
182
- * side after the CLI had already retried it — and a Base branch that fails the
183
- * Check the Ticket's branch failed.
198
+ * What stops a Ticket that says nothing about it: the rate limit, a transient
199
+ * API failure — the API overloaded, broken or silent on its own side after the
200
+ * CLI had already retried it — the API key of an `--api-key` Run out of credit
201
+ * or past its workspace's spend limit, and a Base branch that fails the Check
202
+ * the Ticket's branch failed.
184
203
  */
185
204
  export type ReleaseCause = StageReleaseCause | "failing-base";
186
205
 
187
206
  /** The Stage failures among them. */
188
- type StageReleaseCause = Extract<StageFailure, "rate-limited" | "api-error">;
207
+ type StageReleaseCause = Extract<StageFailure, "rate-limited" | "api-error" | "out-of-budget">;
189
208
 
190
209
  function releaseCause(result: StageResult): StageReleaseCause | undefined {
191
- return result.failure === "rate-limited" || result.failure === "api-error"
210
+ return result.failure === "rate-limited" ||
211
+ result.failure === "api-error" ||
212
+ result.failure === "out-of-budget"
192
213
  ? result.failure
193
214
  : undefined;
194
215
  }
@@ -883,11 +904,16 @@ async function passOver(
883
904
  * Start one Stage under the model, effort, turn and wall-clock limits its own
884
905
  * config names. Everything a Stage differs in is the prompt, where it runs and
885
906
  * where it logs; the limits come from one place so no Stage can quietly skip them.
907
+ *
908
+ * The credential is the Run's too, and what the Stage says it actually ran on
909
+ * is checked here for the same reason: a Stage billed somewhere the Run did not
910
+ * choose is named whichever Stage it was.
886
911
  */
887
- function runStage(
912
+ async function runStage(
888
913
  pipeline: Pipeline,
889
914
  stage: StageName,
890
915
  request: {
916
+ ticket: number;
891
917
  prompt: string;
892
918
  cwd: string;
893
919
  logDir: string;
@@ -896,12 +922,13 @@ function runStage(
896
922
  },
897
923
  ): Promise<StageResult> {
898
924
  const limits = pipeline.config.stages[stage];
899
- return pipeline.runner.run({
925
+ const result = await pipeline.runner.run({
900
926
  stage,
901
927
  prompt: request.prompt,
902
928
  cwd: request.cwd,
903
929
  logDir: request.logDir,
904
930
  permissionMode: pipeline.config.permissionMode,
931
+ credential: pipeline.credential,
905
932
  model: limits.model,
906
933
  effort: limits.effort,
907
934
  maxTurns: limits.maxTurns,
@@ -911,6 +938,30 @@ function runStage(
911
938
  ? {}
912
939
  : { resultRequired: request.resultRequired }),
913
940
  });
941
+ noteStrayCredential(pipeline, request.ticket, stage, result);
942
+ return result;
943
+ }
944
+
945
+ /**
946
+ * Name a Stage whose `system/init` event reported another `apiKeySource` than
947
+ * the Run's credential expects, in the log now and in the summary at the end.
948
+ * A Stage that reported none, one that never started, is not named: there is
949
+ * nothing it said to set against the Run's choice.
950
+ */
951
+ function noteStrayCredential(
952
+ pipeline: Pipeline,
953
+ ticket: number,
954
+ stage: StageName,
955
+ result: StageResult,
956
+ ): void {
957
+ const { apiKeySource } = result;
958
+ const expected = expectedKeySource(pipeline.credential);
959
+ if (apiKeySource === undefined || apiKeySource === expected) return;
960
+ pipeline.strayStages.push({ ticket, stage, apiKeySource });
961
+ pipeline.log?.(
962
+ `#${ticket} ${stage} ran on apiKeySource ${apiKeySource}, ` +
963
+ `not the ${CREDENTIAL_WORDS[pipeline.credential]} (${expected})`,
964
+ );
914
965
  }
915
966
 
916
967
  /**
@@ -924,13 +975,16 @@ function runStage(
924
975
  * work that is already on the branch.
925
976
  */
926
977
  async function releasedStage(
978
+ pipeline: Pipeline,
927
979
  progress: Progress,
928
980
  stage: StageName,
929
981
  point: FailurePoint,
930
982
  result: StageResult,
931
983
  cause: StageReleaseCause,
932
984
  ): Promise<Releasing> {
933
- await progress.record(stageRow(stage, result, `⏸ ${STAGE_FAILURES[cause].cell}`));
985
+ await progress.record(
986
+ stageRow(stage, result, `⏸ ${stageFailureCell(result, pipeline.credential)}`),
987
+ );
934
988
  return new Releasing(
935
989
  point,
936
990
  stage === "implement" ? "claimed" : "implemented",
@@ -955,13 +1009,15 @@ async function stageDidNotFinish(
955
1009
  ): Promise<TicketFailure | Releasing> {
956
1010
  const cause = releaseCause(result);
957
1011
  if (cause !== undefined) {
958
- return await releasedStage(progress, stage, stage, result, cause);
1012
+ return await releasedStage(pipeline, progress, stage, stage, result, cause);
959
1013
  }
960
1014
 
961
- await progress.record(stageRow(stage, result, `❌ ${stageFailureCell(result)}`));
1015
+ await progress.record(
1016
+ stageRow(stage, result, `❌ ${stageFailureCell(result, pipeline.credential)}`),
1017
+ );
962
1018
  return new TicketFailure(
963
1019
  stage,
964
- describeStageFailure(stage, pipeline.config.stages[stage], result),
1020
+ describeStageFailure(stage, pipeline.config.stages[stage], result, pipeline.credential),
965
1021
  );
966
1022
  }
967
1023
 
@@ -1069,6 +1125,7 @@ async function implement(
1069
1125
  ): Promise<void> {
1070
1126
  const stage = pipeline.config.stages.implement;
1071
1127
  const result = await runStage(pipeline, "implement", {
1128
+ ticket: issue.number,
1072
1129
  prompt: implementPrompt(
1073
1130
  issue.url,
1074
1131
  pipeline.baseBranch,
@@ -1314,6 +1371,7 @@ async function verify(
1314
1371
  ): Promise<Verdict> {
1315
1372
  const stage = pipeline.config.stages.verify;
1316
1373
  const result = await runStage(pipeline, "verify", {
1374
+ ticket: issue.number,
1317
1375
  prompt: verifyPrompt(
1318
1376
  issue.url,
1319
1377
  stage.extraPrompt,
@@ -1406,6 +1464,7 @@ async function fix(
1406
1464
  const commitsBefore = await commitCount(pipeline, branch);
1407
1465
 
1408
1466
  const result = await runStage(pipeline, "fix", {
1467
+ ticket: issue.number,
1409
1468
  prompt: fixPrompt(
1410
1469
  issue.url,
1411
1470
  failure,
@@ -1520,6 +1579,7 @@ async function conflictStage(
1520
1579
  const stage = pipeline.config.stages.conflict;
1521
1580
  try {
1522
1581
  const result = await runStage(pipeline, "conflict", {
1582
+ ticket: issue.number,
1523
1583
  prompt: conflictPrompt(
1524
1584
  issue.url,
1525
1585
  conflict,
@@ -1550,7 +1610,7 @@ async function conflictStage(
1550
1610
  // rebases is clean.
1551
1611
  const cause = releaseCause(result);
1552
1612
  if (cause !== undefined) {
1553
- return await releasedStage(progress, "conflict", "rebase", result, cause);
1613
+ return await releasedStage(pipeline, progress, "conflict", "rebase", result, cause);
1554
1614
  }
1555
1615
 
1556
1616
  await progress.record(stageRow("conflict", result, "❌ unresolved"));
@@ -1558,7 +1618,7 @@ async function conflictStage(
1558
1618
  "rebase",
1559
1619
  result.ok
1560
1620
  ? `the conflict Stage did not finish the rebase onto ${pipeline.baseBranch}`
1561
- : describeStageFailure("conflict", stage, result),
1621
+ : describeStageFailure("conflict", stage, result, pipeline.credential),
1562
1622
  [conflict, state.unresolved].join("\n\n"),
1563
1623
  "unresolved-conflict",
1564
1624
  );
@@ -1744,7 +1804,7 @@ async function release(
1744
1804
  await tracker.removeLabel(ticket, config.labels.inProgress);
1745
1805
  await tracker.unassign(ticket, user);
1746
1806
  pipeline.log?.(
1747
- `#${ticket} released at ${stopped.point} · ${RELEASE_WORDS[stopped.reason]} · ${branch}`,
1807
+ `#${ticket} released at ${stopped.point} · ${releaseWords(stopped.reason, pipeline.credential)} · ${branch}`,
1748
1808
  );
1749
1809
 
1750
1810
  return {
@@ -2099,6 +2159,11 @@ const STAGE_FAILURES: Record<
2099
2159
  cell: "API failure",
2100
2160
  sentence: () => "was stopped by an API failure the CLI had given up retrying",
2101
2161
  },
2162
+ // And so is this one.
2163
+ "out-of-budget": {
2164
+ cell: "out of API budget",
2165
+ sentence: () => "ran out of API budget: the key's credit or its workspace's spend limit",
2166
+ },
2102
2167
  "timed-out": {
2103
2168
  cell: "timed out",
2104
2169
  sentence: (limits) => `ran past its ${limits.maxMinutes} minute limit`,
@@ -2117,7 +2182,14 @@ const STAGE_FAILURES: Record<
2117
2182
  /** A Stage that came back without saying why gets the one word that is true. */
2118
2183
  const UNEXPLAINED = { cell: "failed", sentence: () => "failed" } as const;
2119
2184
 
2120
- function stageFailure(result: StageResult) {
2185
+ /**
2186
+ * The rate limit of an `--api-key` Run, which is the API's own rather than the
2187
+ * subscription's, and is named so wherever the subscription's would be.
2188
+ */
2189
+ const API_RATE_LIMIT = { cell: "API rate limited", sentence: () => "hit the API rate limit" } as const;
2190
+
2191
+ function stageFailure(result: StageResult, credential: Credential) {
2192
+ if (result.failure === "rate-limited" && credential === "api-key") return API_RATE_LIMIT;
2121
2193
  return result.failure ? STAGE_FAILURES[result.failure] : UNEXPLAINED;
2122
2194
  }
2123
2195
 
@@ -2125,11 +2197,12 @@ function describeStageFailure(
2125
2197
  stage: string,
2126
2198
  limits: StageLimits,
2127
2199
  result: StageResult,
2200
+ credential: Credential,
2128
2201
  ): string {
2129
- return `the ${stage} Stage ${stageFailure(result).sentence(limits)}`;
2202
+ return `the ${stage} Stage ${stageFailure(result, credential).sentence(limits)}`;
2130
2203
  }
2131
2204
 
2132
2205
  /** The same failure, short enough for a table cell. */
2133
- function stageFailureCell(result: StageResult): string {
2134
- return stageFailure(result).cell;
2206
+ function stageFailureCell(result: StageResult, credential: Credential): string {
2207
+ return stageFailure(result, credential).cell;
2135
2208
  }
@@ -1,3 +1,5 @@
1
+ import type { Credential } from "../credential.js";
2
+
1
3
  /**
2
4
  * One Claude Code session per Stage (ADR-0002).
3
5
  *
@@ -25,14 +27,16 @@ export const EFFORT_LEVELS = ["low", "medium", "high", "xhigh", "max"] as const;
25
27
  export type Effort = (typeof EFFORT_LEVELS)[number];
26
28
 
27
29
  /**
28
- * Why a Stage did not finish cleanly. `rate-limited` and `api-error` are singled
29
- * out because the pipeline releases a claim rather than blaming the Ticket for
30
- * either: one is the subscription running out of room, the other the API failing
31
- * on its side after the CLI had given up retrying it.
30
+ * Why a Stage did not finish cleanly. `rate-limited`, `api-error` and
31
+ * `out-of-budget` are singled out because the pipeline releases a claim rather
32
+ * than blaming the Ticket for any of them: the rate limit, the API failing on
33
+ * its side after the CLI had given up retrying it, and the API key's
34
+ * organization out of credit or its workspace past its spend limit.
32
35
  */
33
36
  export type StageFailure =
34
37
  | "rate-limited"
35
38
  | "api-error"
39
+ | "out-of-budget"
36
40
  | "timed-out"
37
41
  | "turn-capped"
38
42
  | "nonzero-exit"
@@ -48,6 +52,11 @@ export interface StageRequest {
48
52
  maxTurns: number;
49
53
  maxMinutes: number;
50
54
  permissionMode: PermissionMode;
55
+ /**
56
+ * What the session is billed to: the subscription, with the Host's API key
57
+ * kept out of its environment, or that key.
58
+ */
59
+ credential: Credential;
51
60
  /** Where the command line, stdout, stderr and transcript are written. */
52
61
  logDir: string;
53
62
  /** When set, the Stage is asked for structured output matching this schema. */
@@ -80,6 +89,13 @@ export interface StageResult {
80
89
  * when the CLI said.
81
90
  */
82
91
  limitResetsAt?: string;
92
+ /**
93
+ * What the session said it was billed to, as its `system/init` event's
94
+ * `apiKeySource` names it: `none` for the subscription login,
95
+ * `ANTHROPIC_API_KEY`, `apiKeyHelper` and so on for a key. Absent where the
96
+ * session printed no such event.
97
+ */
98
+ apiKeySource?: string;
83
99
  }
84
100
 
85
101
  /**
@@ -374,6 +374,20 @@ export interface Workspace {
374
374
  * where the work is once this Host is gone only when that is true.
375
375
  */
376
376
  hasRemoteBranch(branch: string): Promise<boolean>;
377
+ /**
378
+ * The remote every branch is fetched from and pushed to, by the name a human
379
+ * types after `git push`, so a message can spell a branch on it as `origin/main`.
380
+ */
381
+ readonly remote: string;
382
+ /**
383
+ * The contents of `path` at the tip of `branch` as the remote holds it now,
384
+ * which is what every Ticket's worktree is made from, and nothing when the
385
+ * remote has no such branch or the branch no such file.
386
+ *
387
+ * Asked by Target readiness alone, before a Run takes the lock: a file `init`
388
+ * wrote and nobody pushed is in the checkout and in no Stage's worktree.
389
+ */
390
+ readRemoteFile(branch: string, path: string): Promise<string | undefined>;
377
391
  /**
378
392
  * Delete `branch` on the remote; the PR is merged, so nothing references it.
379
393
  * A branch the remote already deleted, as GitHub does on merge when told to,
package/src/readiness.ts CHANGED
@@ -4,6 +4,7 @@ import type { Labels } from "./config.js";
4
4
  import { CLAUDE_SECTION, CONVENTIONS_PATH, conventionsMark } from "./conventions.js";
5
5
  import { OPERATOR_SKILL_PATH } from "./operator-skill.js";
6
6
  import type { Authentication, Tracker } from "./ports/tracker.js";
7
+ import type { Workspace } from "./ports/workspace.js";
7
8
 
8
9
  /**
9
10
  * What `ticket-runner init` must have left in a Target before a Run may start.
@@ -22,17 +23,26 @@ import type { Authentication, Tracker } from "./ports/tracker.js";
22
23
  * Version it names: an Operator installs the pipeline a cloud Host runs at that
23
24
  * Version, and a Target without one leaves it guessing (ADR-0008).
24
25
  *
25
- * The Target's own files are read here rather than through a port, which every
26
+ * The checkout's files are read here rather than through a port, which every
26
27
  * external effect of a Run goes through. What a port buys is a fake to drive the
27
- * state machine with, and there is no state machine here: this is five questions
28
+ * state machine with, and there is no state machine here: these are questions
28
29
  * asked of a directory before a Run exists, and the answers are already driven
29
30
  * from a temporary repository root, which is how the local state under
30
- * `.ticket-runner/` is tested too (ADR-0004). Widening the `Workspace` port for
31
- * them would put reads no Stage and no Ticket ever makes on the interface the
32
- * orchestrator depends on.
31
+ * `.ticket-runner/` is tested too (ADR-0004).
32
+ *
33
+ * The same files on the remote's Base branch are the one exception, read
34
+ * through `Workspace` like every other question about the remote: every
35
+ * Ticket's worktree is made from that branch, so a file `init` wrote and nobody
36
+ * pushed is in the checkout and in no Stage's worktree. The conventions
37
+ * document, the `CLAUDE.md` pointer and the Operator's skill are asked of it
38
+ * the same way they are asked of the checkout. The gitignore lines are not:
39
+ * the directories they cover exist only in the main checkout, and only its
40
+ * copy takes effect there.
33
41
  *
34
42
  * Every item is asked on every Host, the cloud's own needs included, so a
35
- * Target a workstation accepts is never one a cloud Run then refuses.
43
+ * Target a workstation accepts is never one a cloud Run then refuses. A cloud
44
+ * Host's checkout is a clone of the remote, so there the remote's answers are
45
+ * the checkout's.
36
46
  */
37
47
 
38
48
  /** The file every agent session in a Target reads first. */
@@ -148,6 +158,10 @@ export interface ReadinessOptions {
148
158
  * is refused without a network call at all. Whether `gh` is there to ask comes
149
159
  * next, so a Host without it is refused in words rather than with the spawn
150
160
  * error the first label lookup would crash on.
161
+ *
162
+ * What the remote's Base branch lacks is asked after all of these, by
163
+ * {@link unpushedRefusal}, once the Run has resolved which branch that is: the
164
+ * lookup may ask GitHub, which only a `gh` that runs can.
151
165
  */
152
166
  export async function readinessRefusal({
153
167
  repoRoot,
@@ -155,7 +169,7 @@ export async function readinessRefusal({
155
169
  tracker,
156
170
  }: ReadinessOptions): Promise<string | undefined> {
157
171
  const missing =
158
- missingFileMessage(repoRoot) ??
172
+ (await missingFileMessage(repoRoot)) ??
159
173
  (await missingGhMessage(tracker)) ??
160
174
  (await missingLabelMessage(tracker, labels)) ??
161
175
  (await missingBranchDeletionMessage(tracker));
@@ -168,14 +182,56 @@ export async function readinessRefusal({
168
182
  ].join(" ");
169
183
  }
170
184
 
185
+ export interface UnpushedOptions {
186
+ workspace: Pick<Workspace, "remote" | "readRemoteFile">;
187
+ /** The Run's Base branch, resolved once and carried on into the Run. */
188
+ baseBranch: string;
189
+ }
190
+
191
+ /**
192
+ * The message a Run is refused with when the remote's Base branch lacks a file
193
+ * the checkout has, and nothing when it has them all.
194
+ *
195
+ * Asked once {@link readinessRefusal} has found nothing missing, so a Target
196
+ * that never ran `init` is told to run it and only one that did is told to
197
+ * push. A Base branch the remote does not have lacks every file. The remedy is
198
+ * a commit and a push, which `init` leaves to the human, so this names no
199
+ * command of the pipeline's.
200
+ */
201
+ export async function unpushedRefusal({
202
+ workspace,
203
+ baseBranch,
204
+ }: UnpushedOptions): Promise<string | undefined> {
205
+ const remoteBase = `${workspace.remote}/${baseBranch}`;
206
+ const missing = await missingItemMessage((path) => workspace.readRemoteFile(baseBranch, path));
207
+ if (missing === undefined) return undefined;
208
+
209
+ return [
210
+ `This Target's \`${remoteBase}\` is not set up: ${missing}.`,
211
+ `Every Ticket's worktree is made from \`${remoteBase}\`, not from this checkout:`,
212
+ "commit and push what `init` wrote there, then start again.",
213
+ ].join(" ");
214
+ }
215
+
171
216
  /** Names the first thing `init` writes into a Target that this one does not have. */
172
- function missingFileMessage(repoRoot: string): string | undefined {
217
+ async function missingFileMessage(repoRoot: string): Promise<string | undefined> {
173
218
  const [unignored] = missingIgnoreLines(repoRoot);
174
219
  if (unignored !== undefined) {
175
220
  return `\`.gitignore\` does not ignore \`${unignored.line}\``;
176
221
  }
177
222
 
178
- const conventions = readTargetFile(join(repoRoot, CONVENTIONS_PATH));
223
+ return missingItemMessage(async (path) => readTargetFile(join(repoRoot, path)));
224
+ }
225
+
226
+ /**
227
+ * Names the first of the items both the checkout and the remote's Base branch
228
+ * must carry that `read` does not find, asking each the same question in both
229
+ * places. Where it was asked is the refusal's to say.
230
+ */
231
+ async function missingItemMessage(
232
+ read: (path: string) => Promise<string | undefined>,
233
+ ): Promise<string | undefined> {
234
+ const conventions = await read(CONVENTIONS_PATH);
179
235
  if (conventions === undefined) {
180
236
  return `\`${CONVENTIONS_PATH}\` is not there`;
181
237
  }
@@ -183,13 +239,13 @@ function missingFileMessage(repoRoot: string): string | undefined {
183
239
  return `\`${CONVENTIONS_PATH}\` carries no Version`;
184
240
  }
185
241
 
186
- if (!hasClaudePointer(readTargetFile(join(repoRoot, CLAUDE_FILENAME)))) {
242
+ if (!hasClaudePointer(await read(CLAUDE_FILENAME))) {
187
243
  return `\`${CLAUDE_FILENAME}\` does not point at \`${CONVENTIONS_PATH}\``;
188
244
  }
189
245
 
190
246
  // Asked of a workstation too, though only a cloud session reads it: a Target
191
247
  // a local Run accepts is then one the app can run as well (ADR-0008).
192
- if (readTargetFile(join(repoRoot, OPERATOR_SKILL_PATH)) === undefined) {
248
+ if ((await read(OPERATOR_SKILL_PATH)) === undefined) {
193
249
  return `the Operator's skill \`${OPERATOR_SKILL_PATH}\` is not there`;
194
250
  }
195
251
 
package/src/run.ts CHANGED
@@ -5,7 +5,7 @@ import type { RoutedNote } from "./notes.js";
5
5
  import { type Pipeline, type TicketOutcome, processTicket } from "./orchestrator.js";
6
6
  import { StopSignal, stopLine } from "./stop.js";
7
7
  import { holdsClaim, strandedTickets } from "./stranded.js";
8
- import { RELEASE_WORDS } from "./templates.js";
8
+ import { releaseWords } from "./templates.js";
9
9
 
10
10
  /**
11
11
  * What a Run has to show for itself: every Ticket it took, in the order they
@@ -46,6 +46,7 @@ export type RunStop =
46
46
  resetsAt?: string;
47
47
  }
48
48
  | { reason: "api-error" }
49
+ | { reason: "out-of-budget" }
49
50
  | {
50
51
  reason: "failing-base";
51
52
  baseBranch: string;
@@ -76,10 +77,10 @@ export type RunStop =
76
77
  * not cost the rest of the night.
77
78
  *
78
79
  * A Release stops the Run filling Lanes and nothing more. Nothing was wrong with
79
- * the Ticket — the subscription ran out of room, or the API failed on its own
80
- * side after the CLI had given up retrying it — so the next one would be stopped
81
- * the same way, and claiming it would spend a Claim, a worktree and a doomed
82
- * Stage to learn what the first one already said. The Lanes still busy finish
80
+ * the Ticket — the subscription or the API key ran out of room, or the API
81
+ * failed on its own side after the CLI had given up retrying it — so the next
82
+ * one would be stopped the same way, and claiming it would spend a Claim, a
83
+ * worktree and a doomed Stage to learn what the first one already said. The Lanes still busy finish
83
84
  * what they hold, and the Run reports what stopped it once the last of them
84
85
  * comes back. It does not wait for the limit to reset: a human or an Operator
85
86
  * starts the next Run, and the summary says when the limit resets.
@@ -127,8 +128,8 @@ export async function processRun(
127
128
  // Why this Run stopped, which is also what it reports. Set by whichever of
128
129
  // the Release and the Stop came first: a Stop that arrives after a Release
129
130
  // asked for what was already happening, and a Release afterwards stopped
130
- // nothing that was not stopped already. Only a later rate limit overwrites
131
- // it, and only where limitOutranks says so. Undefined is a Run still taking
131
+ // nothing that was not stopped already. Only a later API budget or rate
132
+ // limit overwrites it, and only where `outranks` says so. Undefined is a Run still taking
132
133
  // Tickets, which is why it doubles as the guard at every refill.
133
134
  let runStop: RunStop | undefined;
134
135
  // A Frontier the tracker refused to list, kept until the Lanes still busy are
@@ -166,14 +167,13 @@ export async function processRun(
166
167
  * Put a Ticket in a Lane, and record what came back when it ends.
167
168
  *
168
169
  * The line about the Run stopping is logged by the Release that stopped it and
169
- * by no other, even where a later rate limit replaces the API failure it
170
- * stopped the Run with. The Ticket's own release is already logged where it
171
- * happened; this says the Run goes no further, so a transcript shows the
172
- * Frontier was left alone rather than found empty, and a Release into a Run
173
- * that an earlier Release or a Stop had stopped already claims to have
174
- * stopped nothing. It leads with the Ticket's number all the same: a
175
- * transcript of interleaved Lanes must not lose the line that explains why
176
- * one Lane was the last to be filled.
170
+ * by no other, even where a later Release outranks it on the last line. The
171
+ * Ticket's own release is already logged where it happened; this says the Run
172
+ * goes no further, so a transcript shows the Frontier was left alone rather
173
+ * than found empty, and a Release into a Run that an earlier Release or a
174
+ * Stop had stopped already claims to have stopped nothing. It leads with the
175
+ * Ticket's number all the same: a transcript of interleaved Lanes must not
176
+ * lose the line that explains why one Lane was the last to be filled.
177
177
  */
178
178
  function fillLane(ticket: Taken): void {
179
179
  taken.add(ticket.number);
@@ -185,8 +185,10 @@ export async function processRun(
185
185
  if (outcome.outcome !== "released") return;
186
186
  if (runStop === undefined) {
187
187
  runStop = releaseStop(outcome, pipeline.baseBranch);
188
- pipeline.log?.(`#${outcome.ticket} stopped the Run · ${RELEASE_WORDS[outcome.cause]}`);
189
- } else if (outcome.cause === "rate-limited" && limitOutranks(runStop)) {
188
+ pipeline.log?.(
189
+ `#${outcome.ticket} stopped the Run · ${releaseWords(outcome.cause, pipeline.credential)}`,
190
+ );
191
+ } else if (outranks(outcome, runStop)) {
190
192
  runStop = releaseStop(outcome, pipeline.baseBranch);
191
193
  }
192
194
  }),
@@ -262,6 +264,7 @@ function releaseStop(
262
264
  baseBranch: string,
263
265
  ): RunStop {
264
266
  if (outcome.cause === "api-error") return { reason: "api-error" };
267
+ if (outcome.cause === "out-of-budget") return { reason: "out-of-budget" };
265
268
  if (outcome.failingBase !== undefined) {
266
269
  return { reason: "failing-base", baseBranch, ...outcome.failingBase };
267
270
  }
@@ -272,14 +275,30 @@ function releaseStop(
272
275
  }
273
276
 
274
277
  /**
275
- * Whether a Ticket the rate limit released after `stop` puts the limit in its
276
- * place. An API failure says nothing of when the next Run can start, so the
277
- * limit outranks it; a limit nobody was told the end of is outranked by one a
278
+ * Whether a Ticket released after `stop` puts its own stop in its place.
279
+ *
280
+ * An API key out of budget outranks the rate limit and an API failure, since
281
+ * it alone needs the human to act: no wait mends it, only the Console. The
282
+ * rate limit outranks an API failure, which says nothing of when the next Run
283
+ * can start, and a limit nobody was told the end of is outranked by one a
278
284
  * Stage was told. A failing Base branch and a Stop stay whatever comes after
279
- * them, and so does a limit already told when it resets.
285
+ * them, and so does an API budget, and a limit already told when it resets.
280
286
  */
281
- function limitOutranks(stop: RunStop): boolean {
282
- return stop.reason === "api-error" || (stop.reason === "rate-limited" && stop.resetsAt === undefined);
287
+ function outranks(
288
+ outcome: Extract<TicketOutcome, { outcome: "released" }>,
289
+ stop: RunStop,
290
+ ): boolean {
291
+ switch (outcome.cause) {
292
+ case "out-of-budget":
293
+ return stop.reason === "rate-limited" || stop.reason === "api-error";
294
+ case "rate-limited":
295
+ return (
296
+ stop.reason === "api-error" ||
297
+ (stop.reason === "rate-limited" && stop.resetsAt === undefined)
298
+ );
299
+ default:
300
+ return false;
301
+ }
283
302
  }
284
303
 
285
304
  /**