ticket-runner 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ko.md +3 -3
- package/README.md +4 -3
- package/package.json +3 -3
- package/src/adapters/claude-agent-runner.ts +108 -15
- package/src/adapters/exec.ts +35 -6
- package/src/adapters/gh-tracker.ts +10 -46
- package/src/adapters/git-workspace.ts +23 -2
- package/src/adapters/process-tree.ts +227 -0
- package/src/branch.ts +8 -0
- package/src/config.ts +3 -3
- package/src/conventions.ts +2 -1
- package/src/failing-base.ts +132 -0
- package/src/guards.ts +1 -1
- package/src/host.ts +1 -1
- package/src/log-tail.ts +51 -0
- package/src/note-schema.ts +21 -0
- package/src/notes.ts +8 -4
- package/src/orchestrator.ts +378 -59
- package/src/ports/agent-runner.ts +11 -2
- package/src/ports/tracker.ts +5 -4
- package/src/ports/workspace.ts +24 -2
- package/src/prompts.ts +125 -18
- package/src/resume.ts +7 -5
- package/src/run.ts +77 -25
- package/src/start.ts +5 -1
- package/src/templates.ts +94 -7
- package/src/title.ts +2 -2
package/src/orchestrator.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { branchName, worktreePath } from "./branch.js";
|
|
2
2
|
import type { Config } from "./config.js";
|
|
3
3
|
import { tickMetCriteria } from "./criteria.js";
|
|
4
|
+
import type { BaseBranchChecks, FailingBase } from "./failing-base.js";
|
|
4
5
|
import { type SkipReason, isGuardReason, skipReason } from "./guards.js";
|
|
5
6
|
import { markHandoffsTaken } from "./handoff.js";
|
|
6
7
|
import type { HostKind } from "./host.js";
|
|
@@ -23,6 +24,7 @@ import type { Issue, Tracker } from "./ports/tracker.js";
|
|
|
23
24
|
import type {
|
|
24
25
|
KeptTranscripts,
|
|
25
26
|
ReachedState,
|
|
27
|
+
RebaseOutcome,
|
|
26
28
|
TicketState,
|
|
27
29
|
Workspace,
|
|
28
30
|
WorktreeFromRemote,
|
|
@@ -38,7 +40,10 @@ import {
|
|
|
38
40
|
import { retryLogDir, stageLogDir } from "./run-log.js";
|
|
39
41
|
import { holdsClaim } from "./stranded.js";
|
|
40
42
|
import {
|
|
43
|
+
RELEASE_WORDS,
|
|
41
44
|
draftPullRequestBody,
|
|
45
|
+
failingBaseNote,
|
|
46
|
+
failingBaseSummary,
|
|
42
47
|
guardComment,
|
|
43
48
|
handoffComment,
|
|
44
49
|
hasGuardWarning,
|
|
@@ -81,8 +86,17 @@ export type TicketOutcome =
|
|
|
81
86
|
ticket: number;
|
|
82
87
|
title: string;
|
|
83
88
|
branch: string;
|
|
84
|
-
/** Where the
|
|
89
|
+
/** Where the Stage was stopped, which is most of what the summary says about it. */
|
|
85
90
|
stage: FailurePoint;
|
|
91
|
+
/** What stopped it, which is the rest. */
|
|
92
|
+
cause: ReleaseCause;
|
|
93
|
+
/**
|
|
94
|
+
* When the rate limit resets, ISO-8601, where the Stage it stopped was
|
|
95
|
+
* told. Absent for an API failure, which says nothing of when it ends.
|
|
96
|
+
*/
|
|
97
|
+
limitResetsAt?: string;
|
|
98
|
+
/** The Check the Base branch fails, and where, for a Release over it. */
|
|
99
|
+
failingBase?: { commit: string; command: string };
|
|
86
100
|
notes: RoutedNote[];
|
|
87
101
|
}
|
|
88
102
|
| {
|
|
@@ -130,6 +144,12 @@ export interface Pipeline {
|
|
|
130
144
|
* them are told which one it is.
|
|
131
145
|
*/
|
|
132
146
|
standingNotes: StandingNotes;
|
|
147
|
+
/**
|
|
148
|
+
* The Checks the Run ran at merge-bases with the Base branch, made once and
|
|
149
|
+
* shared by every Ticket: the Tickets that fail together over one broken
|
|
150
|
+
* Base branch run the Check there once between them, and post one Note.
|
|
151
|
+
*/
|
|
152
|
+
baseChecks: BaseBranchChecks;
|
|
133
153
|
/**
|
|
134
154
|
* Whether the Target is the pipeline's own repository, decided once before
|
|
135
155
|
* the Run started ({@link import("./self-hosting.js").isPipelineRepository}).
|
|
@@ -157,21 +177,42 @@ class TicketFailure extends Error {
|
|
|
157
177
|
}
|
|
158
178
|
|
|
159
179
|
/**
|
|
160
|
-
*
|
|
180
|
+
* What stops a Ticket that says nothing about it: the subscription rate limit,
|
|
181
|
+
* a transient API failure — the API overloaded, broken or silent on its own
|
|
182
|
+
* side after the CLI had already retried it — and a Base branch that fails the
|
|
183
|
+
* Check the Ticket's branch failed.
|
|
184
|
+
*/
|
|
185
|
+
export type ReleaseCause = StageReleaseCause | "failing-base";
|
|
186
|
+
|
|
187
|
+
/** The Stage failures among them. */
|
|
188
|
+
type StageReleaseCause = Extract<StageFailure, "rate-limited" | "api-error">;
|
|
189
|
+
|
|
190
|
+
function releaseCause(result: StageResult): StageReleaseCause | undefined {
|
|
191
|
+
return result.failure === "rate-limited" || result.failure === "api-error"
|
|
192
|
+
? result.failure
|
|
193
|
+
: undefined;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* A Stage something other than the Ticket stopped, on its way to a Release.
|
|
161
198
|
*
|
|
162
|
-
* The
|
|
163
|
-
*
|
|
164
|
-
*
|
|
199
|
+
* The Stage failures that say nothing about the Ticket are the ones that
|
|
200
|
+
* neither end it nor spend its Fix budget: the Claim is released and a State
|
|
201
|
+
* file is left for the next Run to resume from (ADR-0004). It is not a
|
|
165
202
|
* {@link TicketFailure} because every path that handles one of those would
|
|
166
203
|
* either hand the Ticket off or buy it a fix Stage, and this is neither.
|
|
167
204
|
*/
|
|
168
|
-
class
|
|
205
|
+
class Releasing extends Error {
|
|
169
206
|
constructor(
|
|
170
207
|
readonly point: FailurePoint,
|
|
171
208
|
/** The state the Ticket reached, which is where a later Run picks it up. */
|
|
172
209
|
readonly state: ReachedState,
|
|
210
|
+
readonly reason: ReleaseCause,
|
|
211
|
+
readonly limitResetsAt?: string,
|
|
212
|
+
/** The Check the Base branch fails too, for a Release over it. */
|
|
213
|
+
readonly failingBase?: FailingBase,
|
|
173
214
|
) {
|
|
174
|
-
super(`the ${point} Stage
|
|
215
|
+
super(`the ${point} Stage was stopped · ${RELEASE_WORDS[reason]}`);
|
|
175
216
|
}
|
|
176
217
|
}
|
|
177
218
|
|
|
@@ -194,9 +235,10 @@ function asTicketFailure(error: unknown, point: FailurePoint): TicketFailure {
|
|
|
194
235
|
* Nothing ends in a merge that has not been through a green pass of the whole
|
|
195
236
|
* gauntlet.
|
|
196
237
|
*
|
|
197
|
-
* A Stage the subscription rate limit
|
|
198
|
-
* released, and a Run started once the limit has
|
|
199
|
-
*
|
|
238
|
+
* A Stage the subscription rate limit or a transient API failure stopped ends
|
|
239
|
+
* none of that: the Ticket is released, and a Run started once the limit has
|
|
240
|
+
* reset, or the API recovered, resumes it from the state it had reached rather
|
|
241
|
+
* than from the top. A hand-off keeps that state too, so a
|
|
200
242
|
* human who relabels the Ticket `ready-for-agent` hands it back and the next Run
|
|
201
243
|
* carries on from what it reached rather than paying for the implement Stage a
|
|
202
244
|
* second time.
|
|
@@ -289,6 +331,7 @@ async function takeTicket(
|
|
|
289
331
|
fixUsed: resume?.fixUsed ?? false,
|
|
290
332
|
...(resume?.pullRequest === undefined ? {} : { pullRequest: resume.pullRequest }),
|
|
291
333
|
...(resume?.title === undefined ? {} : { title: resume.title }),
|
|
334
|
+
...(resume?.rebaseFirst === true ? { rebaseFirst: true } : {}),
|
|
292
335
|
});
|
|
293
336
|
|
|
294
337
|
// The claim is the first write on the board: it shows what the pipeline holds
|
|
@@ -327,6 +370,10 @@ async function takeTicket(
|
|
|
327
370
|
let fixUsed = resume?.fixUsed ?? false;
|
|
328
371
|
// The pass behind a fix Stage logs beside the first one rather than over it.
|
|
329
372
|
let logDir = stageLogDir(repoRoot, runId, ticket);
|
|
373
|
+
// The unmet criteria the fix Stage was sent in for, and their evidence, which
|
|
374
|
+
// the verify behind it is held to. Only this Run's: the State file keeps no
|
|
375
|
+
// Verdict, so a retry a later Run resumes is graded without them.
|
|
376
|
+
let unmetBefore: string | undefined;
|
|
330
377
|
|
|
331
378
|
// Declared out here because ticking the Acceptance Criteria it proved happens
|
|
332
379
|
// after the merge, where a failure may no longer hand the Ticket off.
|
|
@@ -388,18 +435,28 @@ async function takeTicket(
|
|
|
388
435
|
let commits: string[];
|
|
389
436
|
let coAuthors: string[];
|
|
390
437
|
let title: string;
|
|
438
|
+
// Owed once, by a Ticket released over a failing Base branch, and inside
|
|
439
|
+
// the passes rather than before them: a conflict it meets spends the Fix
|
|
440
|
+
// budget as one at the Landing does.
|
|
441
|
+
let rebaseOwed = resume?.rebaseFirst === true;
|
|
391
442
|
|
|
392
443
|
// Checks through CI, run again from the top when the fix budget buys a
|
|
393
444
|
// second pass: a fix earns no shortcut, so every gate grades it afresh.
|
|
394
445
|
for (;;) {
|
|
395
446
|
try {
|
|
447
|
+
if (rebaseOwed) {
|
|
448
|
+
rebaseOwed = false;
|
|
449
|
+
point = "rebase";
|
|
450
|
+
await rebaseFirst(pipeline, issue, worktree, branch, logDir, landing, progress);
|
|
451
|
+
await record.advance({ rebaseFirst: false });
|
|
452
|
+
}
|
|
396
453
|
point = "checks";
|
|
397
454
|
// Before anything grades the worktree, and so before verify discards
|
|
398
455
|
// what a Stage left uncommitted.
|
|
399
456
|
await requireCommitted(pipeline, worktree, progress);
|
|
400
457
|
await runChecks(pipeline, worktree, progress);
|
|
401
458
|
point = "verify";
|
|
402
|
-
verdict = await verify(pipeline, issue, worktree, logDir, progress, notes);
|
|
459
|
+
verdict = await verify(pipeline, issue, worktree, logDir, progress, notes, unmetBefore);
|
|
403
460
|
|
|
404
461
|
point = "rebase";
|
|
405
462
|
// The Landing, from here until the Base branch has been pulled: the
|
|
@@ -448,12 +505,12 @@ async function takeTicket(
|
|
|
448
505
|
);
|
|
449
506
|
await record.advance({ pullRequest });
|
|
450
507
|
point = "ci";
|
|
451
|
-
await requireGreenCi(pipeline, pullRequest, progress);
|
|
508
|
+
await requireGreenCi(pipeline, worktree, pullRequest, progress, landing);
|
|
452
509
|
break;
|
|
453
510
|
} catch (error) {
|
|
454
|
-
// A rate limit is nobody's defect, so the fix budget
|
|
455
|
-
// for it: the Ticket is released, whole, further up.
|
|
456
|
-
if (error instanceof
|
|
511
|
+
// A rate limit or an API failure is nobody's defect, so the fix budget
|
|
512
|
+
// does not answer for it: the Ticket is released, whole, further up.
|
|
513
|
+
if (error instanceof Releasing) throw error;
|
|
457
514
|
const failure = asTicketFailure(error, point);
|
|
458
515
|
if (fixUsed || failure.kind === undefined) throw failure;
|
|
459
516
|
fixUsed = true;
|
|
@@ -463,6 +520,7 @@ async function takeTicket(
|
|
|
463
520
|
// the queue at the back when it reaches the rebase again.
|
|
464
521
|
landing.leave();
|
|
465
522
|
logDir = retryLogDir(repoRoot, runId, ticket);
|
|
523
|
+
if (failure.kind === "unmet-criteria") unmetBefore = failure.evidence;
|
|
466
524
|
await fix(
|
|
467
525
|
pipeline,
|
|
468
526
|
issue,
|
|
@@ -475,8 +533,8 @@ async function takeTicket(
|
|
|
475
533
|
record,
|
|
476
534
|
);
|
|
477
535
|
// Recorded once the Stage has come back, not when the budget was
|
|
478
|
-
// committed: a fix Stage the rate limit
|
|
479
|
-
// nothing, and the release below says so.
|
|
536
|
+
// committed: a fix Stage the rate limit or an API failure stopped
|
|
537
|
+
// before it ran spends nothing, and the release below says so.
|
|
480
538
|
await record.advance({ fixUsed: true });
|
|
481
539
|
}
|
|
482
540
|
}
|
|
@@ -492,15 +550,15 @@ async function takeTicket(
|
|
|
492
550
|
// pull request and a Release's labels are nothing for the next Ticket to
|
|
493
551
|
// wait behind, and a Ticket that never reached the rebase gives up nothing.
|
|
494
552
|
landing.leave();
|
|
495
|
-
if (error instanceof
|
|
553
|
+
if (error instanceof Releasing) {
|
|
496
554
|
return release(pipeline, {
|
|
497
555
|
issue,
|
|
498
556
|
user,
|
|
499
557
|
branch,
|
|
500
|
-
|
|
558
|
+
stopped: error,
|
|
501
559
|
record,
|
|
502
|
-
// A fix Stage
|
|
503
|
-
//
|
|
560
|
+
// A fix Stage stopped before it ran spends nothing: the budget is
|
|
561
|
+
// still there for the Run that resumes the Ticket.
|
|
504
562
|
fixUsed: fixUsed && error.point !== "fix",
|
|
505
563
|
notes,
|
|
506
564
|
});
|
|
@@ -703,7 +761,9 @@ function howItWasTaken(resume: TicketState | undefined, stranded: boolean): stri
|
|
|
703
761
|
type Reached = Omit<TicketState, "runId" | "updatedAt">;
|
|
704
762
|
|
|
705
763
|
/** What one write moves that on by. */
|
|
706
|
-
type Advance = Partial<
|
|
764
|
+
type Advance = Partial<
|
|
765
|
+
Pick<Reached, "state" | "fixUsed" | "pullRequest" | "title" | "rebaseFirst">
|
|
766
|
+
>;
|
|
707
767
|
|
|
708
768
|
/**
|
|
709
769
|
* The State file a Ticket keeps, and the one place a Ticket writes it.
|
|
@@ -776,8 +836,12 @@ class ResumeRecord {
|
|
|
776
836
|
* read one can still name what wrote it (ADR-0007).
|
|
777
837
|
*/
|
|
778
838
|
private file(): TicketState {
|
|
839
|
+
// Written only while it is owed, so a State file says nothing of a rebase
|
|
840
|
+
// nobody has to do.
|
|
841
|
+
const { rebaseFirst, ...reached } = this.reached;
|
|
779
842
|
return {
|
|
780
|
-
...
|
|
843
|
+
...reached,
|
|
844
|
+
...(rebaseFirst === true ? { rebaseFirst } : {}),
|
|
781
845
|
runId: this.pipeline.runId,
|
|
782
846
|
version: this.pipeline.version,
|
|
783
847
|
updatedAt: new Date().toISOString(),
|
|
@@ -850,7 +914,8 @@ function runStage(
|
|
|
850
914
|
}
|
|
851
915
|
|
|
852
916
|
/**
|
|
853
|
-
* Report a Stage the rate limit stopped, and raise the
|
|
917
|
+
* Report a Stage the rate limit or an API failure stopped, and raise the
|
|
918
|
+
* Ticket's release.
|
|
854
919
|
*
|
|
855
920
|
* The one place the pause is composed, because the conflict Stage reaches it by
|
|
856
921
|
* another road: the row reads as a pause rather than a cross, and the error
|
|
@@ -863,26 +928,34 @@ async function releasedStage(
|
|
|
863
928
|
stage: StageName,
|
|
864
929
|
point: FailurePoint,
|
|
865
930
|
result: StageResult,
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
931
|
+
cause: StageReleaseCause,
|
|
932
|
+
): Promise<Releasing> {
|
|
933
|
+
await progress.record(stageRow(stage, result, `⏸ ${STAGE_FAILURES[cause].cell}`));
|
|
934
|
+
return new Releasing(
|
|
935
|
+
point,
|
|
936
|
+
stage === "implement" ? "claimed" : "implemented",
|
|
937
|
+
cause,
|
|
938
|
+
result.limitResetsAt,
|
|
939
|
+
);
|
|
869
940
|
}
|
|
870
941
|
|
|
871
942
|
/**
|
|
872
943
|
* Report a Stage that did not come back, and say what it costs the Ticket.
|
|
873
944
|
*
|
|
874
|
-
* The rate limit
|
|
875
|
-
*
|
|
876
|
-
* own, and its caller decides whether the fix
|
|
945
|
+
* The rate limit and a transient API failure are the Stage failures the Ticket
|
|
946
|
+
* did nothing to earn, so they release the Ticket instead of ending it. Every
|
|
947
|
+
* other failure is the Stage's own, and its caller decides whether the fix
|
|
948
|
+
* budget can buy another.
|
|
877
949
|
*/
|
|
878
950
|
async function stageDidNotFinish(
|
|
879
951
|
pipeline: Pipeline,
|
|
880
952
|
progress: Progress,
|
|
881
953
|
stage: "implement" | "verify" | "fix",
|
|
882
954
|
result: StageResult,
|
|
883
|
-
): Promise<TicketFailure |
|
|
884
|
-
|
|
885
|
-
|
|
955
|
+
): Promise<TicketFailure | Releasing> {
|
|
956
|
+
const cause = releaseCause(result);
|
|
957
|
+
if (cause !== undefined) {
|
|
958
|
+
return await releasedStage(progress, stage, stage, result, cause);
|
|
886
959
|
}
|
|
887
960
|
|
|
888
961
|
await progress.record(stageRow(stage, result, `❌ ${stageFailureCell(result)}`));
|
|
@@ -1001,6 +1074,7 @@ async function implement(
|
|
|
1001
1074
|
pipeline.baseBranch,
|
|
1002
1075
|
stage.extraPrompt,
|
|
1003
1076
|
pipeline.selfHosted,
|
|
1077
|
+
pipeline.config.checks,
|
|
1004
1078
|
await standingNotesNumber(pipeline),
|
|
1005
1079
|
),
|
|
1006
1080
|
cwd: worktree,
|
|
@@ -1015,15 +1089,15 @@ async function implement(
|
|
|
1015
1089
|
await recordTitle(result, record);
|
|
1016
1090
|
|
|
1017
1091
|
// Pushed whatever became of the Stage, like its Notes: a session that
|
|
1018
|
-
// committed and then
|
|
1019
|
-
// that resumes the Ticket may be another one.
|
|
1092
|
+
// committed and then was stopped by the rate limit or an API failure still
|
|
1093
|
+
// committed, and the Host that resumes the Ticket may be another one.
|
|
1020
1094
|
const commits = await commitCount(pipeline, branch);
|
|
1021
1095
|
if (commits > 0) await pushCommitted(pipeline, issue.number, worktree, branch);
|
|
1022
1096
|
if (!result.ok) throw await stageDidNotFinish(pipeline, progress, "implement", result);
|
|
1023
1097
|
|
|
1024
1098
|
// An agent that gave up silently leaves a clean branch behind. That is a
|
|
1025
1099
|
// failure, not something to verify. A resumed Ticket's branch is not clean —
|
|
1026
|
-
// it carries what the Stage
|
|
1100
|
+
// it carries what the Stage a Release stopped had committed — so this
|
|
1027
1101
|
// asks the same question there: is there anything at all to grade. Whether it
|
|
1028
1102
|
// is enough is the Verdict's business, not this guard's.
|
|
1029
1103
|
if (commits === 0) {
|
|
@@ -1037,13 +1111,53 @@ async function implement(
|
|
|
1037
1111
|
await progress.record(stageRow("implement", result, "✅ committed"));
|
|
1038
1112
|
}
|
|
1039
1113
|
|
|
1114
|
+
/**
|
|
1115
|
+
* Bring a Ticket released over a failing Base branch up to the Base branch
|
|
1116
|
+
* before its Checks grade it, so a fix a human has landed there meanwhile
|
|
1117
|
+
* reaches the branch. Without it the Ticket would meet the same failure with
|
|
1118
|
+
* its budget unspent, and spend it.
|
|
1119
|
+
*
|
|
1120
|
+
* The pull and the rebase go through the Landing, which is where the Base
|
|
1121
|
+
* branch is brought up to the remote's and rebased onto (ADR-0005), and out of
|
|
1122
|
+
* it again at once: nothing is merged behind them. A conflict goes to the
|
|
1123
|
+
* Conflict Stage as one at the Landing does, outside it, since the rebase
|
|
1124
|
+
* there runs again before the pull request anyway. Left unresolved, the
|
|
1125
|
+
* branch would keep the merge-base that fails and be released again by every
|
|
1126
|
+
* Run that took it up.
|
|
1127
|
+
*/
|
|
1128
|
+
async function rebaseFirst(
|
|
1129
|
+
pipeline: Pipeline,
|
|
1130
|
+
issue: Issue,
|
|
1131
|
+
worktree: string,
|
|
1132
|
+
branch: string,
|
|
1133
|
+
logDir: string,
|
|
1134
|
+
landing: LandingTurn,
|
|
1135
|
+
progress: Progress,
|
|
1136
|
+
): Promise<void> {
|
|
1137
|
+
const { workspace, baseBranch } = pipeline;
|
|
1138
|
+
await landing.enter();
|
|
1139
|
+
let rebase: RebaseOutcome;
|
|
1140
|
+
try {
|
|
1141
|
+
await workspace.pullBase(baseBranch);
|
|
1142
|
+
rebase = await workspace.rebase(worktree, baseBranch);
|
|
1143
|
+
} finally {
|
|
1144
|
+
landing.leave();
|
|
1145
|
+
}
|
|
1146
|
+
if (!rebase.ok) {
|
|
1147
|
+
await resolveConflict(pipeline, issue, worktree, branch, logDir, rebase.conflict, progress);
|
|
1148
|
+
return;
|
|
1149
|
+
}
|
|
1150
|
+
await pushCommitted(pipeline, issue.number, worktree, branch);
|
|
1151
|
+
pipeline.log?.(`#${issue.number} rebased onto ${baseBranch} before its Checks`);
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1040
1154
|
/**
|
|
1041
1155
|
* Refuse a worktree holding changes no commit carries, naming every path.
|
|
1042
1156
|
*
|
|
1043
1157
|
* Nothing is discarded here: a fix Stage sent in for it decides what belongs
|
|
1044
1158
|
* to the Ticket, and a hand-off leaves the changes where a human can find them.
|
|
1045
1159
|
* The summary names no Stage, because a resumed Ticket cannot say which one
|
|
1046
|
-
* left them: a fix Stage
|
|
1160
|
+
* left them: a fix Stage a Release stopped spent no budget.
|
|
1047
1161
|
*/
|
|
1048
1162
|
async function requireCommitted(
|
|
1049
1163
|
pipeline: Pipeline,
|
|
@@ -1074,13 +1188,23 @@ async function runChecks(
|
|
|
1074
1188
|
for (const command of pipeline.config.checks) {
|
|
1075
1189
|
const result = await pipeline.workspace.runCheck(command, worktree, minutes * 60_000);
|
|
1076
1190
|
if (!result.ok) {
|
|
1191
|
+
const durationMs = Date.now() - startedAt;
|
|
1192
|
+
// Asked before the row is written, because the row is what differs: a
|
|
1193
|
+
// Check the Base branch fails too is nothing the Ticket did.
|
|
1194
|
+
await releaseIfTheBaseFails(pipeline, {
|
|
1195
|
+
point: "checks",
|
|
1196
|
+
worktree,
|
|
1197
|
+
commands: [command],
|
|
1198
|
+
progress,
|
|
1199
|
+
durationMs,
|
|
1200
|
+
});
|
|
1077
1201
|
// A Check that hung and one that failed are the same failure to the fix
|
|
1078
1202
|
// budget and the hand-off; they are told apart only in what they say, so
|
|
1079
1203
|
// that a fix Stage knows whether it is mending a hang or an assertion.
|
|
1080
1204
|
await progress.record({
|
|
1081
1205
|
point: "checks",
|
|
1082
1206
|
outcome: `❌ \`${command}\` ${result.timedOut ? "timed out" : "failed"}`,
|
|
1083
|
-
durationMs
|
|
1207
|
+
durationMs,
|
|
1084
1208
|
});
|
|
1085
1209
|
throw new TicketFailure(
|
|
1086
1210
|
"checks",
|
|
@@ -1097,6 +1221,75 @@ async function runChecks(
|
|
|
1097
1221
|
});
|
|
1098
1222
|
}
|
|
1099
1223
|
|
|
1224
|
+
/**
|
|
1225
|
+
* The first of `commands` the Base branch fails, at the commit the branch at
|
|
1226
|
+
* `worktree` inherited from it, or nothing when it passes them all.
|
|
1227
|
+
*
|
|
1228
|
+
* Nothing, too, when the merge-base could not be graded at all: the Ticket is
|
|
1229
|
+
* then charged for the failure as it always was, which costs a fix Stage at
|
|
1230
|
+
* worst, where releasing on a guess would leave a Ticket's own defect unmended
|
|
1231
|
+
* and stop the Run.
|
|
1232
|
+
*/
|
|
1233
|
+
async function inheritedFailure(
|
|
1234
|
+
pipeline: Pipeline,
|
|
1235
|
+
worktree: string,
|
|
1236
|
+
commands: readonly string[],
|
|
1237
|
+
): Promise<FailingBase | undefined> {
|
|
1238
|
+
try {
|
|
1239
|
+
const commit = await pipeline.workspace.mergeBase(worktree, pipeline.baseBranch);
|
|
1240
|
+
return await pipeline.baseChecks.firstFailing(
|
|
1241
|
+
{
|
|
1242
|
+
workspace: pipeline.workspace,
|
|
1243
|
+
repoRoot: pipeline.repoRoot,
|
|
1244
|
+
timeoutMs: pipeline.config.checkTimeoutMinutes * 60_000,
|
|
1245
|
+
...(pipeline.log === undefined ? {} : { log: pipeline.log }),
|
|
1246
|
+
},
|
|
1247
|
+
commit,
|
|
1248
|
+
commands,
|
|
1249
|
+
);
|
|
1250
|
+
} catch (error) {
|
|
1251
|
+
pipeline.log?.(
|
|
1252
|
+
`could not run the Checks at the merge-base with ${pipeline.baseBranch}: ${(error as Error).message}`,
|
|
1253
|
+
);
|
|
1254
|
+
return undefined;
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
/**
|
|
1259
|
+
* Release the Ticket when the Base branch fails one of `commands` too, with the
|
|
1260
|
+
* row that says so; return when it does not, for the caller to charge the
|
|
1261
|
+
* failure to the Ticket as it always did.
|
|
1262
|
+
*
|
|
1263
|
+
* Only a Ticket that reached its Checks gets here, so the implement Stage's
|
|
1264
|
+
* work is always on its branch.
|
|
1265
|
+
*/
|
|
1266
|
+
async function releaseIfTheBaseFails(
|
|
1267
|
+
pipeline: Pipeline,
|
|
1268
|
+
{
|
|
1269
|
+
point,
|
|
1270
|
+
worktree,
|
|
1271
|
+
commands,
|
|
1272
|
+
progress,
|
|
1273
|
+
durationMs,
|
|
1274
|
+
}: {
|
|
1275
|
+
point: "checks" | "ci";
|
|
1276
|
+
worktree: string;
|
|
1277
|
+
commands: readonly string[];
|
|
1278
|
+
progress: Progress;
|
|
1279
|
+
/** How long the failure took on the branch, which the row reports. */
|
|
1280
|
+
durationMs: number;
|
|
1281
|
+
},
|
|
1282
|
+
): Promise<void> {
|
|
1283
|
+
const inherited = await inheritedFailure(pipeline, worktree, commands);
|
|
1284
|
+
if (inherited === undefined) return;
|
|
1285
|
+
await progress.record({
|
|
1286
|
+
point,
|
|
1287
|
+
outcome: `⏸ \`${inherited.command}\` fails on the Base branch`,
|
|
1288
|
+
durationMs,
|
|
1289
|
+
});
|
|
1290
|
+
throw new Releasing(point, "implemented", "failing-base", undefined, inherited);
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1100
1293
|
/**
|
|
1101
1294
|
* What the Check printed before the kill, and then a line saying that a kill is
|
|
1102
1295
|
* what ended it. A fix Stage is given the evidence and little else, so the
|
|
@@ -1116,6 +1309,8 @@ async function verify(
|
|
|
1116
1309
|
logDir: string,
|
|
1117
1310
|
progress: Progress,
|
|
1118
1311
|
notes: RoutedNote[],
|
|
1312
|
+
/** The unmet criteria and evidence of the Verdict a fix Stage was sent in for. */
|
|
1313
|
+
unmetBefore: string | undefined,
|
|
1119
1314
|
): Promise<Verdict> {
|
|
1120
1315
|
const stage = pipeline.config.stages.verify;
|
|
1121
1316
|
const result = await runStage(pipeline, "verify", {
|
|
@@ -1123,7 +1318,9 @@ async function verify(
|
|
|
1123
1318
|
issue.url,
|
|
1124
1319
|
stage.extraPrompt,
|
|
1125
1320
|
pipeline.selfHosted,
|
|
1321
|
+
pipeline.config.checks,
|
|
1126
1322
|
await standingNotesNumber(pipeline),
|
|
1323
|
+
unmetBefore,
|
|
1127
1324
|
),
|
|
1128
1325
|
cwd: worktree,
|
|
1129
1326
|
logDir,
|
|
@@ -1215,6 +1412,7 @@ async function fix(
|
|
|
1215
1412
|
pipeline.baseBranch,
|
|
1216
1413
|
stage.extraPrompt,
|
|
1217
1414
|
pipeline.selfHosted,
|
|
1415
|
+
pipeline.config.checks,
|
|
1218
1416
|
await standingNotesNumber(pipeline),
|
|
1219
1417
|
),
|
|
1220
1418
|
cwd: worktree,
|
|
@@ -1239,9 +1437,17 @@ async function fix(
|
|
|
1239
1437
|
// A branch that grew is the whole signal, which is not the same as one that
|
|
1240
1438
|
// changed: a session that squashed the branch shorter, or amended in place,
|
|
1241
1439
|
// is read here as having committed nothing.
|
|
1440
|
+
//
|
|
1441
|
+
// The hand-off carries the evidence the Stage was sent in with. Committing
|
|
1442
|
+
// nothing is also how a fix Stage hands back an unmet criterion only a human
|
|
1443
|
+
// can settle, and the human then reads the question where it was asked.
|
|
1242
1444
|
if (!grew) {
|
|
1243
1445
|
await progress.record(stageRow("fix", result, "❌ no commits"));
|
|
1244
|
-
throw new TicketFailure(
|
|
1446
|
+
throw new TicketFailure(
|
|
1447
|
+
"fix",
|
|
1448
|
+
"the fix Stage left no new commits on the branch",
|
|
1449
|
+
failure.evidence,
|
|
1450
|
+
);
|
|
1245
1451
|
}
|
|
1246
1452
|
|
|
1247
1453
|
await progress.record(stageRow("fix", result, "✅ committed"));
|
|
@@ -1260,9 +1466,10 @@ async function fix(
|
|
|
1260
1466
|
* came back clean because it quietly abandoned the rebase has not, and only the
|
|
1261
1467
|
* worktree can tell the difference.
|
|
1262
1468
|
*
|
|
1263
|
-
* A Stage the rate limit stopped is read the same way, and
|
|
1264
|
-
* one that had already finished the rebase leaves nothing
|
|
1265
|
-
* session, so the Ticket carries on to the Checks, the pull
|
|
1469
|
+
* A Stage the rate limit or an API failure stopped is read the same way, and
|
|
1470
|
+
* only then released: one that had already finished the rebase leaves nothing
|
|
1471
|
+
* that needs an agent session, so the Ticket carries on to the Checks, the pull
|
|
1472
|
+
* request and CI.
|
|
1266
1473
|
*/
|
|
1267
1474
|
async function resolveConflict(
|
|
1268
1475
|
pipeline: Pipeline,
|
|
@@ -1309,7 +1516,7 @@ async function conflictStage(
|
|
|
1309
1516
|
logDir: string,
|
|
1310
1517
|
conflict: string,
|
|
1311
1518
|
progress: Progress,
|
|
1312
|
-
): Promise<TicketFailure |
|
|
1519
|
+
): Promise<TicketFailure | Releasing | undefined> {
|
|
1313
1520
|
const stage = pipeline.config.stages.conflict;
|
|
1314
1521
|
try {
|
|
1315
1522
|
const result = await runStage(pipeline, "conflict", {
|
|
@@ -1319,6 +1526,7 @@ async function conflictStage(
|
|
|
1319
1526
|
pipeline.baseBranch,
|
|
1320
1527
|
stage.extraPrompt,
|
|
1321
1528
|
pipeline.selfHosted,
|
|
1529
|
+
pipeline.config.checks,
|
|
1322
1530
|
),
|
|
1323
1531
|
cwd: worktree,
|
|
1324
1532
|
logDir,
|
|
@@ -1327,8 +1535,8 @@ async function conflictStage(
|
|
|
1327
1535
|
const state = await pipeline.workspace.rebaseState(worktree, pipeline.baseBranch);
|
|
1328
1536
|
if (state.resolved) {
|
|
1329
1537
|
// The worktree decides the row, as it decides the outcome: a Stage that
|
|
1330
|
-
// ran out of turns, or into the rate limit
|
|
1331
|
-
// did the job it was sent to do.
|
|
1538
|
+
// ran out of turns, or into the rate limit or an API failure, after
|
|
1539
|
+
// finishing the rebase did the job it was sent to do.
|
|
1332
1540
|
await progress.record(stageRow("conflict", result, "✅ rebased"));
|
|
1333
1541
|
// The rebase rewrote the branch, and the resolution is a commit of the
|
|
1334
1542
|
// Stage's like any other.
|
|
@@ -1336,11 +1544,13 @@ async function conflictStage(
|
|
|
1336
1544
|
return undefined;
|
|
1337
1545
|
}
|
|
1338
1546
|
|
|
1339
|
-
// A rate limit left the conflict exactly as it found it:
|
|
1340
|
-
// Stage to mend and nothing to blame the Ticket for. The
|
|
1341
|
-
// the worktree its abort, so the branch the resumed Run
|
|
1342
|
-
|
|
1343
|
-
|
|
1547
|
+
// A rate limit or an API failure left the conflict exactly as it found it:
|
|
1548
|
+
// nothing for a fix Stage to mend and nothing to blame the Ticket for. The
|
|
1549
|
+
// caller still owes the worktree its abort, so the branch the resumed Run
|
|
1550
|
+
// rebases is clean.
|
|
1551
|
+
const cause = releaseCause(result);
|
|
1552
|
+
if (cause !== undefined) {
|
|
1553
|
+
return await releasedStage(progress, "conflict", "rebase", result, cause);
|
|
1344
1554
|
}
|
|
1345
1555
|
|
|
1346
1556
|
await progress.record(stageRow("conflict", result, "❌ unresolved"));
|
|
@@ -1422,8 +1632,10 @@ async function publishPullRequest(
|
|
|
1422
1632
|
*/
|
|
1423
1633
|
async function requireGreenCi(
|
|
1424
1634
|
pipeline: Pipeline,
|
|
1635
|
+
worktree: string,
|
|
1425
1636
|
pullRequest: number,
|
|
1426
1637
|
progress: Progress,
|
|
1638
|
+
landing: LandingTurn,
|
|
1427
1639
|
): Promise<void> {
|
|
1428
1640
|
const startedAt = Date.now();
|
|
1429
1641
|
const outcome = await pipeline.tracker.waitForCi(
|
|
@@ -1437,11 +1649,27 @@ async function requireGreenCi(
|
|
|
1437
1649
|
case "passed":
|
|
1438
1650
|
await record("✅ passed");
|
|
1439
1651
|
return;
|
|
1440
|
-
case "failed":
|
|
1652
|
+
case "failed": {
|
|
1653
|
+
// Out of the Landing first: whether the Ticket is released or sent to a
|
|
1654
|
+
// fix Stage, it leaves, and the Checks below take minutes nobody else
|
|
1655
|
+
// should wait behind.
|
|
1656
|
+
landing.leave();
|
|
1657
|
+
// CI's own runs on the Base branch cannot say whether it fails there: a
|
|
1658
|
+
// workflow that runs only on pull requests leaves a squash commit with
|
|
1659
|
+
// none, and a failure the clock brings starts after its last green run.
|
|
1660
|
+
// So the merge-base is graded with the Checks, which CI runs too.
|
|
1661
|
+
await releaseIfTheBaseFails(pipeline, {
|
|
1662
|
+
point: "ci",
|
|
1663
|
+
worktree,
|
|
1664
|
+
commands: pipeline.config.checks,
|
|
1665
|
+
progress,
|
|
1666
|
+
durationMs: Date.now() - startedAt,
|
|
1667
|
+
});
|
|
1441
1668
|
await record("❌ failed");
|
|
1442
1669
|
// The names are the line a human reads; the log excerpt is what a fix
|
|
1443
1670
|
// Stage works from, and it is empty whenever none could be fetched.
|
|
1444
1671
|
throw new TicketFailure("ci", outcome.summary, outcome.excerpt, "failed-ci");
|
|
1672
|
+
}
|
|
1445
1673
|
case "timed-out":
|
|
1446
1674
|
await record("❌ timed out");
|
|
1447
1675
|
throw new TicketFailure(
|
|
@@ -1473,7 +1701,7 @@ interface Release {
|
|
|
1473
1701
|
issue: Issue;
|
|
1474
1702
|
user: string;
|
|
1475
1703
|
branch: string;
|
|
1476
|
-
|
|
1704
|
+
stopped: Releasing;
|
|
1477
1705
|
/** The Ticket's State file, which the release brings up to date and leaves. */
|
|
1478
1706
|
record: ResumeRecord;
|
|
1479
1707
|
/** Whether the Fix budget was spent by a failure of the Ticket's own. */
|
|
@@ -1492,23 +1720,106 @@ interface Release {
|
|
|
1492
1720
|
*/
|
|
1493
1721
|
async function release(
|
|
1494
1722
|
pipeline: Pipeline,
|
|
1495
|
-
{ issue, user, branch,
|
|
1723
|
+
{ issue, user, branch, stopped, record, fixUsed, notes }: Release,
|
|
1496
1724
|
): Promise<TicketOutcome> {
|
|
1497
1725
|
const { tracker, config } = pipeline;
|
|
1498
1726
|
const ticket = issue.number;
|
|
1499
1727
|
|
|
1500
1728
|
// Brought up to date before the Claim comes off: a Ticket back on the Frontier
|
|
1501
|
-
// whose state is out of date is one the next Run would redo Stages for.
|
|
1502
|
-
|
|
1729
|
+
// whose state is out of date is one the next Run would redo Stages for. One
|
|
1730
|
+
// released over a failing Base branch is rebased first when it comes back,
|
|
1731
|
+
// or it would meet the same failure.
|
|
1732
|
+
await record.advance({
|
|
1733
|
+
state: stopped.state,
|
|
1734
|
+
fixUsed,
|
|
1735
|
+
...(stopped.failingBase === undefined ? {} : { rebaseFirst: true }),
|
|
1736
|
+
});
|
|
1737
|
+
if (stopped.failingBase !== undefined) {
|
|
1738
|
+
await noteFailingBase(pipeline, { issue, branch, stopped, failing: stopped.failingBase, notes });
|
|
1739
|
+
}
|
|
1503
1740
|
|
|
1504
1741
|
// The claim, undone in the order it was made, so the assignee — which is what
|
|
1505
1742
|
// another Run reads to tell a taken Ticket from a free one — comes off last.
|
|
1506
1743
|
await tracker.addLabel(ticket, config.labels.readyForAgent);
|
|
1507
1744
|
await tracker.removeLabel(ticket, config.labels.inProgress);
|
|
1508
1745
|
await tracker.unassign(ticket, user);
|
|
1509
|
-
pipeline.log?.(
|
|
1746
|
+
pipeline.log?.(
|
|
1747
|
+
`#${ticket} released at ${stopped.point} · ${RELEASE_WORDS[stopped.reason]} · ${branch}`,
|
|
1748
|
+
);
|
|
1749
|
+
|
|
1750
|
+
return {
|
|
1751
|
+
outcome: "released",
|
|
1752
|
+
ticket,
|
|
1753
|
+
title: issue.title,
|
|
1754
|
+
branch,
|
|
1755
|
+
stage: stopped.point,
|
|
1756
|
+
cause: stopped.reason,
|
|
1757
|
+
...(stopped.limitResetsAt === undefined ? {} : { limitResetsAt: stopped.limitResetsAt }),
|
|
1758
|
+
...(stopped.failingBase === undefined
|
|
1759
|
+
? {}
|
|
1760
|
+
: {
|
|
1761
|
+
failingBase: { commit: stopped.failingBase.commit, command: stopped.failingBase.command },
|
|
1762
|
+
}),
|
|
1763
|
+
notes,
|
|
1764
|
+
};
|
|
1765
|
+
}
|
|
1510
1766
|
|
|
1511
|
-
|
|
1767
|
+
/**
|
|
1768
|
+
* Tell triage the Base branch fails a Check, once this Run per merge-base
|
|
1769
|
+
* commit however many Tickets it releases: the pipeline's own Note, on the
|
|
1770
|
+
* standing Notes issue, since the defect is the Base branch's and no Ticket's.
|
|
1771
|
+
*
|
|
1772
|
+
* Logged and nothing more when it cannot be posted, as a Stage's Note is: the
|
|
1773
|
+
* Release is what the Ticket is owed, and the next Ticket released over the
|
|
1774
|
+
* same commit tries the Note again.
|
|
1775
|
+
*/
|
|
1776
|
+
async function noteFailingBase(
|
|
1777
|
+
pipeline: Pipeline,
|
|
1778
|
+
{
|
|
1779
|
+
issue,
|
|
1780
|
+
branch,
|
|
1781
|
+
stopped,
|
|
1782
|
+
failing,
|
|
1783
|
+
notes,
|
|
1784
|
+
}: {
|
|
1785
|
+
issue: Issue;
|
|
1786
|
+
branch: string;
|
|
1787
|
+
stopped: Releasing;
|
|
1788
|
+
failing: FailingBase;
|
|
1789
|
+
notes: RoutedNote[];
|
|
1790
|
+
},
|
|
1791
|
+
): Promise<void> {
|
|
1792
|
+
const origin = issue.number;
|
|
1793
|
+
try {
|
|
1794
|
+
const note = await pipeline.baseChecks.noteOnce(failing.commit, async () => {
|
|
1795
|
+
const { number, opened } = await pipeline.standingNotes.resolve(
|
|
1796
|
+
standingNotesLookup(pipeline),
|
|
1797
|
+
);
|
|
1798
|
+
await pipeline.tracker.comment(
|
|
1799
|
+
number,
|
|
1800
|
+
failingBaseNote({
|
|
1801
|
+
origin,
|
|
1802
|
+
point: stopped.point === "ci" ? "ci" : "checks",
|
|
1803
|
+
baseBranch: pipeline.baseBranch,
|
|
1804
|
+
branch,
|
|
1805
|
+
...failing,
|
|
1806
|
+
}),
|
|
1807
|
+
);
|
|
1808
|
+
pipeline.log?.(`#${origin} noted on #${number} that ${pipeline.baseBranch} fails \`${failing.command}\``);
|
|
1809
|
+
return {
|
|
1810
|
+
origin,
|
|
1811
|
+
stage: "pipeline",
|
|
1812
|
+
issue: number,
|
|
1813
|
+
opened,
|
|
1814
|
+
summary: failingBaseSummary(pipeline.baseBranch, failing.command),
|
|
1815
|
+
};
|
|
1816
|
+
});
|
|
1817
|
+
if (note !== undefined) notes.push(note);
|
|
1818
|
+
} catch (error) {
|
|
1819
|
+
pipeline.log?.(
|
|
1820
|
+
`#${origin} could not note that ${pipeline.baseBranch} fails a Check: ${(error as Error).message}`,
|
|
1821
|
+
);
|
|
1822
|
+
}
|
|
1512
1823
|
}
|
|
1513
1824
|
|
|
1514
1825
|
/** The worktree a hand-off has to hand, which is not always one of this Run's. */
|
|
@@ -1623,8 +1934,11 @@ async function handOff(
|
|
|
1623
1934
|
const pr = await tracker.createPullRequest({
|
|
1624
1935
|
base: pipeline.baseBranch,
|
|
1625
1936
|
head: branch,
|
|
1626
|
-
//
|
|
1627
|
-
|
|
1937
|
+
// A human can still merge the draft, and its squash commit takes this
|
|
1938
|
+
// title, so it is the one a Stage answered for the branch where there is
|
|
1939
|
+
// one. Without it, the Ticket title rather than a commit subject: the
|
|
1940
|
+
// pipeline merges nothing here, so it derives nothing for a merge.
|
|
1941
|
+
title: record.title ?? issue.title,
|
|
1628
1942
|
body: draftPullRequestBody(draftBody),
|
|
1629
1943
|
draft: true,
|
|
1630
1944
|
});
|
|
@@ -1780,6 +2094,11 @@ const STAGE_FAILURES: Record<
|
|
|
1780
2094
|
// entry per failure and the next path to need it must not have to invent it.
|
|
1781
2095
|
sentence: () => "hit the subscription rate limit",
|
|
1782
2096
|
},
|
|
2097
|
+
// Released like the rate limit, so its sentence is kept for the same reason.
|
|
2098
|
+
"api-error": {
|
|
2099
|
+
cell: "API failure",
|
|
2100
|
+
sentence: () => "was stopped by an API failure the CLI had given up retrying",
|
|
2101
|
+
},
|
|
1783
2102
|
"timed-out": {
|
|
1784
2103
|
cell: "timed out",
|
|
1785
2104
|
sentence: (limits) => `ran past its ${limits.maxMinutes} minute limit`,
|