patchrelay 0.35.11 → 0.35.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -9
- package/dist/build-info.json +3 -3
- package/dist/cli/args.js +19 -1
- package/dist/cli/commands/issues.js +18 -56
- package/dist/cli/commands/watch.js +5 -0
- package/dist/cli/data.js +160 -47
- package/dist/cli/formatters/text.js +51 -90
- package/dist/cli/help.js +15 -8
- package/dist/cli/index.js +3 -58
- package/dist/cli/operator-client.js +0 -82
- package/dist/cli/watch/App.js +21 -12
- package/dist/cli/watch/HelpBar.js +3 -3
- package/dist/cli/watch/IssueDetailView.js +63 -130
- package/dist/cli/watch/IssueRow.js +82 -27
- package/dist/cli/watch/StatusBar.js +8 -4
- package/dist/cli/watch/detail-rows.js +589 -0
- package/dist/cli/watch/render-rich-text.js +226 -0
- package/dist/cli/watch/state-visualization.js +48 -23
- package/dist/cli/watch/timeline-builder.js +2 -1
- package/dist/cli/watch/use-detail-stream.js +10 -104
- package/dist/cli/watch/use-watch-stream.js +11 -102
- package/dist/cli/watch/watch-state.js +129 -56
- package/dist/codex-thread-utils.js +3 -0
- package/dist/db/migrations.js +239 -2
- package/dist/db.js +628 -39
- package/dist/github-app-token.js +7 -0
- package/dist/github-failure-context.js +44 -1
- package/dist/github-rollup.js +47 -0
- package/dist/github-webhook-handler.js +423 -52
- package/dist/github-webhooks.js +7 -0
- package/dist/http.js +12 -264
- package/dist/idle-reconciliation.js +268 -76
- package/dist/issue-query-service.js +221 -129
- package/dist/issue-session-events.js +151 -0
- package/dist/issue-session.js +99 -0
- package/dist/linear-client.js +39 -25
- package/dist/linear-session-reporting.js +12 -0
- package/dist/linear-session-sync.js +253 -24
- package/dist/linear-workflow.js +33 -0
- package/dist/merge-queue-protocol.js +0 -51
- package/dist/preflight.js +1 -4
- package/dist/queue-health-monitor.js +11 -7
- package/dist/run-orchestrator.js +1364 -147
- package/dist/run-reporting.js +5 -3
- package/dist/service.js +279 -102
- package/dist/status-note.js +56 -0
- package/dist/waiting-reason.js +65 -0
- package/dist/webhook-handler.js +270 -79
- package/package.json +3 -2
- package/dist/cli/commands/feed.js +0 -60
- package/dist/cli/watch/FeedView.js +0 -28
- package/dist/cli/watch/use-feed-stream.js +0 -92
package/dist/run-orchestrator.js
CHANGED
|
@@ -1,18 +1,23 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { ACTIVE_RUN_STATES, TERMINAL_STATES } from "./factory-state.js";
|
|
4
5
|
import { buildHookEnv, runProjectHook } from "./hook-runner.js";
|
|
5
6
|
import { buildStageReport, countEventMethods, extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread, } from "./run-reporting.js";
|
|
6
7
|
import { buildRunCompletedActivity, buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
7
|
-
import { requestMergeQueueAdmission, resolveMergeQueueProtocol, } from "./merge-queue-protocol.js";
|
|
8
8
|
import { WorktreeManager } from "./worktree-manager.js";
|
|
9
|
-
import { resolveAuthoritativeLinearStopState } from "./linear-workflow.js";
|
|
9
|
+
import { resolveAuthoritativeLinearStopState, resolvePreferredCompletedLinearState } from "./linear-workflow.js";
|
|
10
10
|
import { execCommand } from "./utils.js";
|
|
11
|
+
import { getThreadTurns } from "./codex-thread-utils.js";
|
|
12
|
+
import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
11
13
|
const DEFAULT_CI_REPAIR_BUDGET = 3;
|
|
12
14
|
const DEFAULT_QUEUE_REPAIR_BUDGET = 3;
|
|
13
15
|
const DEFAULT_REVIEW_FIX_BUDGET = 3;
|
|
14
16
|
const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
|
|
15
17
|
const ZOMBIE_RECOVERY_BASE_DELAY_MS = 15_000; // 15s, 30s, 60s, 120s, 240s
|
|
18
|
+
const ISSUE_SESSION_LEASE_MS = 10 * 60_000;
|
|
19
|
+
const MAX_THREAD_GENERATION_BEFORE_COMPACTION = 4;
|
|
20
|
+
const MAX_FOLLOW_UPS_BEFORE_COMPACTION = 4;
|
|
16
21
|
import { QueueHealthMonitor } from "./queue-health-monitor.js";
|
|
17
22
|
import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
|
|
18
23
|
import { LinearSessionSync } from "./linear-session-sync.js";
|
|
@@ -38,22 +43,340 @@ function readWorkflowFile(repoPath, runType) {
|
|
|
38
43
|
return undefined;
|
|
39
44
|
return readFileSync(filePath, "utf8").trim();
|
|
40
45
|
}
|
|
41
|
-
function
|
|
42
|
-
const
|
|
46
|
+
function collectImplementationInstructionText(issue, context, promptText) {
|
|
47
|
+
const parts = [];
|
|
48
|
+
if (issue.title)
|
|
49
|
+
parts.push(issue.title);
|
|
50
|
+
if (issue.description)
|
|
51
|
+
parts.push(issue.description);
|
|
52
|
+
if (promptText)
|
|
53
|
+
parts.push(promptText);
|
|
54
|
+
const stringFields = ["promptContext", "promptBody", "operatorPrompt", "userComment"];
|
|
55
|
+
for (const field of stringFields) {
|
|
56
|
+
const value = context?.[field];
|
|
57
|
+
if (typeof value === "string" && value.trim()) {
|
|
58
|
+
parts.push(value);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (Array.isArray(context?.followUps)) {
|
|
62
|
+
for (const entry of context.followUps) {
|
|
63
|
+
if (!entry || typeof entry !== "object")
|
|
64
|
+
continue;
|
|
65
|
+
const text = entry.text;
|
|
66
|
+
if (typeof text === "string" && text.trim()) {
|
|
67
|
+
parts.push(text);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return parts.join("\n").toLowerCase();
|
|
72
|
+
}
|
|
73
|
+
export function resolveImplementationDeliveryMode(issue, context, promptText) {
|
|
74
|
+
const instructionText = collectImplementationInstructionText(issue, context, promptText);
|
|
75
|
+
if (!instructionText)
|
|
76
|
+
return "publish_pr";
|
|
77
|
+
const hasExplicitNoPr = [
|
|
78
|
+
/\bdo not open (?:a |any )?pr\b/,
|
|
79
|
+
/\bdo not open (?:a |any )?pull request\b/,
|
|
80
|
+
/\bno pr is opened\b/,
|
|
81
|
+
/\bpatchrelay should not open a pr\b/,
|
|
82
|
+
/\bwithout opening a pr\b/,
|
|
83
|
+
].some((pattern) => pattern.test(instructionText));
|
|
84
|
+
const forbidsRepoChanges = [
|
|
85
|
+
/\bdo not make repository changes\b/,
|
|
86
|
+
/\bdo not make repo changes\b/,
|
|
87
|
+
/\bno repository changes\b/,
|
|
88
|
+
/\bno repo changes\b/,
|
|
89
|
+
/\bdo not modify repo files\b/,
|
|
90
|
+
].some((pattern) => pattern.test(instructionText));
|
|
91
|
+
const planningOnly = [
|
|
92
|
+
/\bplanning\/specification issue only\b/,
|
|
93
|
+
/\bplanning[- ]only\b/,
|
|
94
|
+
/\bspecification[- ]only\b/,
|
|
95
|
+
/\bplanning issue only\b/,
|
|
96
|
+
].some((pattern) => pattern.test(instructionText));
|
|
97
|
+
if (hasExplicitNoPr || (planningOnly && forbidsRepoChanges)) {
|
|
98
|
+
return "linear_only";
|
|
99
|
+
}
|
|
100
|
+
return "publish_pr";
|
|
101
|
+
}
|
|
102
|
+
function appendPublicationContract(lines, runType, issue, context) {
|
|
103
|
+
const deliveryMode = runType === "implementation" && issue
|
|
104
|
+
? resolveImplementationDeliveryMode(issue, context)
|
|
105
|
+
: "publish_pr";
|
|
106
|
+
if (runType === "implementation" && deliveryMode === "linear_only") {
|
|
107
|
+
lines.push("## Delivery Requirements", "");
|
|
108
|
+
lines.push("This issue is planning/specification only.", "Do not modify repo files or open a PR for this issue.", "Deliver the result through Linear artifacts such as follow-up issues, documents, and a concise summary.", "Leave the worktree clean before stopping.", "");
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
lines.push("## Publication Requirements", "");
|
|
112
|
+
if (runType === "implementation") {
|
|
113
|
+
lines.push("Before finishing, publish the result instead of leaving it only in the worktree.", "If the worktree already contains relevant changes for this issue, verify them and publish them.", "If you changed files for this issue, commit them, push the issue branch, and open or update the PR before stopping.", "Do not stop with only local commits or uncommitted changes.", "");
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
lines.push("Before finishing, publish the result to the existing PR branch.", "If you changed files for this repair, commit them and push the same branch before stopping.", "Do not open a new PR.", "Do not stop with only local commits or uncommitted changes.", "");
|
|
117
|
+
}
|
|
118
|
+
function buildPromptHeader(issue) {
|
|
119
|
+
return [
|
|
43
120
|
`Issue: ${issue.issueKey ?? issue.linearIssueId}`,
|
|
44
121
|
issue.title ? `Title: ${issue.title}` : undefined,
|
|
45
122
|
issue.branchName ? `Branch: ${issue.branchName}` : undefined,
|
|
46
123
|
issue.prNumber ? `PR: #${issue.prNumber}` : undefined,
|
|
47
124
|
"",
|
|
48
125
|
].filter(Boolean);
|
|
126
|
+
}
|
|
127
|
+
function appendTaskObjective(lines, issue) {
|
|
128
|
+
const description = issue.description?.trim();
|
|
129
|
+
lines.push("## Task Objective", "");
|
|
130
|
+
lines.push(issue.title || `Complete ${issue.issueKey ?? issue.linearIssueId}.`);
|
|
131
|
+
if (description) {
|
|
132
|
+
lines.push("", description);
|
|
133
|
+
}
|
|
134
|
+
lines.push("");
|
|
135
|
+
}
|
|
136
|
+
function appendLinearContext(lines, context) {
|
|
49
137
|
const promptContext = typeof context?.promptContext === "string" ? context.promptContext.trim() : "";
|
|
50
138
|
const latestPrompt = typeof context?.promptBody === "string" ? context.promptBody.trim() : "";
|
|
139
|
+
const operatorPrompt = typeof context?.operatorPrompt === "string" ? context.operatorPrompt.trim() : "";
|
|
140
|
+
const userComment = typeof context?.userComment === "string" ? context.userComment.trim() : "";
|
|
51
141
|
if (promptContext) {
|
|
52
142
|
lines.push("## Linear Session Context", "", promptContext, "");
|
|
53
143
|
}
|
|
54
144
|
if (latestPrompt) {
|
|
55
145
|
lines.push("## Latest Human Instruction", "", latestPrompt, "");
|
|
56
146
|
}
|
|
147
|
+
if (operatorPrompt) {
|
|
148
|
+
lines.push("## Operator Prompt", "", operatorPrompt, "");
|
|
149
|
+
}
|
|
150
|
+
if (userComment) {
|
|
151
|
+
lines.push("## Human Follow-up Comment", "", userComment, "");
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
function readReviewFixComments(context) {
|
|
155
|
+
const raw = context?.reviewComments;
|
|
156
|
+
if (!Array.isArray(raw)) {
|
|
157
|
+
return [];
|
|
158
|
+
}
|
|
159
|
+
const comments = [];
|
|
160
|
+
for (const entry of raw) {
|
|
161
|
+
if (!entry || typeof entry !== "object")
|
|
162
|
+
continue;
|
|
163
|
+
const record = entry;
|
|
164
|
+
const body = typeof record.body === "string" ? record.body.trim() : "";
|
|
165
|
+
if (!body)
|
|
166
|
+
continue;
|
|
167
|
+
comments.push({
|
|
168
|
+
body,
|
|
169
|
+
...(typeof record.path === "string" ? { path: record.path } : {}),
|
|
170
|
+
...(typeof record.line === "number" ? { line: record.line } : {}),
|
|
171
|
+
...(typeof record.side === "string" ? { side: record.side } : {}),
|
|
172
|
+
...(typeof record.startLine === "number" ? { startLine: record.startLine } : {}),
|
|
173
|
+
...(typeof record.startSide === "string" ? { startSide: record.startSide } : {}),
|
|
174
|
+
...(typeof record.url === "string" ? { url: record.url } : {}),
|
|
175
|
+
...(typeof record.authorLogin === "string" ? { authorLogin: record.authorLogin } : {}),
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
return comments;
|
|
179
|
+
}
|
|
180
|
+
function appendStructuredReviewContext(lines, context) {
|
|
181
|
+
const reviewId = typeof context?.reviewId === "number" ? context.reviewId : undefined;
|
|
182
|
+
const reviewCommitId = typeof context?.reviewCommitId === "string" ? context.reviewCommitId : undefined;
|
|
183
|
+
const reviewUrl = typeof context?.reviewUrl === "string" ? context.reviewUrl : undefined;
|
|
184
|
+
const reviewComments = readReviewFixComments(context);
|
|
185
|
+
if (!reviewId && !reviewCommitId && !reviewUrl && reviewComments.length === 0) {
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
lines.push("## Structured Review Context", "");
|
|
189
|
+
if (reviewId !== undefined) {
|
|
190
|
+
lines.push(`Review ID: ${reviewId}`);
|
|
191
|
+
}
|
|
192
|
+
if (reviewCommitId) {
|
|
193
|
+
lines.push(`Reviewed commit: ${reviewCommitId}`);
|
|
194
|
+
}
|
|
195
|
+
if (reviewUrl) {
|
|
196
|
+
lines.push(`Review URL: ${reviewUrl}`);
|
|
197
|
+
}
|
|
198
|
+
if (reviewComments.length === 0) {
|
|
199
|
+
lines.push("No inline review comments were captured for this review.", "");
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
lines.push(`Inline review comments captured: ${reviewComments.length}`, "Resolve each comment below or verify it is already fixed on the current head before you stop.", "");
|
|
203
|
+
for (const comment of reviewComments) {
|
|
204
|
+
const location = comment.path
|
|
205
|
+
? `${comment.path}${comment.line !== undefined ? `:${comment.line}` : ""}${comment.side ? ` (${comment.side})` : ""}`
|
|
206
|
+
: "general";
|
|
207
|
+
lines.push(`- ${location}`);
|
|
208
|
+
lines.push(comment.body);
|
|
209
|
+
if (comment.url) {
|
|
210
|
+
lines.push(` URL: ${comment.url}`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
lines.push("");
|
|
214
|
+
}
|
|
215
|
+
function collectFollowUpInputs(context) {
|
|
216
|
+
const followUps = Array.isArray(context?.followUps) ? context.followUps : [];
|
|
217
|
+
const inputs = [];
|
|
218
|
+
for (const entry of followUps) {
|
|
219
|
+
const followUp = entry && typeof entry === "object" ? entry : undefined;
|
|
220
|
+
const type = typeof followUp?.type === "string" ? followUp.type : "followup";
|
|
221
|
+
const author = typeof followUp?.author === "string" ? followUp.author : undefined;
|
|
222
|
+
const text = typeof followUp?.text === "string" ? followUp.text.trim() : "";
|
|
223
|
+
if (!text)
|
|
224
|
+
continue;
|
|
225
|
+
inputs.push({ type, text, ...(author ? { author } : {}) });
|
|
226
|
+
}
|
|
227
|
+
return inputs;
|
|
228
|
+
}
|
|
229
|
+
function resolveFollowUpWhy(runType, context) {
|
|
230
|
+
const wakeReason = typeof context?.wakeReason === "string" ? context.wakeReason : undefined;
|
|
231
|
+
switch (wakeReason) {
|
|
232
|
+
case "direct_reply":
|
|
233
|
+
return "A human reply arrived for the outstanding question from the previous turn.";
|
|
234
|
+
case "followup_prompt":
|
|
235
|
+
return "A new Linear agent prompt arrived after the previous turn.";
|
|
236
|
+
case "followup_comment":
|
|
237
|
+
return "A human follow-up comment arrived after the previous turn.";
|
|
238
|
+
case "operator_prompt":
|
|
239
|
+
return "An operator supplied new guidance for this issue.";
|
|
240
|
+
case "review_changes_requested":
|
|
241
|
+
return "GitHub review requested changes on the current PR head.";
|
|
242
|
+
case "settled_red_ci":
|
|
243
|
+
return "Required CI settled red for the current PR head.";
|
|
244
|
+
case "merge_steward_incident":
|
|
245
|
+
return "Merge Steward reported an incident on the current PR head.";
|
|
246
|
+
case "delegated":
|
|
247
|
+
return runType === "implementation"
|
|
248
|
+
? "This is the first implementation turn for the delegated issue."
|
|
249
|
+
: `This turn continues ${runType.replaceAll("_", " ")} work for the delegated issue.`;
|
|
250
|
+
default:
|
|
251
|
+
if (runType === "review_fix")
|
|
252
|
+
return "This turn continues requested-changes work on the existing PR.";
|
|
253
|
+
if (runType === "ci_repair")
|
|
254
|
+
return "This turn continues CI repair work on the existing PR.";
|
|
255
|
+
if (runType === "queue_repair")
|
|
256
|
+
return "This turn continues merge-queue repair work on the existing PR.";
|
|
257
|
+
return "This turn continues implementation on the existing issue session.";
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
function resolveFollowUpAction(runType, context) {
|
|
261
|
+
if (context?.directReplyMode === true) {
|
|
262
|
+
return "Apply the latest human answer, continue from the current branch/session context, and only ask another question if you are still blocked.";
|
|
263
|
+
}
|
|
264
|
+
if (runType === "review_fix" && context?.branchUpkeepRequired === true) {
|
|
265
|
+
const baseBranch = typeof context.baseBranch === "string" ? context.baseBranch : "main";
|
|
266
|
+
return `Update the existing PR branch onto latest ${baseBranch}, resolve conflicts if needed, rerun narrow verification, and push the same branch.`;
|
|
267
|
+
}
|
|
268
|
+
switch (runType) {
|
|
269
|
+
case "review_fix":
|
|
270
|
+
return "Address the review feedback on the current PR branch, verify the fix, and push the same branch.";
|
|
271
|
+
case "ci_repair":
|
|
272
|
+
return "Fix the failing CI root cause on the current PR branch, verify it locally, and push the same branch.";
|
|
273
|
+
case "queue_repair":
|
|
274
|
+
return "Repair the merge-queue incident on the current PR branch, verify the fix, and push the same branch.";
|
|
275
|
+
case "implementation":
|
|
276
|
+
default:
|
|
277
|
+
return "Continue from the latest branch state, incorporate the new input, and publish updates to the existing issue branch if you make changes.";
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
function hasAuthoritativeGitHubFacts(issue, runType, context) {
|
|
281
|
+
return issue.prNumber !== undefined
|
|
282
|
+
|| issue.prHeadSha !== undefined
|
|
283
|
+
|| runType !== "implementation"
|
|
284
|
+
|| typeof context?.failureHeadSha === "string"
|
|
285
|
+
|| typeof context?.failingHeadSha === "string"
|
|
286
|
+
|| typeof context?.mergeStateStatus === "string"
|
|
287
|
+
|| typeof context?.checkName === "string"
|
|
288
|
+
|| typeof context?.reviewerName === "string";
|
|
289
|
+
}
|
|
290
|
+
function appendAuthoritativeGitHubFacts(lines, issue, runType, context) {
|
|
291
|
+
if (!hasAuthoritativeGitHubFacts(issue, runType, context)) {
|
|
292
|
+
return;
|
|
293
|
+
}
|
|
294
|
+
const prNumber = issue.prNumber !== undefined ? `#${issue.prNumber}` : undefined;
|
|
295
|
+
const headSha = typeof context?.failureHeadSha === "string"
|
|
296
|
+
? context.failureHeadSha
|
|
297
|
+
: typeof context?.failingHeadSha === "string"
|
|
298
|
+
? context.failingHeadSha
|
|
299
|
+
: issue.prHeadSha;
|
|
300
|
+
const mergeStateStatus = typeof context?.mergeStateStatus === "string" ? context.mergeStateStatus : undefined;
|
|
301
|
+
const baseBranch = typeof context?.baseBranch === "string" ? context.baseBranch : undefined;
|
|
302
|
+
const checkName = typeof context?.checkName === "string" ? context.checkName : undefined;
|
|
303
|
+
const jobName = typeof context?.jobName === "string" ? context.jobName : undefined;
|
|
304
|
+
const stepName = typeof context?.stepName === "string" ? context.stepName : undefined;
|
|
305
|
+
const reviewerName = typeof context?.reviewerName === "string" ? context.reviewerName : undefined;
|
|
306
|
+
const reviewBody = typeof context?.reviewBody === "string" ? context.reviewBody.trim() : "";
|
|
307
|
+
const summary = typeof context?.summary === "string" ? context.summary : undefined;
|
|
308
|
+
lines.push("## Authoritative GitHub Facts", "");
|
|
309
|
+
if (prNumber) {
|
|
310
|
+
lines.push(`- Current PR: ${prNumber}`);
|
|
311
|
+
}
|
|
312
|
+
if (headSha) {
|
|
313
|
+
lines.push(`- Current relevant head SHA: ${headSha}`);
|
|
314
|
+
}
|
|
315
|
+
if (issue.prReviewState) {
|
|
316
|
+
lines.push(`- Current review state: ${issue.prReviewState}`);
|
|
317
|
+
}
|
|
318
|
+
if (issue.prCheckStatus) {
|
|
319
|
+
lines.push(`- Current check status: ${issue.prCheckStatus}`);
|
|
320
|
+
}
|
|
321
|
+
if (mergeStateStatus) {
|
|
322
|
+
lines.push(`- Merge state against ${baseBranch ?? "base"}: ${mergeStateStatus}`);
|
|
323
|
+
}
|
|
324
|
+
if (checkName) {
|
|
325
|
+
lines.push(`- Relevant check: ${checkName}`);
|
|
326
|
+
}
|
|
327
|
+
if (jobName && jobName !== checkName) {
|
|
328
|
+
lines.push(`- Relevant job: ${jobName}`);
|
|
329
|
+
}
|
|
330
|
+
if (stepName) {
|
|
331
|
+
lines.push(`- Relevant step: ${stepName}`);
|
|
332
|
+
}
|
|
333
|
+
if (reviewerName) {
|
|
334
|
+
lines.push(`- Reviewer: ${reviewerName}`);
|
|
335
|
+
}
|
|
336
|
+
if (summary) {
|
|
337
|
+
lines.push(`- Summary: ${summary}`);
|
|
338
|
+
}
|
|
339
|
+
if (reviewBody) {
|
|
340
|
+
lines.push(`- Review body: ${reviewBody}`);
|
|
341
|
+
}
|
|
342
|
+
lines.push("");
|
|
343
|
+
}
|
|
344
|
+
function appendFactFreshness(lines, issue, runType, context) {
|
|
345
|
+
if (!hasAuthoritativeGitHubFacts(issue, runType, context)) {
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
const hasFreshFacts = context?.githubFactsFresh === true || context?.branchUpkeepRequired === true;
|
|
349
|
+
lines.push("## Fact Freshness", "");
|
|
350
|
+
if (hasFreshFacts) {
|
|
351
|
+
lines.push("GitHub facts below were refreshed immediately before this turn was created.");
|
|
352
|
+
}
|
|
353
|
+
else {
|
|
354
|
+
lines.push("GitHub facts below came from the triggering event or last known reconciliation state and may now be stale.");
|
|
355
|
+
lines.push("Verify the current PR head, review state, and check state in GitHub before making branch-mutating decisions.");
|
|
356
|
+
}
|
|
357
|
+
lines.push("");
|
|
358
|
+
}
|
|
359
|
+
function appendFollowUpPromptPrelude(lines, issue, runType, context) {
|
|
360
|
+
lines.push("## Follow-up Turn", "");
|
|
361
|
+
lines.push(`Why this turn exists: ${resolveFollowUpWhy(runType, context)}`);
|
|
362
|
+
lines.push(`Required action now: ${resolveFollowUpAction(runType, context)}`);
|
|
363
|
+
lines.push("");
|
|
364
|
+
appendLinearContext(lines, context);
|
|
365
|
+
const followUps = collectFollowUpInputs(context);
|
|
366
|
+
if (followUps.length > 0) {
|
|
367
|
+
lines.push("## What Changed Since The Last Turn", "");
|
|
368
|
+
for (const followUp of followUps) {
|
|
369
|
+
lines.push(`- ${followUp.type}${followUp.author ? ` from ${followUp.author}` : ""}: ${followUp.text}`);
|
|
370
|
+
}
|
|
371
|
+
lines.push("");
|
|
372
|
+
}
|
|
373
|
+
appendFactFreshness(lines, issue, runType, context);
|
|
374
|
+
appendAuthoritativeGitHubFacts(lines, issue, runType, context);
|
|
375
|
+
}
|
|
376
|
+
export function buildInitialRunPrompt(issue, runType, repoPath, context) {
|
|
377
|
+
const lines = buildPromptHeader(issue);
|
|
378
|
+
appendTaskObjective(lines, issue);
|
|
379
|
+
appendLinearContext(lines, context);
|
|
57
380
|
// Add run-type-specific context for reactive runs
|
|
58
381
|
switch (runType) {
|
|
59
382
|
case "ci_repair": {
|
|
@@ -68,24 +391,85 @@ function buildRunPrompt(issue, runType, repoPath, context) {
|
|
|
68
391
|
break;
|
|
69
392
|
}
|
|
70
393
|
case "review_fix":
|
|
71
|
-
lines.push("## Review Changes Requested", "", "A reviewer has requested changes on your PR. Address the feedback and push.", context?.reviewerName ? `Reviewer: ${String(context.reviewerName)}` : "", context?.reviewBody ? `\n## Review comment\n\n${String(context.reviewBody)}` : "", "", "Steps:", "1.
|
|
394
|
+
lines.push("## Review Changes Requested", "", "A reviewer has requested changes on your PR. Address the feedback and push.", context?.reviewerName ? `Reviewer: ${String(context.reviewerName)}` : "", context?.reviewBody ? `\n## Review comment\n\n${String(context.reviewBody)}` : "", "", "Steps:", "1. Start with the structured review context below. Treat the inline review comments as the primary repair checklist for this turn.", "2. Check the current diff (`git diff origin/main`) — a prior rebase may have already resolved some concerns (e.g., scope-bundling from stale commits).", "3. For each review point: if already resolved on the current head, note why. If not, fix it.", "4. If the structured review context looks incomplete, inspect the latest GitHub review threads directly before deciding you are done.", "5. Run verification, commit and push.", "6. If you believe all concerns are resolved, request a re-review: `gh pr edit <PR#> --add-reviewer <reviewer>`.", " Do NOT just post a comment saying \"resolved\" — the reviewer must re-review to dismiss the CHANGES_REQUESTED state.", "");
|
|
395
|
+
appendStructuredReviewContext(lines, context);
|
|
72
396
|
break;
|
|
73
397
|
case "queue_repair":
|
|
74
398
|
appendQueueRepairContext(lines, context);
|
|
75
399
|
lines.push("## Merge Queue Failure", "", "The merge queue rejected this PR. Rebase onto latest main and fix conflicts.", context?.failureReason ? `Failure reason: ${String(context.failureReason)}` : "", "", "Fetch and rebase onto latest main, resolve conflicts, run verification, push.", "If the conflict is a semantic contradiction, explain and stop.", "");
|
|
76
400
|
break;
|
|
77
401
|
}
|
|
78
|
-
// Append the repo's workflow file
|
|
79
402
|
const workflowBody = readWorkflowFile(repoPath, runType);
|
|
80
403
|
if (workflowBody) {
|
|
81
404
|
lines.push(workflowBody);
|
|
82
405
|
}
|
|
83
406
|
else if (runType === "implementation") {
|
|
84
|
-
|
|
85
|
-
lines.push("Implement the Linear issue. Read the issue via MCP for details.", "Run verification before finishing. Commit, push, and open a PR.");
|
|
407
|
+
lines.push("Implement the Linear issue. Read the issue via MCP for details.");
|
|
86
408
|
}
|
|
409
|
+
appendPublicationContract(lines, runType, issue, context);
|
|
87
410
|
return lines.join("\n");
|
|
88
411
|
}
|
|
412
|
+
export function buildFollowUpRunPrompt(issue, runType, repoPath, context) {
|
|
413
|
+
const lines = buildPromptHeader(issue);
|
|
414
|
+
appendFollowUpPromptPrelude(lines, issue, runType, context);
|
|
415
|
+
// Add run-type-specific context for reactive runs
|
|
416
|
+
switch (runType) {
|
|
417
|
+
case "ci_repair": {
|
|
418
|
+
const snapshot = context?.ciSnapshot && typeof context.ciSnapshot === "object"
|
|
419
|
+
? context.ciSnapshot
|
|
420
|
+
: undefined;
|
|
421
|
+
lines.push("## CI Repair", "", "A full CI iteration has settled failed on your PR. Start from the specific failing check/job/step below on the latest remote PR branch tip, fix that concrete failure first, then push to the same PR branch.", snapshot?.gateCheckName ? `Gate check: ${String(snapshot.gateCheckName)}` : "", snapshot?.gateCheckStatus ? `Gate status: ${String(snapshot.gateCheckStatus)}` : "", snapshot?.settledAt ? `Settled at: ${String(snapshot.settledAt)}` : "", context?.failureHeadSha ? `Failing head SHA: ${String(context.failureHeadSha)}` : "", context?.checkName ? `Failed check: ${String(context.checkName)}` : "", context?.jobName && context?.jobName !== context?.checkName ? `Failed job: ${String(context.jobName)}` : "", context?.stepName ? `Failed step: ${String(context.stepName)}` : "", context?.summary ? `Failure summary: ${String(context.summary)}` : "", Array.isArray(snapshot?.failedChecks) && snapshot.failedChecks.length > 0
|
|
422
|
+
? `Other failed checks in the settled snapshot (context only; ignore unless the logs show the same root cause):\n${snapshot.failedChecks.map((entry) => `- ${String(entry.name ?? "unknown")}${entry.summary ? `: ${String(entry.summary)}` : ""}`).join("\n")}`
|
|
423
|
+
: "", context?.checkUrl ? `Check URL: ${String(context.checkUrl)}` : "", Array.isArray(context?.annotations) && context.annotations.length > 0
|
|
424
|
+
? `Annotations:\n${context.annotations.map((entry) => `- ${String(entry)}`).join("\n")}`
|
|
425
|
+
: "", "", "Fetch the latest remote branch state first. If the branch moved since this failure, restart from the new tip instead of pushing older work.", "Read the latest logs for the named failing check, fix that root cause, and only broaden scope when the logs show direct fallout from the same issue.", "Do not change workflows, dependency installation, or unrelated tests unless the failing logs clearly point there.", "Run focused verification for the named failure, then commit and push.", "Do not open a new PR. Keep working on the existing branch until CI goes green or the situation is clearly stuck.", "Do not change test expectations unless the test is genuinely wrong.", "");
|
|
426
|
+
break;
|
|
427
|
+
}
|
|
428
|
+
case "review_fix":
|
|
429
|
+
lines.push("## Review Changes Requested", "", "A reviewer has requested changes on your PR. Address the feedback and push.", context?.reviewerName ? `Reviewer: ${String(context.reviewerName)}` : "", context?.reviewBody ? `\n## Review comment\n\n${String(context.reviewBody)}` : "", "", "Steps:", "1. Start with the structured review context below. Treat the inline review comments as the primary repair checklist for this turn.", "2. Check the current diff (`git diff origin/main`) — a prior rebase may have already resolved some concerns (e.g., scope-bundling from stale commits).", "3. For each review point: if already resolved on the current head, note why. If not, fix it.", "4. If the structured review context looks incomplete, inspect the latest GitHub review threads directly before deciding you are done.", "5. Run verification, commit and push.", "6. If you believe all concerns are resolved, request a re-review: `gh pr edit <PR#> --add-reviewer <reviewer>`.", " Do NOT just post a comment saying \"resolved\" — the reviewer must re-review to dismiss the CHANGES_REQUESTED state.", "");
|
|
430
|
+
appendStructuredReviewContext(lines, context);
|
|
431
|
+
break;
|
|
432
|
+
case "queue_repair":
|
|
433
|
+
appendQueueRepairContext(lines, context);
|
|
434
|
+
lines.push("## Merge Queue Failure", "", "The merge queue rejected this PR. Rebase onto latest main and fix conflicts.", context?.failureReason ? `Failure reason: ${String(context.failureReason)}` : "", "", "Fetch and rebase onto latest main, resolve conflicts, run verification, push.", "If the conflict is a semantic contradiction, explain and stop.", "");
|
|
435
|
+
break;
|
|
436
|
+
}
|
|
437
|
+
const workflowBody = readWorkflowFile(repoPath, runType);
|
|
438
|
+
if (workflowBody) {
|
|
439
|
+
lines.push(workflowBody);
|
|
440
|
+
}
|
|
441
|
+
else if (runType === "implementation") {
|
|
442
|
+
lines.push("Implement the Linear issue. Read the issue via MCP for details.");
|
|
443
|
+
}
|
|
444
|
+
appendPublicationContract(lines, runType, issue, context);
|
|
445
|
+
return lines.join("\n");
|
|
446
|
+
}
|
|
447
|
+
function shouldBuildFollowUpPrompt(runType, context) {
|
|
448
|
+
if (context?.followUpMode)
|
|
449
|
+
return true;
|
|
450
|
+
if (runType !== "implementation")
|
|
451
|
+
return true;
|
|
452
|
+
const wakeReason = typeof context?.wakeReason === "string" ? context.wakeReason : undefined;
|
|
453
|
+
return Boolean(wakeReason && wakeReason !== "delegated");
|
|
454
|
+
}
|
|
455
|
+
export function buildRunPrompt(issue, runType, repoPath, context) {
|
|
456
|
+
if (shouldBuildFollowUpPrompt(runType, context)) {
|
|
457
|
+
return buildFollowUpRunPrompt(issue, runType, repoPath, context);
|
|
458
|
+
}
|
|
459
|
+
return buildInitialRunPrompt(issue, runType, repoPath, context);
|
|
460
|
+
}
|
|
461
|
+
function shouldCompactThread(issue, threadGeneration, context) {
|
|
462
|
+
const followUpCount = typeof context?.followUpCount === "number" ? context.followUpCount : 0;
|
|
463
|
+
return issue.threadId !== undefined
|
|
464
|
+
&& (threadGeneration ?? 0) >= MAX_THREAD_GENERATION_BEFORE_COMPACTION
|
|
465
|
+
&& followUpCount >= MAX_FOLLOW_UPS_BEFORE_COMPACTION;
|
|
466
|
+
}
|
|
467
|
+
export function shouldReuseIssueThread(params) {
|
|
468
|
+
return Boolean(params.existingThreadId) && !params.compactThread && params.resumeThread;
|
|
469
|
+
}
|
|
470
|
+
function isBranchUpkeepRequired(context) {
|
|
471
|
+
return context?.branchUpkeepRequired === true;
|
|
472
|
+
}
|
|
89
473
|
export class RunOrchestrator {
|
|
90
474
|
config;
|
|
91
475
|
db;
|
|
@@ -100,6 +484,8 @@ export class RunOrchestrator {
|
|
|
100
484
|
idleReconciler;
|
|
101
485
|
linearSync;
|
|
102
486
|
activeThreadId;
|
|
487
|
+
workerId = `patchrelay:${process.pid}`;
|
|
488
|
+
activeSessionLeases = new Map();
|
|
103
489
|
botIdentity;
|
|
104
490
|
constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
|
|
105
491
|
this.config = config;
|
|
@@ -112,28 +498,103 @@ export class RunOrchestrator {
|
|
|
112
498
|
this.worktreeManager = new WorktreeManager(config);
|
|
113
499
|
this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
|
|
114
500
|
this.idleReconciler = new IdleIssueReconciler(db, config, {
|
|
115
|
-
requestMergeQueueAdmission: (issue, projectId) => this.requestMergeQueueAdmission(issue, projectId),
|
|
116
501
|
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
117
502
|
}, logger, feed);
|
|
118
503
|
this.queueHealthMonitor = new QueueHealthMonitor(db, config, {
|
|
119
504
|
advanceIdleIssue: (issue, newState, options) => this.idleReconciler.advanceIdleIssue(issue, newState, options),
|
|
505
|
+
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
120
506
|
}, logger, feed);
|
|
121
507
|
}
|
|
508
|
+
resolveRunWake(issue) {
|
|
509
|
+
const sessionWake = this.db.peekIssueSessionWake(issue.projectId, issue.linearIssueId);
|
|
510
|
+
if (sessionWake) {
|
|
511
|
+
return {
|
|
512
|
+
runType: sessionWake.runType,
|
|
513
|
+
context: sessionWake.context,
|
|
514
|
+
wakeReason: sessionWake.wakeReason,
|
|
515
|
+
resumeThread: sessionWake.resumeThread,
|
|
516
|
+
eventIds: sessionWake.eventIds,
|
|
517
|
+
};
|
|
518
|
+
}
|
|
519
|
+
return undefined;
|
|
520
|
+
}
|
|
521
|
+
appendWakeEventWithLease(lease, issue, runType, context, dedupeScope) {
|
|
522
|
+
let eventType;
|
|
523
|
+
let dedupeKey;
|
|
524
|
+
if (runType === "queue_repair") {
|
|
525
|
+
eventType = "merge_steward_incident";
|
|
526
|
+
dedupeKey = `${dedupeScope ?? "wake"}:queue_repair:${issue.linearIssueId}:${issue.prHeadSha ?? issue.lastGitHubFailureHeadSha ?? "unknown-sha"}`;
|
|
527
|
+
}
|
|
528
|
+
else if (runType === "ci_repair") {
|
|
529
|
+
eventType = "settled_red_ci";
|
|
530
|
+
dedupeKey = `${dedupeScope ?? "wake"}:ci_repair:${issue.linearIssueId}:${issue.lastGitHubFailureSignature ?? issue.prHeadSha ?? "unknown-sha"}`;
|
|
531
|
+
}
|
|
532
|
+
else if (runType === "review_fix") {
|
|
533
|
+
eventType = "review_changes_requested";
|
|
534
|
+
dedupeKey = `${dedupeScope ?? "wake"}:review_fix:${issue.linearIssueId}:${issue.prHeadSha ?? "unknown-sha"}`;
|
|
535
|
+
}
|
|
536
|
+
else {
|
|
537
|
+
eventType = "delegated";
|
|
538
|
+
dedupeKey = `${dedupeScope ?? "wake"}:implementation:${issue.linearIssueId}`;
|
|
539
|
+
}
|
|
540
|
+
return Boolean(this.db.appendIssueSessionEventWithLease(lease, {
|
|
541
|
+
projectId: issue.projectId,
|
|
542
|
+
linearIssueId: issue.linearIssueId,
|
|
543
|
+
eventType,
|
|
544
|
+
...(context ? { eventJson: JSON.stringify(context) } : {}),
|
|
545
|
+
dedupeKey,
|
|
546
|
+
}));
|
|
547
|
+
}
|
|
548
|
+
materializeLegacyPendingWake(issue, lease) {
|
|
549
|
+
if (!issue.pendingRunType)
|
|
550
|
+
return issue;
|
|
551
|
+
const context = issue.pendingRunContextJson
|
|
552
|
+
? JSON.parse(issue.pendingRunContextJson)
|
|
553
|
+
: undefined;
|
|
554
|
+
this.appendWakeEventWithLease(lease, issue, issue.pendingRunType, context, "legacy_pending");
|
|
555
|
+
const updated = this.db.upsertIssueWithLease(lease, {
|
|
556
|
+
projectId: issue.projectId,
|
|
557
|
+
linearIssueId: issue.linearIssueId,
|
|
558
|
+
pendingRunType: null,
|
|
559
|
+
pendingRunContextJson: null,
|
|
560
|
+
});
|
|
561
|
+
if (!updated)
|
|
562
|
+
return issue;
|
|
563
|
+
return this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
|
|
564
|
+
}
|
|
122
565
|
// ─── Run ────────────────────────────────────────────────────────
|
|
123
566
|
async run(item) {
|
|
124
567
|
const project = this.config.projects.find((p) => p.id === item.projectId);
|
|
125
568
|
if (!project)
|
|
126
569
|
return;
|
|
570
|
+
if (this.activeSessionLeases.has(this.issueSessionLeaseKey(item.projectId, item.issueId))) {
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
127
573
|
const issue = this.db.getIssue(item.projectId, item.issueId);
|
|
128
|
-
if (!issue
|
|
574
|
+
if (!issue || issue.activeRunId !== undefined)
|
|
129
575
|
return;
|
|
576
|
+
const issueSession = this.db.getIssueSession(item.projectId, item.issueId);
|
|
577
|
+
const leaseId = this.acquireIssueSessionLease(item.projectId, item.issueId);
|
|
578
|
+
if (!leaseId) {
|
|
579
|
+
this.logger.info({ issueKey: issue.issueKey, projectId: item.projectId }, "Skipped run because another worker holds the session lease");
|
|
580
|
+
return;
|
|
581
|
+
}
|
|
130
582
|
if (issue.prState === "merged") {
|
|
131
|
-
this.db.
|
|
583
|
+
this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
|
|
584
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
132
585
|
return;
|
|
133
586
|
}
|
|
134
|
-
const
|
|
135
|
-
const
|
|
136
|
-
|
|
587
|
+
const wakeIssue = this.materializeLegacyPendingWake(issue, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
|
|
588
|
+
const wake = this.resolveRunWake(wakeIssue);
|
|
589
|
+
if (!wake) {
|
|
590
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
591
|
+
return;
|
|
592
|
+
}
|
|
593
|
+
const { runType, context, resumeThread } = wake;
|
|
594
|
+
const effectiveContext = runType === "review_fix"
|
|
595
|
+
? await this.resolveReviewFixWakeContext(issue, context, project)
|
|
596
|
+
: context;
|
|
597
|
+
const isReviewFixBranchUpkeep = runType === "review_fix" && isBranchUpkeepRequired(effectiveContext);
|
|
137
598
|
// Check repair budgets
|
|
138
599
|
if (runType === "ci_repair" && issue.ciRepairAttempts >= DEFAULT_CI_REPAIR_BUDGET) {
|
|
139
600
|
this.escalate(issue, runType, `CI repair budget exhausted (${DEFAULT_CI_REPAIR_BUDGET} attempts)`);
|
|
@@ -143,22 +604,34 @@ export class RunOrchestrator {
|
|
|
143
604
|
this.escalate(issue, runType, `Queue repair budget exhausted (${DEFAULT_QUEUE_REPAIR_BUDGET} attempts)`);
|
|
144
605
|
return;
|
|
145
606
|
}
|
|
146
|
-
if (runType === "review_fix" && issue.reviewFixAttempts >= DEFAULT_REVIEW_FIX_BUDGET) {
|
|
607
|
+
if (runType === "review_fix" && !isReviewFixBranchUpkeep && issue.reviewFixAttempts >= DEFAULT_REVIEW_FIX_BUDGET) {
|
|
147
608
|
this.escalate(issue, runType, `Review fix budget exhausted (${DEFAULT_REVIEW_FIX_BUDGET} attempts)`);
|
|
148
609
|
return;
|
|
149
610
|
}
|
|
150
611
|
// Increment repair counters
|
|
151
612
|
if (runType === "ci_repair") {
|
|
152
|
-
this.db.
|
|
613
|
+
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, ciRepairAttempts: issue.ciRepairAttempts + 1 });
|
|
614
|
+
if (!updated) {
|
|
615
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
616
|
+
return;
|
|
617
|
+
}
|
|
153
618
|
}
|
|
154
619
|
if (runType === "queue_repair") {
|
|
155
|
-
this.db.
|
|
620
|
+
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueRepairAttempts: issue.queueRepairAttempts + 1 });
|
|
621
|
+
if (!updated) {
|
|
622
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
623
|
+
return;
|
|
624
|
+
}
|
|
156
625
|
}
|
|
157
|
-
if (runType === "review_fix") {
|
|
158
|
-
this.db.
|
|
626
|
+
if (runType === "review_fix" && !isReviewFixBranchUpkeep) {
|
|
627
|
+
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, reviewFixAttempts: issue.reviewFixAttempts + 1 });
|
|
628
|
+
if (!updated) {
|
|
629
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
630
|
+
return;
|
|
631
|
+
}
|
|
159
632
|
}
|
|
160
633
|
// Build prompt
|
|
161
|
-
const prompt = buildRunPrompt(issue, runType, project.repoPath,
|
|
634
|
+
const prompt = buildRunPrompt(issue, runType, project.repoPath, effectiveContext);
|
|
162
635
|
// Resolve workspace
|
|
163
636
|
const issueRef = sanitizePathSegment(issue.issueKey ?? issue.linearIssueId);
|
|
164
637
|
const slug = issue.title ? slugify(issue.title) : "";
|
|
@@ -166,9 +639,13 @@ export class RunOrchestrator {
|
|
|
166
639
|
const branchName = issue.branchName ?? `${project.branchPrefix}/${branchSuffix}`;
|
|
167
640
|
const worktreePath = issue.worktreePath ?? `${project.worktreeRoot}/${issueRef}`;
|
|
168
641
|
// Claim the run atomically
|
|
169
|
-
const run = this.db.
|
|
642
|
+
const run = this.db.withIssueSessionLease(item.projectId, item.issueId, leaseId, () => {
|
|
170
643
|
const fresh = this.db.getIssue(item.projectId, item.issueId);
|
|
171
|
-
if (!fresh
|
|
644
|
+
if (!fresh || fresh.activeRunId !== undefined)
|
|
645
|
+
return undefined;
|
|
646
|
+
const wakeIssue = this.materializeLegacyPendingWake(fresh, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
|
|
647
|
+
const freshWake = this.resolveRunWake(wakeIssue);
|
|
648
|
+
if (!freshWake || freshWake.runType !== runType)
|
|
172
649
|
return undefined;
|
|
173
650
|
const created = this.db.createRun({
|
|
174
651
|
issueId: fresh.id,
|
|
@@ -177,10 +654,10 @@ export class RunOrchestrator {
|
|
|
177
654
|
runType,
|
|
178
655
|
promptText: prompt,
|
|
179
656
|
});
|
|
180
|
-
const failureHeadSha = typeof
|
|
181
|
-
?
|
|
182
|
-
: typeof
|
|
183
|
-
const failureSignature = typeof
|
|
657
|
+
const failureHeadSha = typeof effectiveContext?.failureHeadSha === "string"
|
|
658
|
+
? effectiveContext.failureHeadSha
|
|
659
|
+
: typeof effectiveContext?.headSha === "string" ? effectiveContext.headSha : undefined;
|
|
660
|
+
const failureSignature = typeof effectiveContext?.failureSignature === "string" ? effectiveContext.failureSignature : undefined;
|
|
184
661
|
this.db.upsertIssue({
|
|
185
662
|
projectId: item.projectId,
|
|
186
663
|
linearIssueId: item.issueId,
|
|
@@ -201,11 +678,15 @@ export class RunOrchestrator {
|
|
|
201
678
|
}
|
|
202
679
|
: {}),
|
|
203
680
|
});
|
|
204
|
-
this.db.
|
|
681
|
+
this.db.consumeIssueSessionEvents(item.projectId, item.issueId, freshWake.eventIds, created.id);
|
|
682
|
+
this.db.setIssueSessionLastWakeReason(item.projectId, item.issueId, freshWake.wakeReason ?? null);
|
|
683
|
+
this.db.setBranchOwnerWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, "patchrelay");
|
|
205
684
|
return created;
|
|
206
685
|
});
|
|
207
|
-
if (!run)
|
|
686
|
+
if (!run) {
|
|
687
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
208
688
|
return;
|
|
689
|
+
}
|
|
209
690
|
this.feed?.publish({
|
|
210
691
|
level: "info",
|
|
211
692
|
kind: "stage",
|
|
@@ -217,6 +698,7 @@ export class RunOrchestrator {
|
|
|
217
698
|
});
|
|
218
699
|
let threadId;
|
|
219
700
|
let turnId;
|
|
701
|
+
let parentThreadId;
|
|
220
702
|
try {
|
|
221
703
|
// Ensure worktree
|
|
222
704
|
await this.worktreeManager.ensureIssueWorktree(project.repoPath, project.worktreeRoot, worktreePath, branchName, { allowExistingOutsideRoot: issue.branchName !== undefined });
|
|
@@ -232,6 +714,7 @@ export class RunOrchestrator {
|
|
|
232
714
|
const credentialHelper = `!f() { echo "username=x-access-token"; echo "password=$(cat ${this.botIdentity.tokenFile})"; }; f`;
|
|
233
715
|
await execCommand(gitBin, ["-C", worktreePath, "config", "credential.helper", credentialHelper], { timeoutMs: 5_000 });
|
|
234
716
|
}
|
|
717
|
+
await this.resetWorktreeToTrackedBranch(worktreePath, branchName, issue);
|
|
235
718
|
// Freshen the worktree: fetch + rebase onto latest base branch.
|
|
236
719
|
// This prevents branch contamination when local main has drifted
|
|
237
720
|
// and avoids scope-bundling review rejections from stale commits.
|
|
@@ -245,15 +728,24 @@ export class RunOrchestrator {
|
|
|
245
728
|
if (prepareResult.ran && prepareResult.exitCode !== 0) {
|
|
246
729
|
throw new Error(`prepare-worktree hook failed (exit ${prepareResult.exitCode}): ${prepareResult.stderr?.slice(0, 500) ?? ""}`);
|
|
247
730
|
}
|
|
248
|
-
|
|
249
|
-
//
|
|
250
|
-
|
|
731
|
+
this.assertLaunchLease(run, "before starting the Codex turn");
|
|
732
|
+
// Reuse the existing thread only for additive follow-ups that explicitly
|
|
733
|
+
// request continuity. Fresh review-fix runs now start a new thread so the
|
|
734
|
+
// model is not anchored to the implementation conversation that produced
|
|
735
|
+
// the rejected patch. If the thread has accumulated many resumptions and
|
|
736
|
+
// batched follow-ups, compact by starting a fresh main thread while
|
|
737
|
+
// keeping a parent link.
|
|
738
|
+
const compactThread = shouldCompactThread(issue, issueSession?.threadGeneration, effectiveContext);
|
|
739
|
+
if (compactThread && issue.threadId) {
|
|
740
|
+
parentThreadId = issue.threadId;
|
|
741
|
+
}
|
|
742
|
+
if (shouldReuseIssueThread({ existingThreadId: issue.threadId, compactThread, resumeThread })) {
|
|
251
743
|
threadId = issue.threadId;
|
|
252
744
|
}
|
|
253
745
|
else {
|
|
254
746
|
const thread = await this.codex.startThread({ cwd: worktreePath });
|
|
255
747
|
threadId = thread.id;
|
|
256
|
-
this.db.
|
|
748
|
+
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
|
|
257
749
|
}
|
|
258
750
|
try {
|
|
259
751
|
const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
|
|
@@ -266,7 +758,7 @@ export class RunOrchestrator {
|
|
|
266
758
|
this.logger.info({ issueKey: issue.issueKey, staleThreadId: threadId }, "Thread is stale, retrying with fresh thread");
|
|
267
759
|
const thread = await this.codex.startThread({ cwd: worktreePath });
|
|
268
760
|
threadId = thread.id;
|
|
269
|
-
this.db.
|
|
761
|
+
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
|
|
270
762
|
const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
|
|
271
763
|
turnId = turn.turnId;
|
|
272
764
|
}
|
|
@@ -274,26 +766,39 @@ export class RunOrchestrator {
|
|
|
274
766
|
throw turnError;
|
|
275
767
|
}
|
|
276
768
|
}
|
|
769
|
+
this.assertLaunchLease(run, "after starting the Codex turn");
|
|
277
770
|
}
|
|
278
771
|
catch (error) {
|
|
279
772
|
const message = error instanceof Error ? error.message : String(error);
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
projectId: item.projectId,
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
773
|
+
const lostLease = error instanceof Error && error.name === "IssueSessionLeaseLostError";
|
|
774
|
+
if (!lostLease) {
|
|
775
|
+
this.db.finishRunWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, run.id, {
|
|
776
|
+
status: "failed",
|
|
777
|
+
failureReason: message,
|
|
778
|
+
});
|
|
779
|
+
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
780
|
+
projectId: item.projectId,
|
|
781
|
+
linearIssueId: item.issueId,
|
|
782
|
+
activeRunId: null,
|
|
783
|
+
factoryState: "failed",
|
|
784
|
+
});
|
|
785
|
+
}
|
|
287
786
|
this.logger.error({ issueKey: issue.issueKey, runType, error: message }, `Failed to launch ${runType} run`);
|
|
288
787
|
const failedIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
289
788
|
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(runType, `Failed to start ${lowerCaseFirst(message)}`));
|
|
290
789
|
void this.linearSync.syncSession(failedIssue, { activeRunType: runType });
|
|
790
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
291
791
|
throw error;
|
|
292
792
|
}
|
|
293
|
-
this.
|
|
793
|
+
this.assertLaunchLease(run, "before recording the active thread");
|
|
794
|
+
if (!this.db.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
|
|
795
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping run thread update after losing issue-session lease");
|
|
796
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
294
799
|
// Reset zombie recovery counter — this run started successfully
|
|
295
800
|
if (issue.zombieRecoveryAttempts > 0) {
|
|
296
|
-
this.db.
|
|
801
|
+
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
297
802
|
projectId: item.projectId,
|
|
298
803
|
linearIssueId: item.issueId,
|
|
299
804
|
zombieRecoveryAttempts: 0,
|
|
@@ -357,6 +862,44 @@ export class RunOrchestrator {
|
|
|
357
862
|
if (didStash)
|
|
358
863
|
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
359
864
|
}
|
|
865
|
+
async resetWorktreeToTrackedBranch(worktreePath, branchName, issue) {
|
|
866
|
+
const gitBin = this.config.runner.gitBin;
|
|
867
|
+
const branchFetch = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", branchName], { timeoutMs: 60_000 });
|
|
868
|
+
const hasRemoteBranch = branchFetch.exitCode === 0;
|
|
869
|
+
await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
|
|
870
|
+
await execCommand(gitBin, ["-C", worktreePath, "merge", "--abort"], { timeoutMs: 10_000 });
|
|
871
|
+
await execCommand(gitBin, ["-C", worktreePath, "cherry-pick", "--abort"], { timeoutMs: 10_000 });
|
|
872
|
+
await execCommand(gitBin, ["-C", worktreePath, "am", "--abort"], { timeoutMs: 10_000 });
|
|
873
|
+
await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", "HEAD"], { timeoutMs: 30_000 });
|
|
874
|
+
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
875
|
+
const checkoutTarget = hasRemoteBranch ? `origin/${branchName}` : branchName;
|
|
876
|
+
const checkoutResult = await execCommand(gitBin, ["-C", worktreePath, "checkout", "-B", branchName, checkoutTarget], { timeoutMs: 30_000 });
|
|
877
|
+
if (checkoutResult.exitCode !== 0) {
|
|
878
|
+
throw new Error(`Failed to restore ${branchName} worktree state: ${checkoutResult.stderr?.slice(0, 300) ?? "git checkout failed"}`);
|
|
879
|
+
}
|
|
880
|
+
const resetTarget = hasRemoteBranch ? `origin/${branchName}` : "HEAD";
|
|
881
|
+
const resetResult = await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", resetTarget], { timeoutMs: 30_000 });
|
|
882
|
+
if (resetResult.exitCode !== 0) {
|
|
883
|
+
throw new Error(`Failed to reset ${branchName} worktree state: ${resetResult.stderr?.slice(0, 300) ?? "git reset failed"}`);
|
|
884
|
+
}
|
|
885
|
+
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
886
|
+
this.logger.debug({ issueKey: issue.issueKey, branchName, hasRemoteBranch }, "Reset issue worktree to tracked branch state");
|
|
887
|
+
}
|
|
888
|
+
async restoreIdleWorktree(issue) {
|
|
889
|
+
if (!issue.worktreePath || !issue.branchName)
|
|
890
|
+
return;
|
|
891
|
+
try {
|
|
892
|
+
await this.resetWorktreeToTrackedBranch(issue.worktreePath, issue.branchName, issue);
|
|
893
|
+
}
|
|
894
|
+
catch (error) {
|
|
895
|
+
this.logger.warn({
|
|
896
|
+
issueKey: issue.issueKey,
|
|
897
|
+
branchName: issue.branchName,
|
|
898
|
+
worktreePath: issue.worktreePath,
|
|
899
|
+
error: error instanceof Error ? error.message : String(error),
|
|
900
|
+
}, "Failed to restore idle worktree after interrupted run");
|
|
901
|
+
}
|
|
902
|
+
}
|
|
360
903
|
// ─── Notification handler ─────────────────────────────────────────
|
|
361
904
|
async handleCodexNotification(notification) {
|
|
362
905
|
// threadId is present on turn-level notifications but NOT on item-level ones.
|
|
@@ -374,6 +917,10 @@ export class RunOrchestrator {
|
|
|
374
917
|
const run = this.db.getRunByThreadId(threadId);
|
|
375
918
|
if (!run)
|
|
376
919
|
return;
|
|
920
|
+
if (!this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
921
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Ignoring Codex notification after losing issue-session lease");
|
|
922
|
+
return;
|
|
923
|
+
}
|
|
377
924
|
const turnId = typeof notification.params.turnId === "string" ? notification.params.turnId : undefined;
|
|
378
925
|
if (this.config.runner.codex.persistExtendedHistory) {
|
|
379
926
|
this.db.saveThreadEvent({
|
|
@@ -402,18 +949,26 @@ export class RunOrchestrator {
|
|
|
402
949
|
const completedTurnId = extractTurnId(notification.params);
|
|
403
950
|
const status = resolveRunCompletionStatus(notification.params);
|
|
404
951
|
if (status === "failed") {
|
|
405
|
-
this.
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
952
|
+
const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
953
|
+
this.db.finishRunWithLease(lease, run.id, {
|
|
954
|
+
status: "failed",
|
|
955
|
+
threadId,
|
|
956
|
+
...(completedTurnId ? { turnId: completedTurnId } : {}),
|
|
957
|
+
failureReason: "Codex reported the turn completed in a failed state",
|
|
958
|
+
});
|
|
959
|
+
this.db.upsertIssueWithLease(lease, {
|
|
960
|
+
projectId: run.projectId,
|
|
961
|
+
linearIssueId: run.linearIssueId,
|
|
962
|
+
activeRunId: null,
|
|
963
|
+
factoryState: "failed",
|
|
964
|
+
});
|
|
965
|
+
return true;
|
|
416
966
|
});
|
|
967
|
+
if (!updated) {
|
|
968
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping failed-turn cleanup after losing issue-session lease");
|
|
969
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
417
972
|
this.feed?.publish({
|
|
418
973
|
level: "error",
|
|
419
974
|
kind: "turn",
|
|
@@ -428,11 +983,12 @@ export class RunOrchestrator {
|
|
|
428
983
|
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
429
984
|
this.linearSync.clearProgress(run.id);
|
|
430
985
|
this.activeThreadId = undefined;
|
|
986
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
431
987
|
return;
|
|
432
988
|
}
|
|
433
989
|
// Complete the run
|
|
434
990
|
const trackedIssue = this.db.issueToTrackedIssue(issue);
|
|
435
|
-
const report = buildStageReport(run, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
|
|
991
|
+
const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
|
|
436
992
|
// Determine post-run state based on current PR metadata.
|
|
437
993
|
const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
438
994
|
const verifiedRepairError = await this.verifyReactiveRunAdvancedBranch(run, freshIssue);
|
|
@@ -453,10 +1009,32 @@ export class RunOrchestrator {
|
|
|
453
1009
|
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
454
1010
|
this.linearSync.clearProgress(run.id);
|
|
455
1011
|
this.activeThreadId = undefined;
|
|
1012
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
456
1013
|
return;
|
|
457
1014
|
}
|
|
458
|
-
const
|
|
459
|
-
|
|
1015
|
+
const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
|
|
1016
|
+
if (publishedOutcomeError) {
|
|
1017
|
+
this.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
1018
|
+
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1019
|
+
this.feed?.publish({
|
|
1020
|
+
level: "warn",
|
|
1021
|
+
kind: "turn",
|
|
1022
|
+
issueKey: freshIssue.issueKey,
|
|
1023
|
+
projectId: run.projectId,
|
|
1024
|
+
stage: run.runType,
|
|
1025
|
+
status: "publish_incomplete",
|
|
1026
|
+
summary: publishedOutcomeError,
|
|
1027
|
+
});
|
|
1028
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
1029
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1030
|
+
this.linearSync.clearProgress(run.id);
|
|
1031
|
+
this.activeThreadId = undefined;
|
|
1032
|
+
return;
|
|
1033
|
+
}
|
|
1034
|
+
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
1035
|
+
const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
|
|
1036
|
+
const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
|
|
1037
|
+
const completed = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
460
1038
|
this.db.finishRun(run.id, {
|
|
461
1039
|
status: "completed",
|
|
462
1040
|
threadId,
|
|
@@ -468,9 +1046,10 @@ export class RunOrchestrator {
|
|
|
468
1046
|
projectId: run.projectId,
|
|
469
1047
|
linearIssueId: run.linearIssueId,
|
|
470
1048
|
activeRunId: null,
|
|
471
|
-
...(postRunState === "awaiting_queue" ? { queueLabelApplied: false } : {}),
|
|
472
1049
|
...(postRunState ? { factoryState: postRunState } : {}),
|
|
473
|
-
|
|
1050
|
+
pendingRunType: null,
|
|
1051
|
+
pendingRunContextJson: null,
|
|
1052
|
+
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
474
1053
|
? {
|
|
475
1054
|
lastGitHubFailureSource: null,
|
|
476
1055
|
lastGitHubFailureHeadSha: null,
|
|
@@ -483,15 +1062,31 @@ export class RunOrchestrator {
|
|
|
483
1062
|
lastAttemptedFailureHeadSha: null,
|
|
484
1063
|
lastAttemptedFailureSignature: null,
|
|
485
1064
|
}
|
|
486
|
-
: {}),
|
|
1065
|
+
: {})),
|
|
487
1066
|
});
|
|
488
|
-
if (
|
|
489
|
-
this.
|
|
1067
|
+
if (postRunFollowUp) {
|
|
1068
|
+
return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
490
1069
|
}
|
|
1070
|
+
return true;
|
|
491
1071
|
});
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
this.
|
|
1072
|
+
if (!completed) {
|
|
1073
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping completion writes after losing issue-session lease");
|
|
1074
|
+
this.linearSync.clearProgress(run.id);
|
|
1075
|
+
this.activeThreadId = undefined;
|
|
1076
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1077
|
+
return;
|
|
1078
|
+
}
|
|
1079
|
+
if (postRunFollowUp) {
|
|
1080
|
+
this.feed?.publish({
|
|
1081
|
+
level: "info",
|
|
1082
|
+
kind: "stage",
|
|
1083
|
+
issueKey: issue.issueKey,
|
|
1084
|
+
projectId: run.projectId,
|
|
1085
|
+
stage: postRunFollowUp.factoryState,
|
|
1086
|
+
status: "follow_up_queued",
|
|
1087
|
+
summary: postRunFollowUp.summary,
|
|
1088
|
+
});
|
|
1089
|
+
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
495
1090
|
}
|
|
496
1091
|
this.feed?.publish({
|
|
497
1092
|
level: "info",
|
|
@@ -504,7 +1099,7 @@ export class RunOrchestrator {
|
|
|
504
1099
|
detail: summarizeCurrentThread(thread).latestAgentMessage,
|
|
505
1100
|
});
|
|
506
1101
|
// Emit Linear completion activity + plan
|
|
507
|
-
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ??
|
|
1102
|
+
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
508
1103
|
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
509
1104
|
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
510
1105
|
runType: run.runType,
|
|
@@ -515,6 +1110,7 @@ export class RunOrchestrator {
|
|
|
515
1110
|
void this.linearSync.syncSession(updatedIssue);
|
|
516
1111
|
this.linearSync.clearProgress(run.id);
|
|
517
1112
|
this.activeThreadId = undefined;
|
|
1113
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
518
1114
|
}
|
|
519
1115
|
// ─── Active status for query ──────────────────────────────────────
|
|
520
1116
|
async getActiveRunStatus(issueKey) {
|
|
@@ -543,6 +1139,44 @@ export class RunOrchestrator {
|
|
|
543
1139
|
// Advance issues stuck in pr_open whose stored PR metadata already
|
|
544
1140
|
// shows they should transition (e.g. approved PR, missed webhook).
|
|
545
1141
|
await this.idleReconciler.reconcile();
|
|
1142
|
+
await this.reconcileMergedLinearCompletion();
|
|
1143
|
+
}
|
|
1144
|
+
async reconcileMergedLinearCompletion() {
|
|
1145
|
+
for (const issue of this.db.listIssues()) {
|
|
1146
|
+
if (issue.prState !== "merged")
|
|
1147
|
+
continue;
|
|
1148
|
+
if (issue.currentLinearStateType?.trim().toLowerCase() === "completed")
|
|
1149
|
+
continue;
|
|
1150
|
+
const linear = await this.linearProvider.forProject(issue.projectId).catch(() => undefined);
|
|
1151
|
+
if (!linear)
|
|
1152
|
+
continue;
|
|
1153
|
+
try {
|
|
1154
|
+
const liveIssue = await linear.getIssue(issue.linearIssueId);
|
|
1155
|
+
const targetState = resolvePreferredCompletedLinearState(liveIssue);
|
|
1156
|
+
if (!targetState)
|
|
1157
|
+
continue;
|
|
1158
|
+
const normalizedCurrent = liveIssue.stateName?.trim().toLowerCase();
|
|
1159
|
+
if (normalizedCurrent === targetState.trim().toLowerCase()) {
|
|
1160
|
+
this.db.upsertIssue({
|
|
1161
|
+
projectId: issue.projectId,
|
|
1162
|
+
linearIssueId: issue.linearIssueId,
|
|
1163
|
+
...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
|
|
1164
|
+
...(liveIssue.stateType ? { currentLinearStateType: liveIssue.stateType } : {}),
|
|
1165
|
+
});
|
|
1166
|
+
continue;
|
|
1167
|
+
}
|
|
1168
|
+
const updated = await linear.setIssueState(issue.linearIssueId, targetState);
|
|
1169
|
+
this.db.upsertIssue({
|
|
1170
|
+
projectId: issue.projectId,
|
|
1171
|
+
linearIssueId: issue.linearIssueId,
|
|
1172
|
+
...(updated.stateName ? { currentLinearState: updated.stateName } : {}),
|
|
1173
|
+
...(updated.stateType ? { currentLinearStateType: updated.stateType } : {}),
|
|
1174
|
+
});
|
|
1175
|
+
}
|
|
1176
|
+
catch (error) {
|
|
1177
|
+
this.logger.warn({ issueKey: issue.issueKey, error: error instanceof Error ? error.message : String(error) }, "Failed to reconcile merged issue to a completed Linear state");
|
|
1178
|
+
}
|
|
1179
|
+
}
|
|
546
1180
|
}
|
|
547
1181
|
// advanceIdleIssue is now on IdleIssueReconciler — delegate for internal callers
|
|
548
1182
|
advanceIdleIssue(issue, newState, options) {
|
|
@@ -560,24 +1194,41 @@ export class RunOrchestrator {
|
|
|
560
1194
|
return;
|
|
561
1195
|
// If PR already merged, transition to done — no retry needed
|
|
562
1196
|
if (fresh.prState === "merged") {
|
|
563
|
-
this.
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
1197
|
+
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
1198
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1199
|
+
projectId: fresh.projectId,
|
|
1200
|
+
linearIssueId: fresh.linearIssueId,
|
|
1201
|
+
factoryState: "done",
|
|
1202
|
+
zombieRecoveryAttempts: 0,
|
|
1203
|
+
lastZombieRecoveryAt: null,
|
|
1204
|
+
});
|
|
1205
|
+
return true;
|
|
569
1206
|
});
|
|
1207
|
+
if (!updated) {
|
|
1208
|
+
this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping merged recovery completion after losing issue-session lease");
|
|
1209
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
1210
|
+
return;
|
|
1211
|
+
}
|
|
570
1212
|
this.logger.info({ issueKey: fresh.issueKey, reason }, "Recovery: PR already merged — transitioning to done");
|
|
1213
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
571
1214
|
return;
|
|
572
1215
|
}
|
|
573
1216
|
// Budget check
|
|
574
1217
|
const attempts = fresh.zombieRecoveryAttempts + 1;
|
|
575
1218
|
if (attempts > DEFAULT_ZOMBIE_RECOVERY_BUDGET) {
|
|
576
|
-
this.
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
1219
|
+
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
1220
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1221
|
+
projectId: fresh.projectId,
|
|
1222
|
+
linearIssueId: fresh.linearIssueId,
|
|
1223
|
+
factoryState: "escalated",
|
|
1224
|
+
});
|
|
1225
|
+
return true;
|
|
580
1226
|
});
|
|
1227
|
+
if (!updated) {
|
|
1228
|
+
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery escalation after losing issue-session lease");
|
|
1229
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
1230
|
+
return;
|
|
1231
|
+
}
|
|
581
1232
|
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: budget exhausted — escalating");
|
|
582
1233
|
this.feed?.publish({
|
|
583
1234
|
level: "error",
|
|
@@ -588,6 +1239,7 @@ export class RunOrchestrator {
|
|
|
588
1239
|
status: "budget_exhausted",
|
|
589
1240
|
summary: `${reason} recovery failed after ${DEFAULT_ZOMBIE_RECOVERY_BUDGET} attempts`,
|
|
590
1241
|
});
|
|
1242
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
591
1243
|
return;
|
|
592
1244
|
}
|
|
593
1245
|
// Exponential backoff — skip if delay hasn't elapsed
|
|
@@ -600,14 +1252,22 @@ export class RunOrchestrator {
|
|
|
600
1252
|
}
|
|
601
1253
|
}
|
|
602
1254
|
// Re-enqueue with backoff tracking
|
|
603
|
-
this.
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
1255
|
+
const requeued = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
1256
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1257
|
+
projectId: fresh.projectId,
|
|
1258
|
+
linearIssueId: fresh.linearIssueId,
|
|
1259
|
+
pendingRunType: null,
|
|
1260
|
+
pendingRunContextJson: null,
|
|
1261
|
+
zombieRecoveryAttempts: attempts,
|
|
1262
|
+
lastZombieRecoveryAt: new Date().toISOString(),
|
|
1263
|
+
});
|
|
1264
|
+
return this.appendWakeEventWithLease(lease, fresh, runType, undefined, `recovery:${attempts}`);
|
|
610
1265
|
});
|
|
1266
|
+
if (!requeued) {
|
|
1267
|
+
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery re-enqueue after losing issue-session lease");
|
|
1268
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
1269
|
+
return;
|
|
1270
|
+
}
|
|
611
1271
|
this.enqueueIssue(fresh.projectId, fresh.linearIssueId);
|
|
612
1272
|
this.logger.info({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: re-enqueued with backoff");
|
|
613
1273
|
}
|
|
@@ -615,24 +1275,38 @@ export class RunOrchestrator {
|
|
|
615
1275
|
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
616
1276
|
if (!issue)
|
|
617
1277
|
return;
|
|
1278
|
+
let recoveryLease = this.claimLeaseForReconciliation(run.projectId, run.linearIssueId);
|
|
1279
|
+
if (recoveryLease === "skip" && await this.reclaimForeignRecoveryLeaseIfSafe(run, issue)) {
|
|
1280
|
+
recoveryLease = true;
|
|
1281
|
+
}
|
|
1282
|
+
if (recoveryLease === "skip")
|
|
1283
|
+
return;
|
|
1284
|
+
const acquiredRecoveryLease = recoveryLease === true;
|
|
618
1285
|
// If the issue reached a terminal state while this run was active
|
|
619
1286
|
// (e.g. pr_merged processed, DB manually edited), just release the run.
|
|
620
1287
|
if (TERMINAL_STATES.has(issue.factoryState)) {
|
|
621
|
-
this.
|
|
1288
|
+
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
622
1289
|
this.db.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
623
1290
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
624
1291
|
});
|
|
625
1292
|
this.logger.info({ issueKey: issue.issueKey, runId: run.id, factoryState: issue.factoryState }, "Reconciliation: released run on terminal issue");
|
|
1293
|
+
const releasedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1294
|
+
void this.linearSync.syncSession(releasedIssue, { activeRunType: run.runType });
|
|
1295
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
626
1296
|
return;
|
|
627
1297
|
}
|
|
628
1298
|
// Zombie run: claimed in DB but Codex never started (no thread).
|
|
629
1299
|
if (!run.threadId) {
|
|
630
1300
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
|
|
631
|
-
this.
|
|
1301
|
+
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
632
1302
|
this.db.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
633
1303
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
634
1304
|
});
|
|
635
1305
|
this.recoverOrEscalate(issue, run.runType, "zombie");
|
|
1306
|
+
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1307
|
+
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "The Codex turn never started before PatchRelay restarted."));
|
|
1308
|
+
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
1309
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
636
1310
|
return;
|
|
637
1311
|
}
|
|
638
1312
|
// Read Codex state — thread may not exist after app-server restart.
|
|
@@ -642,11 +1316,15 @@ export class RunOrchestrator {
|
|
|
642
1316
|
}
|
|
643
1317
|
catch {
|
|
644
1318
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
|
|
645
|
-
this.
|
|
1319
|
+
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
646
1320
|
this.db.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
647
1321
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
648
1322
|
});
|
|
649
1323
|
this.recoverOrEscalate(issue, run.runType, "stale_thread");
|
|
1324
|
+
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1325
|
+
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "PatchRelay lost the active Codex thread after restart and needs to recover."));
|
|
1326
|
+
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
1327
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
650
1328
|
return;
|
|
651
1329
|
}
|
|
652
1330
|
// Check Linear state (non-fatal — token refresh may fail)
|
|
@@ -656,7 +1334,7 @@ export class RunOrchestrator {
|
|
|
656
1334
|
if (linearIssue) {
|
|
657
1335
|
const stopState = resolveAuthoritativeLinearStopState(linearIssue);
|
|
658
1336
|
if (stopState?.isFinal) {
|
|
659
|
-
this.
|
|
1337
|
+
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
660
1338
|
this.db.finishRun(run.id, { status: "released" });
|
|
661
1339
|
this.db.upsertIssue({
|
|
662
1340
|
projectId: run.projectId,
|
|
@@ -675,28 +1353,60 @@ export class RunOrchestrator {
|
|
|
675
1353
|
status: "reconciled",
|
|
676
1354
|
summary: `Linear state ${stopState.stateName} \u2192 done`,
|
|
677
1355
|
});
|
|
1356
|
+
const doneIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1357
|
+
void this.linearSync.syncSession(doneIssue, { activeRunType: run.runType });
|
|
1358
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
678
1359
|
return;
|
|
679
1360
|
}
|
|
680
1361
|
}
|
|
681
1362
|
}
|
|
682
|
-
const latestTurn = thread.
|
|
1363
|
+
const latestTurn = getThreadTurns(thread).at(-1);
|
|
683
1364
|
// Handle interrupted turn — fail the run rather than retrying indefinitely.
|
|
684
1365
|
// The agent may have partially completed work (commits, PR) before interruption.
|
|
685
1366
|
// Reactive loops (CI repair, review fix) will handle follow-up if needed.
|
|
686
1367
|
if (latestTurn?.status === "interrupted") {
|
|
687
1368
|
this.logger.warn({ issueKey: issue.issueKey, runType: run.runType, threadId: run.threadId }, "Run has interrupted turn — marking as failed");
|
|
688
1369
|
// Interrupted runs are not real failures — undo the budget increment.
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
1370
|
+
const repairedCounters = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
1371
|
+
if (run.runType === "ci_repair" && issue.ciRepairAttempts > 0) {
|
|
1372
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1373
|
+
projectId: issue.projectId,
|
|
1374
|
+
linearIssueId: issue.linearIssueId,
|
|
1375
|
+
ciRepairAttempts: issue.ciRepairAttempts - 1,
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1378
|
+
else if (run.runType === "queue_repair" && issue.queueRepairAttempts > 0) {
|
|
1379
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1380
|
+
projectId: issue.projectId,
|
|
1381
|
+
linearIssueId: issue.linearIssueId,
|
|
1382
|
+
queueRepairAttempts: issue.queueRepairAttempts - 1,
|
|
1383
|
+
});
|
|
1384
|
+
}
|
|
1385
|
+
else if (run.runType === "review_fix" && issue.reviewFixAttempts > 0) {
|
|
1386
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1387
|
+
projectId: issue.projectId,
|
|
1388
|
+
linearIssueId: issue.linearIssueId,
|
|
1389
|
+
reviewFixAttempts: issue.reviewFixAttempts - 1,
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
if (run.runType === "ci_repair" || run.runType === "queue_repair") {
|
|
1393
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1394
|
+
projectId: issue.projectId,
|
|
1395
|
+
linearIssueId: issue.linearIssueId,
|
|
1396
|
+
lastAttemptedFailureHeadSha: null,
|
|
1397
|
+
lastAttemptedFailureSignature: null,
|
|
1398
|
+
});
|
|
1399
|
+
}
|
|
1400
|
+
return true;
|
|
1401
|
+
});
|
|
1402
|
+
if (!repairedCounters) {
|
|
1403
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping interrupted-run recovery after losing issue-session lease");
|
|
1404
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1405
|
+
return;
|
|
697
1406
|
}
|
|
698
1407
|
const recoveredState = resolveRecoverablePostRunState(this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
|
|
699
1408
|
this.failRunAndClear(run, "Codex turn was interrupted", recoveredState);
|
|
1409
|
+
await this.restoreIdleWorktree(issue);
|
|
700
1410
|
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
701
1411
|
if (recoveredState) {
|
|
702
1412
|
this.feed?.publish({
|
|
@@ -713,12 +1423,13 @@ export class RunOrchestrator {
|
|
|
713
1423
|
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, "The Codex turn was interrupted."));
|
|
714
1424
|
}
|
|
715
1425
|
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1426
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
716
1427
|
return;
|
|
717
1428
|
}
|
|
718
1429
|
// Handle completed turn discovered during reconciliation
|
|
719
1430
|
if (latestTurn?.status === "completed") {
|
|
720
1431
|
const trackedIssue = this.db.issueToTrackedIssue(issue);
|
|
721
|
-
const report = buildStageReport(run, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
|
|
1432
|
+
const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
|
|
722
1433
|
const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
723
1434
|
const verifiedRepairError = await this.verifyReactiveRunAdvancedBranch(run, freshIssue);
|
|
724
1435
|
if (verifiedRepairError) {
|
|
@@ -733,10 +1444,34 @@ export class RunOrchestrator {
|
|
|
733
1444
|
status: "branch_not_advanced",
|
|
734
1445
|
summary: verifiedRepairError,
|
|
735
1446
|
});
|
|
1447
|
+
const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1448
|
+
void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
|
|
1449
|
+
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
1450
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1451
|
+
return;
|
|
1452
|
+
}
|
|
1453
|
+
const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
|
|
1454
|
+
if (publishedOutcomeError) {
|
|
1455
|
+
this.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
1456
|
+
this.feed?.publish({
|
|
1457
|
+
level: "warn",
|
|
1458
|
+
kind: "turn",
|
|
1459
|
+
issueKey: issue.issueKey,
|
|
1460
|
+
projectId: run.projectId,
|
|
1461
|
+
stage: run.runType,
|
|
1462
|
+
status: "publish_incomplete",
|
|
1463
|
+
summary: publishedOutcomeError,
|
|
1464
|
+
});
|
|
1465
|
+
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1466
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
1467
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1468
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
736
1469
|
return;
|
|
737
1470
|
}
|
|
738
|
-
const
|
|
739
|
-
this.
|
|
1471
|
+
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
1472
|
+
const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
|
|
1473
|
+
const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
|
|
1474
|
+
const reconciled = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
740
1475
|
this.db.finishRun(run.id, {
|
|
741
1476
|
status: "completed",
|
|
742
1477
|
...(run.threadId ? { threadId: run.threadId } : {}),
|
|
@@ -748,9 +1483,10 @@ export class RunOrchestrator {
|
|
|
748
1483
|
projectId: run.projectId,
|
|
749
1484
|
linearIssueId: run.linearIssueId,
|
|
750
1485
|
activeRunId: null,
|
|
751
|
-
...(postRunState === "awaiting_queue" ? { queueLabelApplied: false } : {}),
|
|
752
1486
|
...(postRunState ? { factoryState: postRunState } : {}),
|
|
753
|
-
|
|
1487
|
+
pendingRunType: null,
|
|
1488
|
+
pendingRunContextJson: null,
|
|
1489
|
+
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
754
1490
|
? {
|
|
755
1491
|
lastGitHubFailureSource: null,
|
|
756
1492
|
lastGitHubFailureHeadSha: null,
|
|
@@ -763,12 +1499,30 @@ export class RunOrchestrator {
|
|
|
763
1499
|
lastAttemptedFailureHeadSha: null,
|
|
764
1500
|
lastAttemptedFailureSignature: null,
|
|
765
1501
|
}
|
|
766
|
-
: {}),
|
|
1502
|
+
: {})),
|
|
767
1503
|
});
|
|
768
|
-
if (
|
|
769
|
-
this.
|
|
1504
|
+
if (postRunFollowUp) {
|
|
1505
|
+
return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
770
1506
|
}
|
|
1507
|
+
return true;
|
|
771
1508
|
});
|
|
1509
|
+
if (!reconciled) {
|
|
1510
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping reconciled completion writes after losing issue-session lease");
|
|
1511
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1512
|
+
return;
|
|
1513
|
+
}
|
|
1514
|
+
if (postRunFollowUp) {
|
|
1515
|
+
this.feed?.publish({
|
|
1516
|
+
level: "info",
|
|
1517
|
+
kind: "stage",
|
|
1518
|
+
issueKey: issue.issueKey,
|
|
1519
|
+
projectId: run.projectId,
|
|
1520
|
+
stage: postRunFollowUp.factoryState,
|
|
1521
|
+
status: "follow_up_queued",
|
|
1522
|
+
summary: postRunFollowUp.summary,
|
|
1523
|
+
});
|
|
1524
|
+
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
1525
|
+
}
|
|
772
1526
|
if (postRunState) {
|
|
773
1527
|
this.feed?.publish({
|
|
774
1528
|
level: "info",
|
|
@@ -780,25 +1534,43 @@ export class RunOrchestrator {
|
|
|
780
1534
|
summary: `Reconciliation: ${run.runType} completed \u2192 ${postRunState}`,
|
|
781
1535
|
});
|
|
782
1536
|
}
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
1537
|
+
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
1538
|
+
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
1539
|
+
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
1540
|
+
runType: run.runType,
|
|
1541
|
+
completionSummary,
|
|
1542
|
+
postRunState: updatedIssue.factoryState,
|
|
1543
|
+
...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
|
|
1544
|
+
}));
|
|
1545
|
+
void this.linearSync.syncSession(updatedIssue);
|
|
1546
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1547
|
+
return;
|
|
786
1548
|
}
|
|
1549
|
+
if (acquiredRecoveryLease)
|
|
1550
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
787
1551
|
}
|
|
788
1552
|
// ─── Internal helpers ─────────────────────────────────────────────
|
|
789
1553
|
escalate(issue, runType, reason) {
|
|
790
1554
|
this.logger.warn({ issueKey: issue.issueKey, runType, reason }, "Escalating to human");
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
1555
|
+
const escalated = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
1556
|
+
if (issue.activeRunId) {
|
|
1557
|
+
this.db.finishRunWithLease(lease, issue.activeRunId, { status: "released" });
|
|
1558
|
+
}
|
|
1559
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1560
|
+
projectId: issue.projectId,
|
|
1561
|
+
linearIssueId: issue.linearIssueId,
|
|
1562
|
+
pendingRunType: null,
|
|
1563
|
+
pendingRunContextJson: null,
|
|
1564
|
+
activeRunId: null,
|
|
1565
|
+
factoryState: "escalated",
|
|
1566
|
+
});
|
|
1567
|
+
return true;
|
|
801
1568
|
});
|
|
1569
|
+
if (!escalated) {
|
|
1570
|
+
this.logger.warn({ issueKey: issue.issueKey, runType }, "Skipping escalation write after losing issue-session lease");
|
|
1571
|
+
this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
1572
|
+
return;
|
|
1573
|
+
}
|
|
802
1574
|
this.feed?.publish({
|
|
803
1575
|
level: "error",
|
|
804
1576
|
kind: "workflow",
|
|
@@ -814,23 +1586,10 @@ export class RunOrchestrator {
|
|
|
814
1586
|
body: `PatchRelay needs human help to continue.\n\n${reason}`,
|
|
815
1587
|
});
|
|
816
1588
|
void this.linearSync.syncSession(escalatedIssue);
|
|
817
|
-
|
|
818
|
-
/** Add the merge queue admission label for external-queue projects (best-effort). */
|
|
819
|
-
async requestMergeQueueAdmission(issue, projectId) {
|
|
820
|
-
const project = this.config.projects.find((p) => p.id === projectId);
|
|
821
|
-
const protocol = resolveMergeQueueProtocol(project);
|
|
822
|
-
const applied = await requestMergeQueueAdmission({
|
|
823
|
-
issue,
|
|
824
|
-
protocol,
|
|
825
|
-
logger: this.logger,
|
|
826
|
-
feed: this.feed,
|
|
827
|
-
});
|
|
828
|
-
if (applied) {
|
|
829
|
-
this.db.upsertIssue({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueLabelApplied: true });
|
|
830
|
-
}
|
|
1589
|
+
this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
831
1590
|
}
|
|
832
1591
|
failRunAndClear(run, message, nextState = "failed") {
|
|
833
|
-
this.
|
|
1592
|
+
const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
834
1593
|
this.db.finishRun(run.id, { status: "failed", failureReason: message });
|
|
835
1594
|
this.db.upsertIssue({
|
|
836
1595
|
projectId: run.projectId,
|
|
@@ -840,9 +1599,17 @@ export class RunOrchestrator {
|
|
|
840
1599
|
});
|
|
841
1600
|
const branchOwner = this.resolveBranchOwnerForStateTransition(nextState);
|
|
842
1601
|
if (branchOwner) {
|
|
843
|
-
this.
|
|
1602
|
+
const lease = this.getHeldIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1603
|
+
if (lease) {
|
|
1604
|
+
this.db.setBranchOwnerWithLease(lease, branchOwner);
|
|
1605
|
+
}
|
|
844
1606
|
}
|
|
1607
|
+
return true;
|
|
845
1608
|
});
|
|
1609
|
+
if (!updated) {
|
|
1610
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping failure cleanup after losing issue-session lease");
|
|
1611
|
+
}
|
|
1612
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
846
1613
|
}
|
|
847
1614
|
resolveBranchOwnerForStateTransition(newState, pendingRunType) {
|
|
848
1615
|
return resolveBranchOwnerForStateTransition(newState, pendingRunType);
|
|
@@ -859,15 +1626,8 @@ export class RunOrchestrator {
|
|
|
859
1626
|
return undefined;
|
|
860
1627
|
}
|
|
861
1628
|
try {
|
|
862
|
-
const
|
|
863
|
-
|
|
864
|
-
"--repo", project.github.repoFullName,
|
|
865
|
-
"--json", "headRefOid,state",
|
|
866
|
-
], { timeoutMs: 10_000 });
|
|
867
|
-
if (exitCode !== 0)
|
|
868
|
-
return undefined;
|
|
869
|
-
const pr = JSON.parse(stdout);
|
|
870
|
-
if (pr.state?.toUpperCase() !== "OPEN")
|
|
1629
|
+
const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
|
|
1630
|
+
if (!pr || pr.state?.toUpperCase() !== "OPEN")
|
|
871
1631
|
return undefined;
|
|
872
1632
|
if (!pr.headRefOid || pr.headRefOid !== issue.lastGitHubFailureHeadSha)
|
|
873
1633
|
return undefined;
|
|
@@ -882,6 +1642,271 @@ export class RunOrchestrator {
|
|
|
882
1642
|
return undefined;
|
|
883
1643
|
}
|
|
884
1644
|
}
|
|
1645
|
+
async refreshIssueAfterReactivePublish(run, issue) {
|
|
1646
|
+
if (run.runType !== "ci_repair" && run.runType !== "queue_repair") {
|
|
1647
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1648
|
+
}
|
|
1649
|
+
if (!issue.prNumber) {
|
|
1650
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1651
|
+
}
|
|
1652
|
+
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1653
|
+
const repoFullName = project?.github?.repoFullName;
|
|
1654
|
+
if (!repoFullName) {
|
|
1655
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1656
|
+
}
|
|
1657
|
+
try {
|
|
1658
|
+
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1659
|
+
if (!pr) {
|
|
1660
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1661
|
+
}
|
|
1662
|
+
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1663
|
+
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1664
|
+
const gateCheckName = project?.gateChecks?.find((entry) => entry.trim())?.trim() ?? "verify";
|
|
1665
|
+
const headAdvanced = Boolean(pr.headRefOid && pr.headRefOid !== issue.lastGitHubFailureHeadSha);
|
|
1666
|
+
this.upsertIssueIfLeaseHeld(run.projectId, run.linearIssueId, {
|
|
1667
|
+
projectId: run.projectId,
|
|
1668
|
+
linearIssueId: run.linearIssueId,
|
|
1669
|
+
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1670
|
+
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1671
|
+
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1672
|
+
...(headAdvanced
|
|
1673
|
+
? {
|
|
1674
|
+
prCheckStatus: "pending",
|
|
1675
|
+
lastGitHubFailureSource: null,
|
|
1676
|
+
lastGitHubFailureHeadSha: null,
|
|
1677
|
+
lastGitHubFailureSignature: null,
|
|
1678
|
+
lastGitHubFailureCheckName: null,
|
|
1679
|
+
lastGitHubFailureCheckUrl: null,
|
|
1680
|
+
lastGitHubFailureContextJson: null,
|
|
1681
|
+
lastGitHubFailureAt: null,
|
|
1682
|
+
lastQueueIncidentJson: null,
|
|
1683
|
+
lastAttemptedFailureHeadSha: null,
|
|
1684
|
+
lastAttemptedFailureSignature: null,
|
|
1685
|
+
lastGitHubCiSnapshotHeadSha: pr.headRefOid ?? null,
|
|
1686
|
+
lastGitHubCiSnapshotGateCheckName: gateCheckName,
|
|
1687
|
+
lastGitHubCiSnapshotGateCheckStatus: "pending",
|
|
1688
|
+
lastGitHubCiSnapshotJson: null,
|
|
1689
|
+
lastGitHubCiSnapshotSettledAt: null,
|
|
1690
|
+
}
|
|
1691
|
+
: {}),
|
|
1692
|
+
}, "reactive publish refresh");
|
|
1693
|
+
}
|
|
1694
|
+
catch (error) {
|
|
1695
|
+
this.logger.debug({
|
|
1696
|
+
issueKey: issue.issueKey,
|
|
1697
|
+
prNumber: issue.prNumber,
|
|
1698
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1699
|
+
}, "Failed to refresh PR state after reactive publish");
|
|
1700
|
+
}
|
|
1701
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1702
|
+
}
|
|
1703
|
+
async loadRemotePrState(repoFullName, prNumber) {
|
|
1704
|
+
const { stdout, exitCode } = await execCommand("gh", [
|
|
1705
|
+
"pr", "view", String(prNumber),
|
|
1706
|
+
"--repo", repoFullName,
|
|
1707
|
+
"--json", "headRefOid,state,reviewDecision,mergeStateStatus",
|
|
1708
|
+
], { timeoutMs: 10_000 });
|
|
1709
|
+
if (exitCode !== 0)
|
|
1710
|
+
return undefined;
|
|
1711
|
+
return JSON.parse(stdout);
|
|
1712
|
+
}
|
|
1713
|
+
async resolveReviewFixWakeContext(issue, context, project) {
|
|
1714
|
+
if (isBranchUpkeepRequired(context)) {
|
|
1715
|
+
return context;
|
|
1716
|
+
}
|
|
1717
|
+
if (!issue.prNumber || issue.prState !== "open" || issue.prReviewState !== "changes_requested") {
|
|
1718
|
+
return context;
|
|
1719
|
+
}
|
|
1720
|
+
const repoFullName = project.github?.repoFullName;
|
|
1721
|
+
if (!repoFullName) {
|
|
1722
|
+
return context;
|
|
1723
|
+
}
|
|
1724
|
+
try {
|
|
1725
|
+
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1726
|
+
if (!pr)
|
|
1727
|
+
return context;
|
|
1728
|
+
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1729
|
+
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1730
|
+
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1731
|
+
projectId: issue.projectId,
|
|
1732
|
+
linearIssueId: issue.linearIssueId,
|
|
1733
|
+
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1734
|
+
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1735
|
+
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1736
|
+
}, "review-fix wake refresh");
|
|
1737
|
+
if (nextPrState !== "open")
|
|
1738
|
+
return context;
|
|
1739
|
+
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
1740
|
+
return context;
|
|
1741
|
+
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
1742
|
+
return context;
|
|
1743
|
+
return buildReviewFixBranchUpkeepContext(issue.prNumber, project.github?.baseBranch ?? "main", pr, context);
|
|
1744
|
+
}
|
|
1745
|
+
catch (error) {
|
|
1746
|
+
this.logger.debug({
|
|
1747
|
+
issueKey: issue.issueKey,
|
|
1748
|
+
prNumber: issue.prNumber,
|
|
1749
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1750
|
+
}, "Failed to resolve review-fix wake context");
|
|
1751
|
+
return context;
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
async resolvePostRunFollowUp(run, issue, projectOverride) {
|
|
1755
|
+
if (run.runType !== "review_fix") {
|
|
1756
|
+
return undefined;
|
|
1757
|
+
}
|
|
1758
|
+
if (!issue.prNumber || issue.prState !== "open") {
|
|
1759
|
+
return undefined;
|
|
1760
|
+
}
|
|
1761
|
+
if (issue.prReviewState !== "changes_requested") {
|
|
1762
|
+
return undefined;
|
|
1763
|
+
}
|
|
1764
|
+
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1765
|
+
const repoFullName = project?.github?.repoFullName;
|
|
1766
|
+
if (!repoFullName) {
|
|
1767
|
+
return undefined;
|
|
1768
|
+
}
|
|
1769
|
+
try {
|
|
1770
|
+
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1771
|
+
if (!pr)
|
|
1772
|
+
return undefined;
|
|
1773
|
+
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1774
|
+
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1775
|
+
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1776
|
+
projectId: issue.projectId,
|
|
1777
|
+
linearIssueId: issue.linearIssueId,
|
|
1778
|
+
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1779
|
+
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1780
|
+
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1781
|
+
}, "post-run follow-up refresh");
|
|
1782
|
+
if (nextPrState !== "open")
|
|
1783
|
+
return undefined;
|
|
1784
|
+
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
1785
|
+
return undefined;
|
|
1786
|
+
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
1787
|
+
return undefined;
|
|
1788
|
+
return {
|
|
1789
|
+
pendingRunType: "review_fix",
|
|
1790
|
+
factoryState: "changes_requested",
|
|
1791
|
+
context: buildReviewFixBranchUpkeepContext(issue.prNumber, project?.github?.baseBranch ?? "main", pr),
|
|
1792
|
+
summary: `PR #${issue.prNumber} is still dirty after review fix; queued branch upkeep`,
|
|
1793
|
+
};
|
|
1794
|
+
}
|
|
1795
|
+
catch (error) {
|
|
1796
|
+
this.logger.debug({
|
|
1797
|
+
issueKey: issue.issueKey,
|
|
1798
|
+
prNumber: issue.prNumber,
|
|
1799
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1800
|
+
}, "Failed to resolve post-run PR upkeep");
|
|
1801
|
+
return undefined;
|
|
1802
|
+
}
|
|
1803
|
+
}
|
|
1804
|
+
async verifyPublishedRunOutcome(run, issue, projectOverride) {
|
|
1805
|
+
if (run.runType !== "implementation") {
|
|
1806
|
+
return undefined;
|
|
1807
|
+
}
|
|
1808
|
+
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1809
|
+
const baseBranch = project?.github?.baseBranch ?? "main";
|
|
1810
|
+
const deliveryMode = resolveImplementationDeliveryMode(issue, undefined, run.promptText);
|
|
1811
|
+
if (deliveryMode === "linear_only") {
|
|
1812
|
+
if (issue.prNumber !== undefined) {
|
|
1813
|
+
return `Planning-only implementation should not open a PR, but PR #${issue.prNumber} was observed`;
|
|
1814
|
+
}
|
|
1815
|
+
return this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
1816
|
+
}
|
|
1817
|
+
if (issue.prNumber && issue.prState && issue.prState !== "closed") {
|
|
1818
|
+
return undefined;
|
|
1819
|
+
}
|
|
1820
|
+
if (project?.github?.repoFullName && issue.branchName) {
|
|
1821
|
+
try {
|
|
1822
|
+
const { stdout, exitCode } = await execCommand("gh", [
|
|
1823
|
+
"pr",
|
|
1824
|
+
"list",
|
|
1825
|
+
"--repo",
|
|
1826
|
+
project.github.repoFullName,
|
|
1827
|
+
"--head",
|
|
1828
|
+
issue.branchName,
|
|
1829
|
+
"--state",
|
|
1830
|
+
"all",
|
|
1831
|
+
"--json",
|
|
1832
|
+
"number,url,state,author,headRefOid",
|
|
1833
|
+
], { timeoutMs: 10_000 });
|
|
1834
|
+
if (exitCode === 0) {
|
|
1835
|
+
const matches = JSON.parse(stdout);
|
|
1836
|
+
const pr = matches[0];
|
|
1837
|
+
if (pr?.number) {
|
|
1838
|
+
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1839
|
+
projectId: issue.projectId,
|
|
1840
|
+
linearIssueId: issue.linearIssueId,
|
|
1841
|
+
prNumber: pr.number,
|
|
1842
|
+
...(pr.url ? { prUrl: pr.url } : {}),
|
|
1843
|
+
...(pr.state ? { prState: pr.state.toLowerCase() } : {}),
|
|
1844
|
+
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1845
|
+
...(pr.author?.login ? { prAuthorLogin: pr.author.login } : {}),
|
|
1846
|
+
}, "published PR verification refresh");
|
|
1847
|
+
return undefined;
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
catch (error) {
|
|
1852
|
+
this.logger.debug({
|
|
1853
|
+
issueKey: issue.issueKey,
|
|
1854
|
+
branchName: issue.branchName,
|
|
1855
|
+
repoFullName: project.github.repoFullName,
|
|
1856
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1857
|
+
}, "Failed to verify published PR state after implementation");
|
|
1858
|
+
}
|
|
1859
|
+
}
|
|
1860
|
+
const details = await this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
1861
|
+
return details ?? `Implementation completed without opening a PR for branch ${issue.branchName ?? issue.linearIssueId}`;
|
|
1862
|
+
}
|
|
1863
|
+
async describeLocalImplementationOutcome(issue, baseBranch, deliveryMode = "publish_pr") {
|
|
1864
|
+
if (!issue.worktreePath) {
|
|
1865
|
+
return undefined;
|
|
1866
|
+
}
|
|
1867
|
+
try {
|
|
1868
|
+
const status = await execCommand(this.config.runner.gitBin, [
|
|
1869
|
+
"-C",
|
|
1870
|
+
issue.worktreePath,
|
|
1871
|
+
"status",
|
|
1872
|
+
"--short",
|
|
1873
|
+
], { timeoutMs: 10_000 });
|
|
1874
|
+
const dirtyEntries = status.exitCode === 0
|
|
1875
|
+
? status.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
1876
|
+
: [];
|
|
1877
|
+
if (dirtyEntries.length > 0) {
|
|
1878
|
+
if (deliveryMode === "linear_only") {
|
|
1879
|
+
return `Planning-only implementation should not modify the repo; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1880
|
+
}
|
|
1881
|
+
return `Implementation completed without opening a PR; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1882
|
+
}
|
|
1883
|
+
}
|
|
1884
|
+
catch {
|
|
1885
|
+
// Best effort only.
|
|
1886
|
+
}
|
|
1887
|
+
try {
|
|
1888
|
+
const ahead = await execCommand(this.config.runner.gitBin, [
|
|
1889
|
+
"-C",
|
|
1890
|
+
issue.worktreePath,
|
|
1891
|
+
"rev-list",
|
|
1892
|
+
"--count",
|
|
1893
|
+
`origin/${baseBranch}..HEAD`,
|
|
1894
|
+
], { timeoutMs: 10_000 });
|
|
1895
|
+
if (ahead.exitCode === 0) {
|
|
1896
|
+
const count = Number(ahead.stdout.trim());
|
|
1897
|
+
if (Number.isFinite(count) && count > 0) {
|
|
1898
|
+
if (deliveryMode === "linear_only") {
|
|
1899
|
+
return `Planning-only implementation should not create repo commits; worktree is ${count} local commit(s) ahead of origin/${baseBranch}`;
|
|
1900
|
+
}
|
|
1901
|
+
return `Implementation completed with ${count} local commit(s) ahead of origin/${baseBranch} but no PR was observed`;
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
}
|
|
1905
|
+
catch {
|
|
1906
|
+
// Best effort only.
|
|
1907
|
+
}
|
|
1908
|
+
return undefined;
|
|
1909
|
+
}
|
|
885
1910
|
async readThreadWithRetry(threadId, maxRetries = 3) {
|
|
886
1911
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
887
1912
|
try {
|
|
@@ -895,6 +1920,151 @@ export class RunOrchestrator {
|
|
|
895
1920
|
}
|
|
896
1921
|
throw new Error(`Failed to read thread ${threadId}`);
|
|
897
1922
|
}
|
|
1923
|
+
issueSessionLeaseKey(projectId, linearIssueId) {
|
|
1924
|
+
return `${projectId}:${linearIssueId}`;
|
|
1925
|
+
}
|
|
1926
|
+
getHeldIssueSessionLease(projectId, linearIssueId) {
|
|
1927
|
+
const leaseId = this.activeSessionLeases.get(this.issueSessionLeaseKey(projectId, linearIssueId));
|
|
1928
|
+
if (!leaseId)
|
|
1929
|
+
return undefined;
|
|
1930
|
+
return { projectId, linearIssueId, leaseId };
|
|
1931
|
+
}
|
|
1932
|
+
withHeldIssueSessionLease(projectId, linearIssueId, fn) {
|
|
1933
|
+
const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
|
|
1934
|
+
if (!lease)
|
|
1935
|
+
return undefined;
|
|
1936
|
+
return this.db.withIssueSessionLease(projectId, linearIssueId, lease.leaseId, () => fn(lease));
|
|
1937
|
+
}
|
|
1938
|
+
upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
|
|
1939
|
+
const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
|
|
1940
|
+
if (!lease) {
|
|
1941
|
+
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write without a held issue-session lease");
|
|
1942
|
+
return undefined;
|
|
1943
|
+
}
|
|
1944
|
+
const updated = this.db.upsertIssueWithLease(lease, params);
|
|
1945
|
+
if (!updated) {
|
|
1946
|
+
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
|
|
1947
|
+
}
|
|
1948
|
+
return updated;
|
|
1949
|
+
}
|
|
1950
|
+
assertLaunchLease(run, phase) {
|
|
1951
|
+
if (this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
1952
|
+
return;
|
|
1953
|
+
}
|
|
1954
|
+
const error = new Error(`Lost issue-session lease ${phase}`);
|
|
1955
|
+
error.name = "IssueSessionLeaseLostError";
|
|
1956
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId, phase }, "Aborting run launch after losing issue-session lease");
|
|
1957
|
+
throw error;
|
|
1958
|
+
}
|
|
1959
|
+
acquireIssueSessionLease(projectId, linearIssueId) {
|
|
1960
|
+
const leaseId = randomUUID();
|
|
1961
|
+
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
1962
|
+
const acquired = this.db.acquireIssueSessionLease({
|
|
1963
|
+
projectId,
|
|
1964
|
+
linearIssueId,
|
|
1965
|
+
leaseId,
|
|
1966
|
+
workerId: this.workerId,
|
|
1967
|
+
leasedUntil,
|
|
1968
|
+
});
|
|
1969
|
+
if (!acquired)
|
|
1970
|
+
return undefined;
|
|
1971
|
+
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
1972
|
+
return leaseId;
|
|
1973
|
+
}
|
|
1974
|
+
forceAcquireIssueSessionLease(projectId, linearIssueId) {
|
|
1975
|
+
const leaseId = randomUUID();
|
|
1976
|
+
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
1977
|
+
const acquired = this.db.forceAcquireIssueSessionLease({
|
|
1978
|
+
projectId,
|
|
1979
|
+
linearIssueId,
|
|
1980
|
+
leaseId,
|
|
1981
|
+
workerId: this.workerId,
|
|
1982
|
+
leasedUntil,
|
|
1983
|
+
});
|
|
1984
|
+
if (!acquired)
|
|
1985
|
+
return undefined;
|
|
1986
|
+
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
1987
|
+
return leaseId;
|
|
1988
|
+
}
|
|
1989
|
+
claimLeaseForReconciliation(projectId, linearIssueId) {
|
|
1990
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
1991
|
+
if (this.activeSessionLeases.has(key)) {
|
|
1992
|
+
return "owned";
|
|
1993
|
+
}
|
|
1994
|
+
const session = this.db.getIssueSession(projectId, linearIssueId);
|
|
1995
|
+
if (!session)
|
|
1996
|
+
return "skip";
|
|
1997
|
+
const leasedUntilMs = session.leasedUntil ? Date.parse(session.leasedUntil) : undefined;
|
|
1998
|
+
if (leasedUntilMs !== undefined && Number.isFinite(leasedUntilMs) && leasedUntilMs > Date.now()) {
|
|
1999
|
+
return "skip";
|
|
2000
|
+
}
|
|
2001
|
+
return this.acquireIssueSessionLease(projectId, linearIssueId) ? true : "skip";
|
|
2002
|
+
}
|
|
2003
|
+
async reclaimForeignRecoveryLeaseIfSafe(run, issue) {
|
|
2004
|
+
const key = this.issueSessionLeaseKey(run.projectId, run.linearIssueId);
|
|
2005
|
+
if (this.activeSessionLeases.has(key)) {
|
|
2006
|
+
return false;
|
|
2007
|
+
}
|
|
2008
|
+
const session = this.db.getIssueSession(run.projectId, run.linearIssueId);
|
|
2009
|
+
if (!session?.leaseId || !session.workerId || session.workerId === this.workerId) {
|
|
2010
|
+
return false;
|
|
2011
|
+
}
|
|
2012
|
+
if (issue.activeRunId !== run.id) {
|
|
2013
|
+
return false;
|
|
2014
|
+
}
|
|
2015
|
+
let safeToReclaim = !run.threadId;
|
|
2016
|
+
if (!safeToReclaim && run.threadId) {
|
|
2017
|
+
try {
|
|
2018
|
+
const thread = await this.readThreadWithRetry(run.threadId, 1);
|
|
2019
|
+
const latestTurn = getThreadTurns(thread).at(-1);
|
|
2020
|
+
safeToReclaim = thread.status === "notLoaded"
|
|
2021
|
+
|| latestTurn?.status === "interrupted"
|
|
2022
|
+
|| latestTurn?.status === "completed";
|
|
2023
|
+
}
|
|
2024
|
+
catch {
|
|
2025
|
+
safeToReclaim = true;
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
if (!safeToReclaim) {
|
|
2029
|
+
return false;
|
|
2030
|
+
}
|
|
2031
|
+
const leaseId = this.forceAcquireIssueSessionLease(run.projectId, run.linearIssueId);
|
|
2032
|
+
if (!leaseId) {
|
|
2033
|
+
return false;
|
|
2034
|
+
}
|
|
2035
|
+
this.logger.info({
|
|
2036
|
+
issueKey: issue.issueKey,
|
|
2037
|
+
runId: run.id,
|
|
2038
|
+
previousWorkerId: session.workerId,
|
|
2039
|
+
previousLeaseId: session.leaseId,
|
|
2040
|
+
reclaimedLeaseId: leaseId,
|
|
2041
|
+
}, "Reclaimed foreign issue-session lease for active-run recovery");
|
|
2042
|
+
return true;
|
|
2043
|
+
}
|
|
2044
|
+
heartbeatIssueSessionLease(projectId, linearIssueId) {
|
|
2045
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
2046
|
+
const leaseId = this.activeSessionLeases.get(key) ?? this.db.getIssueSession(projectId, linearIssueId)?.leaseId;
|
|
2047
|
+
if (!leaseId)
|
|
2048
|
+
return false;
|
|
2049
|
+
const renewed = this.db.renewIssueSessionLease({
|
|
2050
|
+
projectId,
|
|
2051
|
+
linearIssueId,
|
|
2052
|
+
leaseId,
|
|
2053
|
+
leasedUntil: new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString(),
|
|
2054
|
+
});
|
|
2055
|
+
if (renewed) {
|
|
2056
|
+
this.activeSessionLeases.set(key, leaseId);
|
|
2057
|
+
return true;
|
|
2058
|
+
}
|
|
2059
|
+
this.activeSessionLeases.delete(key);
|
|
2060
|
+
return false;
|
|
2061
|
+
}
|
|
2062
|
+
releaseIssueSessionLease(projectId, linearIssueId) {
|
|
2063
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
2064
|
+
const leaseId = this.activeSessionLeases.get(key);
|
|
2065
|
+
this.db.releaseIssueSessionLease(projectId, linearIssueId, leaseId);
|
|
2066
|
+
this.activeSessionLeases.delete(key);
|
|
2067
|
+
}
|
|
898
2068
|
}
|
|
899
2069
|
/**
|
|
900
2070
|
* Determine post-run factory state from current PR metadata.
|
|
@@ -912,6 +2082,12 @@ function resolvePostRunState(issue) {
|
|
|
912
2082
|
}
|
|
913
2083
|
return undefined;
|
|
914
2084
|
}
|
|
2085
|
+
function resolveCompletedRunState(issue, run) {
|
|
2086
|
+
if (run.runType === "implementation" && resolveImplementationDeliveryMode(issue, undefined, run.promptText) === "linear_only") {
|
|
2087
|
+
return "done";
|
|
2088
|
+
}
|
|
2089
|
+
return resolvePostRunState(issue);
|
|
2090
|
+
}
|
|
915
2091
|
function resolveRecoverablePostRunState(issue) {
|
|
916
2092
|
if (!issue.prNumber) {
|
|
917
2093
|
return resolvePostRunState(issue);
|
|
@@ -919,18 +2095,59 @@ function resolveRecoverablePostRunState(issue) {
|
|
|
919
2095
|
if (issue.prState === "merged")
|
|
920
2096
|
return "done";
|
|
921
2097
|
if (issue.prState === "open") {
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
2098
|
+
const reactiveIntent = deriveIssueSessionReactiveIntent({
|
|
2099
|
+
prNumber: issue.prNumber,
|
|
2100
|
+
prState: issue.prState,
|
|
2101
|
+
prReviewState: issue.prReviewState,
|
|
2102
|
+
prCheckStatus: issue.prCheckStatus,
|
|
2103
|
+
latestFailureSource: issue.lastGitHubFailureSource,
|
|
2104
|
+
});
|
|
2105
|
+
if (reactiveIntent)
|
|
2106
|
+
return reactiveIntent.compatibilityFactoryState;
|
|
928
2107
|
if (issue.prReviewState === "approved")
|
|
929
2108
|
return "awaiting_queue";
|
|
930
2109
|
return "pr_open";
|
|
931
2110
|
}
|
|
932
2111
|
return resolvePostRunState(issue);
|
|
933
2112
|
}
|
|
2113
|
+
function normalizeRemotePrState(value) {
|
|
2114
|
+
const normalized = value?.trim().toUpperCase();
|
|
2115
|
+
if (normalized === "OPEN")
|
|
2116
|
+
return "open";
|
|
2117
|
+
if (normalized === "CLOSED")
|
|
2118
|
+
return "closed";
|
|
2119
|
+
if (normalized === "MERGED")
|
|
2120
|
+
return "merged";
|
|
2121
|
+
return undefined;
|
|
2122
|
+
}
|
|
2123
|
+
function normalizeRemoteReviewDecision(value) {
|
|
2124
|
+
const normalized = value?.trim().toUpperCase();
|
|
2125
|
+
if (normalized === "APPROVED")
|
|
2126
|
+
return "approved";
|
|
2127
|
+
if (normalized === "CHANGES_REQUESTED")
|
|
2128
|
+
return "changes_requested";
|
|
2129
|
+
if (normalized === "REVIEW_REQUIRED")
|
|
2130
|
+
return "commented";
|
|
2131
|
+
return undefined;
|
|
2132
|
+
}
|
|
2133
|
+
function isDirtyMergeStateStatus(value) {
|
|
2134
|
+
return value?.trim().toUpperCase() === "DIRTY";
|
|
2135
|
+
}
|
|
2136
|
+
function buildReviewFixBranchUpkeepContext(prNumber, baseBranch, pr, context) {
|
|
2137
|
+
const promptContext = [
|
|
2138
|
+
`The requested review change is already addressed, but GitHub still reports PR #${prNumber} as ${String(pr.mergeStateStatus)} against latest ${baseBranch}.`,
|
|
2139
|
+
`Before stopping, update the existing PR branch onto latest ${baseBranch}, resolve any conflicts, rerun the narrowest relevant verification, and push again.`,
|
|
2140
|
+
"Do not stop just because the requested code change is already present.",
|
|
2141
|
+
].join(" ");
|
|
2142
|
+
return {
|
|
2143
|
+
...(context ?? {}),
|
|
2144
|
+
branchUpkeepRequired: true,
|
|
2145
|
+
promptContext,
|
|
2146
|
+
...(pr.mergeStateStatus ? { mergeStateStatus: pr.mergeStateStatus } : {}),
|
|
2147
|
+
...(pr.headRefOid ? { failingHeadSha: pr.headRefOid } : {}),
|
|
2148
|
+
baseBranch,
|
|
2149
|
+
};
|
|
2150
|
+
}
|
|
934
2151
|
function appendQueueRepairContext(lines, context) {
|
|
935
2152
|
const incidentTitle = typeof context?.incidentTitle === "string" ? context.incidentTitle.trim() : "";
|
|
936
2153
|
const incidentSummary = typeof context?.incidentSummary === "string" ? context.incidentSummary.trim() : "";
|