patchrelay 0.35.11 → 0.35.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -9
- package/dist/build-info.json +3 -3
- package/dist/cli/args.js +0 -1
- package/dist/cli/commands/issues.js +2 -56
- package/dist/cli/commands/watch.js +5 -0
- package/dist/cli/data.js +110 -47
- package/dist/cli/formatters/text.js +6 -90
- package/dist/cli/help.js +3 -8
- package/dist/cli/index.js +0 -48
- package/dist/cli/operator-client.js +0 -82
- package/dist/cli/watch/App.js +1 -12
- package/dist/cli/watch/HelpBar.js +2 -2
- package/dist/cli/watch/IssueDetailView.js +57 -26
- package/dist/cli/watch/IssueRow.js +71 -27
- package/dist/cli/watch/StatusBar.js +7 -4
- package/dist/cli/watch/state-visualization.js +48 -23
- package/dist/cli/watch/timeline-builder.js +2 -1
- package/dist/cli/watch/use-detail-stream.js +10 -104
- package/dist/cli/watch/use-watch-stream.js +11 -102
- package/dist/cli/watch/watch-state.js +18 -50
- package/dist/codex-thread-utils.js +3 -0
- package/dist/db/migrations.js +239 -2
- package/dist/db.js +628 -39
- package/dist/github-app-token.js +7 -0
- package/dist/github-failure-context.js +44 -1
- package/dist/github-rollup.js +47 -0
- package/dist/github-webhook-handler.js +248 -51
- package/dist/github-webhooks.js +5 -0
- package/dist/http.js +12 -264
- package/dist/idle-reconciliation.js +268 -76
- package/dist/issue-query-service.js +221 -129
- package/dist/issue-session-events.js +151 -0
- package/dist/issue-session.js +99 -0
- package/dist/linear-client.js +39 -25
- package/dist/linear-session-reporting.js +12 -0
- package/dist/linear-session-sync.js +253 -24
- package/dist/linear-workflow.js +33 -0
- package/dist/merge-queue-protocol.js +0 -51
- package/dist/preflight.js +1 -4
- package/dist/queue-health-monitor.js +11 -7
- package/dist/run-orchestrator.js +1295 -146
- package/dist/run-reporting.js +5 -3
- package/dist/service.js +279 -102
- package/dist/status-note.js +56 -0
- package/dist/waiting-reason.js +65 -0
- package/dist/webhook-handler.js +270 -79
- package/package.json +1 -1
- package/dist/cli/commands/feed.js +0 -60
- package/dist/cli/watch/FeedView.js +0 -28
- package/dist/cli/watch/use-feed-stream.js +0 -92
package/dist/run-orchestrator.js
CHANGED
|
@@ -1,18 +1,23 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
1
2
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
3
|
import path from "node:path";
|
|
3
4
|
import { ACTIVE_RUN_STATES, TERMINAL_STATES } from "./factory-state.js";
|
|
4
5
|
import { buildHookEnv, runProjectHook } from "./hook-runner.js";
|
|
5
6
|
import { buildStageReport, countEventMethods, extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread, } from "./run-reporting.js";
|
|
6
7
|
import { buildRunCompletedActivity, buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
7
|
-
import { requestMergeQueueAdmission, resolveMergeQueueProtocol, } from "./merge-queue-protocol.js";
|
|
8
8
|
import { WorktreeManager } from "./worktree-manager.js";
|
|
9
|
-
import { resolveAuthoritativeLinearStopState } from "./linear-workflow.js";
|
|
9
|
+
import { resolveAuthoritativeLinearStopState, resolvePreferredCompletedLinearState } from "./linear-workflow.js";
|
|
10
10
|
import { execCommand } from "./utils.js";
|
|
11
|
+
import { getThreadTurns } from "./codex-thread-utils.js";
|
|
12
|
+
import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
11
13
|
const DEFAULT_CI_REPAIR_BUDGET = 3;
|
|
12
14
|
const DEFAULT_QUEUE_REPAIR_BUDGET = 3;
|
|
13
15
|
const DEFAULT_REVIEW_FIX_BUDGET = 3;
|
|
14
16
|
const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
|
|
15
17
|
const ZOMBIE_RECOVERY_BASE_DELAY_MS = 15_000; // 15s, 30s, 60s, 120s, 240s
|
|
18
|
+
const ISSUE_SESSION_LEASE_MS = 10 * 60_000;
|
|
19
|
+
const MAX_THREAD_GENERATION_BEFORE_COMPACTION = 4;
|
|
20
|
+
const MAX_FOLLOW_UPS_BEFORE_COMPACTION = 4;
|
|
16
21
|
import { QueueHealthMonitor } from "./queue-health-monitor.js";
|
|
17
22
|
import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
|
|
18
23
|
import { LinearSessionSync } from "./linear-session-sync.js";
|
|
@@ -38,22 +43,279 @@ function readWorkflowFile(repoPath, runType) {
|
|
|
38
43
|
return undefined;
|
|
39
44
|
return readFileSync(filePath, "utf8").trim();
|
|
40
45
|
}
|
|
41
|
-
function
|
|
42
|
-
const
|
|
46
|
+
function collectImplementationInstructionText(issue, context, promptText) {
|
|
47
|
+
const parts = [];
|
|
48
|
+
if (issue.title)
|
|
49
|
+
parts.push(issue.title);
|
|
50
|
+
if (issue.description)
|
|
51
|
+
parts.push(issue.description);
|
|
52
|
+
if (promptText)
|
|
53
|
+
parts.push(promptText);
|
|
54
|
+
const stringFields = ["promptContext", "promptBody", "operatorPrompt", "userComment"];
|
|
55
|
+
for (const field of stringFields) {
|
|
56
|
+
const value = context?.[field];
|
|
57
|
+
if (typeof value === "string" && value.trim()) {
|
|
58
|
+
parts.push(value);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (Array.isArray(context?.followUps)) {
|
|
62
|
+
for (const entry of context.followUps) {
|
|
63
|
+
if (!entry || typeof entry !== "object")
|
|
64
|
+
continue;
|
|
65
|
+
const text = entry.text;
|
|
66
|
+
if (typeof text === "string" && text.trim()) {
|
|
67
|
+
parts.push(text);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return parts.join("\n").toLowerCase();
|
|
72
|
+
}
|
|
73
|
+
export function resolveImplementationDeliveryMode(issue, context, promptText) {
|
|
74
|
+
const instructionText = collectImplementationInstructionText(issue, context, promptText);
|
|
75
|
+
if (!instructionText)
|
|
76
|
+
return "publish_pr";
|
|
77
|
+
const hasExplicitNoPr = [
|
|
78
|
+
/\bdo not open (?:a |any )?pr\b/,
|
|
79
|
+
/\bdo not open (?:a |any )?pull request\b/,
|
|
80
|
+
/\bno pr is opened\b/,
|
|
81
|
+
/\bpatchrelay should not open a pr\b/,
|
|
82
|
+
/\bwithout opening a pr\b/,
|
|
83
|
+
].some((pattern) => pattern.test(instructionText));
|
|
84
|
+
const forbidsRepoChanges = [
|
|
85
|
+
/\bdo not make repository changes\b/,
|
|
86
|
+
/\bdo not make repo changes\b/,
|
|
87
|
+
/\bno repository changes\b/,
|
|
88
|
+
/\bno repo changes\b/,
|
|
89
|
+
/\bdo not modify repo files\b/,
|
|
90
|
+
].some((pattern) => pattern.test(instructionText));
|
|
91
|
+
const planningOnly = [
|
|
92
|
+
/\bplanning\/specification issue only\b/,
|
|
93
|
+
/\bplanning[- ]only\b/,
|
|
94
|
+
/\bspecification[- ]only\b/,
|
|
95
|
+
/\bplanning issue only\b/,
|
|
96
|
+
].some((pattern) => pattern.test(instructionText));
|
|
97
|
+
if (hasExplicitNoPr || (planningOnly && forbidsRepoChanges)) {
|
|
98
|
+
return "linear_only";
|
|
99
|
+
}
|
|
100
|
+
return "publish_pr";
|
|
101
|
+
}
|
|
102
|
+
function appendPublicationContract(lines, runType, issue, context) {
|
|
103
|
+
const deliveryMode = runType === "implementation" && issue
|
|
104
|
+
? resolveImplementationDeliveryMode(issue, context)
|
|
105
|
+
: "publish_pr";
|
|
106
|
+
if (runType === "implementation" && deliveryMode === "linear_only") {
|
|
107
|
+
lines.push("## Delivery Requirements", "");
|
|
108
|
+
lines.push("This issue is planning/specification only.", "Do not modify repo files or open a PR for this issue.", "Deliver the result through Linear artifacts such as follow-up issues, documents, and a concise summary.", "Leave the worktree clean before stopping.", "");
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
lines.push("## Publication Requirements", "");
|
|
112
|
+
if (runType === "implementation") {
|
|
113
|
+
lines.push("Before finishing, publish the result instead of leaving it only in the worktree.", "If the worktree already contains relevant changes for this issue, verify them and publish them.", "If you changed files for this issue, commit them, push the issue branch, and open or update the PR before stopping.", "Do not stop with only local commits or uncommitted changes.", "");
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
lines.push("Before finishing, publish the result to the existing PR branch.", "If you changed files for this repair, commit them and push the same branch before stopping.", "Do not open a new PR.", "Do not stop with only local commits or uncommitted changes.", "");
|
|
117
|
+
}
|
|
118
|
+
function buildPromptHeader(issue) {
|
|
119
|
+
return [
|
|
43
120
|
`Issue: ${issue.issueKey ?? issue.linearIssueId}`,
|
|
44
121
|
issue.title ? `Title: ${issue.title}` : undefined,
|
|
45
122
|
issue.branchName ? `Branch: ${issue.branchName}` : undefined,
|
|
46
123
|
issue.prNumber ? `PR: #${issue.prNumber}` : undefined,
|
|
47
124
|
"",
|
|
48
125
|
].filter(Boolean);
|
|
126
|
+
}
|
|
127
|
+
function appendTaskObjective(lines, issue) {
|
|
128
|
+
const description = issue.description?.trim();
|
|
129
|
+
lines.push("## Task Objective", "");
|
|
130
|
+
lines.push(issue.title || `Complete ${issue.issueKey ?? issue.linearIssueId}.`);
|
|
131
|
+
if (description) {
|
|
132
|
+
lines.push("", description);
|
|
133
|
+
}
|
|
134
|
+
lines.push("");
|
|
135
|
+
}
|
|
136
|
+
function appendLinearContext(lines, context) {
|
|
49
137
|
const promptContext = typeof context?.promptContext === "string" ? context.promptContext.trim() : "";
|
|
50
138
|
const latestPrompt = typeof context?.promptBody === "string" ? context.promptBody.trim() : "";
|
|
139
|
+
const operatorPrompt = typeof context?.operatorPrompt === "string" ? context.operatorPrompt.trim() : "";
|
|
140
|
+
const userComment = typeof context?.userComment === "string" ? context.userComment.trim() : "";
|
|
51
141
|
if (promptContext) {
|
|
52
142
|
lines.push("## Linear Session Context", "", promptContext, "");
|
|
53
143
|
}
|
|
54
144
|
if (latestPrompt) {
|
|
55
145
|
lines.push("## Latest Human Instruction", "", latestPrompt, "");
|
|
56
146
|
}
|
|
147
|
+
if (operatorPrompt) {
|
|
148
|
+
lines.push("## Operator Prompt", "", operatorPrompt, "");
|
|
149
|
+
}
|
|
150
|
+
if (userComment) {
|
|
151
|
+
lines.push("## Human Follow-up Comment", "", userComment, "");
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
function collectFollowUpInputs(context) {
|
|
155
|
+
const followUps = Array.isArray(context?.followUps) ? context.followUps : [];
|
|
156
|
+
const inputs = [];
|
|
157
|
+
for (const entry of followUps) {
|
|
158
|
+
const followUp = entry && typeof entry === "object" ? entry : undefined;
|
|
159
|
+
const type = typeof followUp?.type === "string" ? followUp.type : "followup";
|
|
160
|
+
const author = typeof followUp?.author === "string" ? followUp.author : undefined;
|
|
161
|
+
const text = typeof followUp?.text === "string" ? followUp.text.trim() : "";
|
|
162
|
+
if (!text)
|
|
163
|
+
continue;
|
|
164
|
+
inputs.push({ type, text, ...(author ? { author } : {}) });
|
|
165
|
+
}
|
|
166
|
+
return inputs;
|
|
167
|
+
}
|
|
168
|
+
function resolveFollowUpWhy(runType, context) {
|
|
169
|
+
const wakeReason = typeof context?.wakeReason === "string" ? context.wakeReason : undefined;
|
|
170
|
+
switch (wakeReason) {
|
|
171
|
+
case "direct_reply":
|
|
172
|
+
return "A human reply arrived for the outstanding question from the previous turn.";
|
|
173
|
+
case "followup_prompt":
|
|
174
|
+
return "A new Linear agent prompt arrived after the previous turn.";
|
|
175
|
+
case "followup_comment":
|
|
176
|
+
return "A human follow-up comment arrived after the previous turn.";
|
|
177
|
+
case "operator_prompt":
|
|
178
|
+
return "An operator supplied new guidance for this issue.";
|
|
179
|
+
case "review_changes_requested":
|
|
180
|
+
return "GitHub review requested changes on the current PR head.";
|
|
181
|
+
case "settled_red_ci":
|
|
182
|
+
return "Required CI settled red for the current PR head.";
|
|
183
|
+
case "merge_steward_incident":
|
|
184
|
+
return "Merge Steward reported an incident on the current PR head.";
|
|
185
|
+
case "delegated":
|
|
186
|
+
return runType === "implementation"
|
|
187
|
+
? "This is the first implementation turn for the delegated issue."
|
|
188
|
+
: `This turn continues ${runType.replaceAll("_", " ")} work for the delegated issue.`;
|
|
189
|
+
default:
|
|
190
|
+
if (runType === "review_fix")
|
|
191
|
+
return "This turn continues requested-changes work on the existing PR.";
|
|
192
|
+
if (runType === "ci_repair")
|
|
193
|
+
return "This turn continues CI repair work on the existing PR.";
|
|
194
|
+
if (runType === "queue_repair")
|
|
195
|
+
return "This turn continues merge-queue repair work on the existing PR.";
|
|
196
|
+
return "This turn continues implementation on the existing issue session.";
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
function resolveFollowUpAction(runType, context) {
|
|
200
|
+
if (context?.directReplyMode === true) {
|
|
201
|
+
return "Apply the latest human answer, continue from the current branch/session context, and only ask another question if you are still blocked.";
|
|
202
|
+
}
|
|
203
|
+
if (runType === "review_fix" && context?.branchUpkeepRequired === true) {
|
|
204
|
+
const baseBranch = typeof context.baseBranch === "string" ? context.baseBranch : "main";
|
|
205
|
+
return `Update the existing PR branch onto latest ${baseBranch}, resolve conflicts if needed, rerun narrow verification, and push the same branch.`;
|
|
206
|
+
}
|
|
207
|
+
switch (runType) {
|
|
208
|
+
case "review_fix":
|
|
209
|
+
return "Address the review feedback on the current PR branch, verify the fix, and push the same branch.";
|
|
210
|
+
case "ci_repair":
|
|
211
|
+
return "Fix the failing CI root cause on the current PR branch, verify it locally, and push the same branch.";
|
|
212
|
+
case "queue_repair":
|
|
213
|
+
return "Repair the merge-queue incident on the current PR branch, verify the fix, and push the same branch.";
|
|
214
|
+
case "implementation":
|
|
215
|
+
default:
|
|
216
|
+
return "Continue from the latest branch state, incorporate the new input, and publish updates to the existing issue branch if you make changes.";
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
function hasAuthoritativeGitHubFacts(issue, runType, context) {
|
|
220
|
+
return issue.prNumber !== undefined
|
|
221
|
+
|| issue.prHeadSha !== undefined
|
|
222
|
+
|| runType !== "implementation"
|
|
223
|
+
|| typeof context?.failureHeadSha === "string"
|
|
224
|
+
|| typeof context?.failingHeadSha === "string"
|
|
225
|
+
|| typeof context?.mergeStateStatus === "string"
|
|
226
|
+
|| typeof context?.checkName === "string"
|
|
227
|
+
|| typeof context?.reviewerName === "string";
|
|
228
|
+
}
|
|
229
|
+
function appendAuthoritativeGitHubFacts(lines, issue, runType, context) {
|
|
230
|
+
if (!hasAuthoritativeGitHubFacts(issue, runType, context)) {
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
const prNumber = issue.prNumber !== undefined ? `#${issue.prNumber}` : undefined;
|
|
234
|
+
const headSha = typeof context?.failureHeadSha === "string"
|
|
235
|
+
? context.failureHeadSha
|
|
236
|
+
: typeof context?.failingHeadSha === "string"
|
|
237
|
+
? context.failingHeadSha
|
|
238
|
+
: issue.prHeadSha;
|
|
239
|
+
const mergeStateStatus = typeof context?.mergeStateStatus === "string" ? context.mergeStateStatus : undefined;
|
|
240
|
+
const baseBranch = typeof context?.baseBranch === "string" ? context.baseBranch : undefined;
|
|
241
|
+
const checkName = typeof context?.checkName === "string" ? context.checkName : undefined;
|
|
242
|
+
const jobName = typeof context?.jobName === "string" ? context.jobName : undefined;
|
|
243
|
+
const stepName = typeof context?.stepName === "string" ? context.stepName : undefined;
|
|
244
|
+
const reviewerName = typeof context?.reviewerName === "string" ? context.reviewerName : undefined;
|
|
245
|
+
const reviewBody = typeof context?.reviewBody === "string" ? context.reviewBody.trim() : "";
|
|
246
|
+
const summary = typeof context?.summary === "string" ? context.summary : undefined;
|
|
247
|
+
lines.push("## Authoritative GitHub Facts", "");
|
|
248
|
+
if (prNumber) {
|
|
249
|
+
lines.push(`- Current PR: ${prNumber}`);
|
|
250
|
+
}
|
|
251
|
+
if (headSha) {
|
|
252
|
+
lines.push(`- Current relevant head SHA: ${headSha}`);
|
|
253
|
+
}
|
|
254
|
+
if (issue.prReviewState) {
|
|
255
|
+
lines.push(`- Current review state: ${issue.prReviewState}`);
|
|
256
|
+
}
|
|
257
|
+
if (issue.prCheckStatus) {
|
|
258
|
+
lines.push(`- Current check status: ${issue.prCheckStatus}`);
|
|
259
|
+
}
|
|
260
|
+
if (mergeStateStatus) {
|
|
261
|
+
lines.push(`- Merge state against ${baseBranch ?? "base"}: ${mergeStateStatus}`);
|
|
262
|
+
}
|
|
263
|
+
if (checkName) {
|
|
264
|
+
lines.push(`- Relevant check: ${checkName}`);
|
|
265
|
+
}
|
|
266
|
+
if (jobName && jobName !== checkName) {
|
|
267
|
+
lines.push(`- Relevant job: ${jobName}`);
|
|
268
|
+
}
|
|
269
|
+
if (stepName) {
|
|
270
|
+
lines.push(`- Relevant step: ${stepName}`);
|
|
271
|
+
}
|
|
272
|
+
if (reviewerName) {
|
|
273
|
+
lines.push(`- Reviewer: ${reviewerName}`);
|
|
274
|
+
}
|
|
275
|
+
if (summary) {
|
|
276
|
+
lines.push(`- Summary: ${summary}`);
|
|
277
|
+
}
|
|
278
|
+
if (reviewBody) {
|
|
279
|
+
lines.push(`- Review body: ${reviewBody}`);
|
|
280
|
+
}
|
|
281
|
+
lines.push("");
|
|
282
|
+
}
|
|
283
|
+
function appendFactFreshness(lines, issue, runType, context) {
|
|
284
|
+
if (!hasAuthoritativeGitHubFacts(issue, runType, context)) {
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
const hasFreshFacts = context?.githubFactsFresh === true || context?.branchUpkeepRequired === true;
|
|
288
|
+
lines.push("## Fact Freshness", "");
|
|
289
|
+
if (hasFreshFacts) {
|
|
290
|
+
lines.push("GitHub facts below were refreshed immediately before this turn was created.");
|
|
291
|
+
}
|
|
292
|
+
else {
|
|
293
|
+
lines.push("GitHub facts below came from the triggering event or last known reconciliation state and may now be stale.");
|
|
294
|
+
lines.push("Verify the current PR head, review state, and check state in GitHub before making branch-mutating decisions.");
|
|
295
|
+
}
|
|
296
|
+
lines.push("");
|
|
297
|
+
}
|
|
298
|
+
function appendFollowUpPromptPrelude(lines, issue, runType, context) {
|
|
299
|
+
lines.push("## Follow-up Turn", "");
|
|
300
|
+
lines.push(`Why this turn exists: ${resolveFollowUpWhy(runType, context)}`);
|
|
301
|
+
lines.push(`Required action now: ${resolveFollowUpAction(runType, context)}`);
|
|
302
|
+
lines.push("");
|
|
303
|
+
appendLinearContext(lines, context);
|
|
304
|
+
const followUps = collectFollowUpInputs(context);
|
|
305
|
+
if (followUps.length > 0) {
|
|
306
|
+
lines.push("## What Changed Since The Last Turn", "");
|
|
307
|
+
for (const followUp of followUps) {
|
|
308
|
+
lines.push(`- ${followUp.type}${followUp.author ? ` from ${followUp.author}` : ""}: ${followUp.text}`);
|
|
309
|
+
}
|
|
310
|
+
lines.push("");
|
|
311
|
+
}
|
|
312
|
+
appendFactFreshness(lines, issue, runType, context);
|
|
313
|
+
appendAuthoritativeGitHubFacts(lines, issue, runType, context);
|
|
314
|
+
}
|
|
315
|
+
export function buildInitialRunPrompt(issue, runType, repoPath, context) {
|
|
316
|
+
const lines = buildPromptHeader(issue);
|
|
317
|
+
appendTaskObjective(lines, issue);
|
|
318
|
+
appendLinearContext(lines, context);
|
|
57
319
|
// Add run-type-specific context for reactive runs
|
|
58
320
|
switch (runType) {
|
|
59
321
|
case "ci_repair": {
|
|
@@ -75,17 +337,73 @@ function buildRunPrompt(issue, runType, repoPath, context) {
|
|
|
75
337
|
lines.push("## Merge Queue Failure", "", "The merge queue rejected this PR. Rebase onto latest main and fix conflicts.", context?.failureReason ? `Failure reason: ${String(context.failureReason)}` : "", "", "Fetch and rebase onto latest main, resolve conflicts, run verification, push.", "If the conflict is a semantic contradiction, explain and stop.", "");
|
|
76
338
|
break;
|
|
77
339
|
}
|
|
78
|
-
// Append the repo's workflow file
|
|
79
340
|
const workflowBody = readWorkflowFile(repoPath, runType);
|
|
80
341
|
if (workflowBody) {
|
|
81
342
|
lines.push(workflowBody);
|
|
82
343
|
}
|
|
83
344
|
else if (runType === "implementation") {
|
|
84
|
-
|
|
85
|
-
lines.push("Implement the Linear issue. Read the issue via MCP for details.", "Run verification before finishing. Commit, push, and open a PR.");
|
|
345
|
+
lines.push("Implement the Linear issue. Read the issue via MCP for details.");
|
|
86
346
|
}
|
|
347
|
+
appendPublicationContract(lines, runType, issue, context);
|
|
87
348
|
return lines.join("\n");
|
|
88
349
|
}
|
|
350
|
+
export function buildFollowUpRunPrompt(issue, runType, repoPath, context) {
|
|
351
|
+
const lines = buildPromptHeader(issue);
|
|
352
|
+
appendFollowUpPromptPrelude(lines, issue, runType, context);
|
|
353
|
+
// Add run-type-specific context for reactive runs
|
|
354
|
+
switch (runType) {
|
|
355
|
+
case "ci_repair": {
|
|
356
|
+
const snapshot = context?.ciSnapshot && typeof context.ciSnapshot === "object"
|
|
357
|
+
? context.ciSnapshot
|
|
358
|
+
: undefined;
|
|
359
|
+
lines.push("## CI Repair", "", "A full CI iteration has settled failed on your PR. Start from the specific failing check/job/step below on the latest remote PR branch tip, fix that concrete failure first, then push to the same PR branch.", snapshot?.gateCheckName ? `Gate check: ${String(snapshot.gateCheckName)}` : "", snapshot?.gateCheckStatus ? `Gate status: ${String(snapshot.gateCheckStatus)}` : "", snapshot?.settledAt ? `Settled at: ${String(snapshot.settledAt)}` : "", context?.failureHeadSha ? `Failing head SHA: ${String(context.failureHeadSha)}` : "", context?.checkName ? `Failed check: ${String(context.checkName)}` : "", context?.jobName && context?.jobName !== context?.checkName ? `Failed job: ${String(context.jobName)}` : "", context?.stepName ? `Failed step: ${String(context.stepName)}` : "", context?.summary ? `Failure summary: ${String(context.summary)}` : "", Array.isArray(snapshot?.failedChecks) && snapshot.failedChecks.length > 0
|
|
360
|
+
? `Other failed checks in the settled snapshot (context only; ignore unless the logs show the same root cause):\n${snapshot.failedChecks.map((entry) => `- ${String(entry.name ?? "unknown")}${entry.summary ? `: ${String(entry.summary)}` : ""}`).join("\n")}`
|
|
361
|
+
: "", context?.checkUrl ? `Check URL: ${String(context.checkUrl)}` : "", Array.isArray(context?.annotations) && context.annotations.length > 0
|
|
362
|
+
? `Annotations:\n${context.annotations.map((entry) => `- ${String(entry)}`).join("\n")}`
|
|
363
|
+
: "", "", "Fetch the latest remote branch state first. If the branch moved since this failure, restart from the new tip instead of pushing older work.", "Read the latest logs for the named failing check, fix that root cause, and only broaden scope when the logs show direct fallout from the same issue.", "Do not change workflows, dependency installation, or unrelated tests unless the failing logs clearly point there.", "Run focused verification for the named failure, then commit and push.", "Do not open a new PR. Keep working on the existing branch until CI goes green or the situation is clearly stuck.", "Do not change test expectations unless the test is genuinely wrong.", "");
|
|
364
|
+
break;
|
|
365
|
+
}
|
|
366
|
+
case "review_fix":
|
|
367
|
+
lines.push("## Review Changes Requested", "", "A reviewer has requested changes on your PR. Address the feedback and push.", context?.reviewerName ? `Reviewer: ${String(context.reviewerName)}` : "", context?.reviewBody ? `\n## Review comment\n\n${String(context.reviewBody)}` : "", "", "Steps:", "1. Read the review feedback and PR comments (`gh pr view --comments`).", "2. Check the current diff (`git diff origin/main`) — a prior rebase may have already resolved some concerns (e.g., scope-bundling from stale commits).", "3. For each review point: if already resolved, note why. If not, fix it.", "4. Run verification, commit and push.", "5. If you believe all concerns are resolved, request a re-review: `gh pr edit <PR#> --add-reviewer <reviewer>`.", " Do NOT just post a comment saying \"resolved\" — the reviewer must re-review to dismiss the CHANGES_REQUESTED state.", "");
|
|
368
|
+
break;
|
|
369
|
+
case "queue_repair":
|
|
370
|
+
appendQueueRepairContext(lines, context);
|
|
371
|
+
lines.push("## Merge Queue Failure", "", "The merge queue rejected this PR. Rebase onto latest main and fix conflicts.", context?.failureReason ? `Failure reason: ${String(context.failureReason)}` : "", "", "Fetch and rebase onto latest main, resolve conflicts, run verification, push.", "If the conflict is a semantic contradiction, explain and stop.", "");
|
|
372
|
+
break;
|
|
373
|
+
}
|
|
374
|
+
const workflowBody = readWorkflowFile(repoPath, runType);
|
|
375
|
+
if (workflowBody) {
|
|
376
|
+
lines.push(workflowBody);
|
|
377
|
+
}
|
|
378
|
+
else if (runType === "implementation") {
|
|
379
|
+
lines.push("Implement the Linear issue. Read the issue via MCP for details.");
|
|
380
|
+
}
|
|
381
|
+
appendPublicationContract(lines, runType, issue, context);
|
|
382
|
+
return lines.join("\n");
|
|
383
|
+
}
|
|
384
|
+
function shouldBuildFollowUpPrompt(runType, context) {
|
|
385
|
+
if (context?.followUpMode)
|
|
386
|
+
return true;
|
|
387
|
+
if (runType !== "implementation")
|
|
388
|
+
return true;
|
|
389
|
+
const wakeReason = typeof context?.wakeReason === "string" ? context.wakeReason : undefined;
|
|
390
|
+
return Boolean(wakeReason && wakeReason !== "delegated");
|
|
391
|
+
}
|
|
392
|
+
export function buildRunPrompt(issue, runType, repoPath, context) {
|
|
393
|
+
if (shouldBuildFollowUpPrompt(runType, context)) {
|
|
394
|
+
return buildFollowUpRunPrompt(issue, runType, repoPath, context);
|
|
395
|
+
}
|
|
396
|
+
return buildInitialRunPrompt(issue, runType, repoPath, context);
|
|
397
|
+
}
|
|
398
|
+
function shouldCompactThread(issue, threadGeneration, context) {
|
|
399
|
+
const followUpCount = typeof context?.followUpCount === "number" ? context.followUpCount : 0;
|
|
400
|
+
return issue.threadId !== undefined
|
|
401
|
+
&& (threadGeneration ?? 0) >= MAX_THREAD_GENERATION_BEFORE_COMPACTION
|
|
402
|
+
&& followUpCount >= MAX_FOLLOW_UPS_BEFORE_COMPACTION;
|
|
403
|
+
}
|
|
404
|
+
function isBranchUpkeepRequired(context) {
|
|
405
|
+
return context?.branchUpkeepRequired === true;
|
|
406
|
+
}
|
|
89
407
|
export class RunOrchestrator {
|
|
90
408
|
config;
|
|
91
409
|
db;
|
|
@@ -100,6 +418,8 @@ export class RunOrchestrator {
|
|
|
100
418
|
idleReconciler;
|
|
101
419
|
linearSync;
|
|
102
420
|
activeThreadId;
|
|
421
|
+
workerId = `patchrelay:${process.pid}`;
|
|
422
|
+
activeSessionLeases = new Map();
|
|
103
423
|
botIdentity;
|
|
104
424
|
constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
|
|
105
425
|
this.config = config;
|
|
@@ -112,28 +432,103 @@ export class RunOrchestrator {
|
|
|
112
432
|
this.worktreeManager = new WorktreeManager(config);
|
|
113
433
|
this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
|
|
114
434
|
this.idleReconciler = new IdleIssueReconciler(db, config, {
|
|
115
|
-
requestMergeQueueAdmission: (issue, projectId) => this.requestMergeQueueAdmission(issue, projectId),
|
|
116
435
|
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
117
436
|
}, logger, feed);
|
|
118
437
|
this.queueHealthMonitor = new QueueHealthMonitor(db, config, {
|
|
119
438
|
advanceIdleIssue: (issue, newState, options) => this.idleReconciler.advanceIdleIssue(issue, newState, options),
|
|
439
|
+
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
120
440
|
}, logger, feed);
|
|
121
441
|
}
|
|
442
|
+
resolveRunWake(issue) {
|
|
443
|
+
const sessionWake = this.db.peekIssueSessionWake(issue.projectId, issue.linearIssueId);
|
|
444
|
+
if (sessionWake) {
|
|
445
|
+
return {
|
|
446
|
+
runType: sessionWake.runType,
|
|
447
|
+
context: sessionWake.context,
|
|
448
|
+
wakeReason: sessionWake.wakeReason,
|
|
449
|
+
resumeThread: sessionWake.resumeThread,
|
|
450
|
+
eventIds: sessionWake.eventIds,
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
return undefined;
|
|
454
|
+
}
|
|
455
|
+
appendWakeEventWithLease(lease, issue, runType, context, dedupeScope) {
|
|
456
|
+
let eventType;
|
|
457
|
+
let dedupeKey;
|
|
458
|
+
if (runType === "queue_repair") {
|
|
459
|
+
eventType = "merge_steward_incident";
|
|
460
|
+
dedupeKey = `${dedupeScope ?? "wake"}:queue_repair:${issue.linearIssueId}:${issue.prHeadSha ?? issue.lastGitHubFailureHeadSha ?? "unknown-sha"}`;
|
|
461
|
+
}
|
|
462
|
+
else if (runType === "ci_repair") {
|
|
463
|
+
eventType = "settled_red_ci";
|
|
464
|
+
dedupeKey = `${dedupeScope ?? "wake"}:ci_repair:${issue.linearIssueId}:${issue.lastGitHubFailureSignature ?? issue.prHeadSha ?? "unknown-sha"}`;
|
|
465
|
+
}
|
|
466
|
+
else if (runType === "review_fix") {
|
|
467
|
+
eventType = "review_changes_requested";
|
|
468
|
+
dedupeKey = `${dedupeScope ?? "wake"}:review_fix:${issue.linearIssueId}:${issue.prHeadSha ?? "unknown-sha"}`;
|
|
469
|
+
}
|
|
470
|
+
else {
|
|
471
|
+
eventType = "delegated";
|
|
472
|
+
dedupeKey = `${dedupeScope ?? "wake"}:implementation:${issue.linearIssueId}`;
|
|
473
|
+
}
|
|
474
|
+
return Boolean(this.db.appendIssueSessionEventWithLease(lease, {
|
|
475
|
+
projectId: issue.projectId,
|
|
476
|
+
linearIssueId: issue.linearIssueId,
|
|
477
|
+
eventType,
|
|
478
|
+
...(context ? { eventJson: JSON.stringify(context) } : {}),
|
|
479
|
+
dedupeKey,
|
|
480
|
+
}));
|
|
481
|
+
}
|
|
482
|
+
materializeLegacyPendingWake(issue, lease) {
|
|
483
|
+
if (!issue.pendingRunType)
|
|
484
|
+
return issue;
|
|
485
|
+
const context = issue.pendingRunContextJson
|
|
486
|
+
? JSON.parse(issue.pendingRunContextJson)
|
|
487
|
+
: undefined;
|
|
488
|
+
this.appendWakeEventWithLease(lease, issue, issue.pendingRunType, context, "legacy_pending");
|
|
489
|
+
const updated = this.db.upsertIssueWithLease(lease, {
|
|
490
|
+
projectId: issue.projectId,
|
|
491
|
+
linearIssueId: issue.linearIssueId,
|
|
492
|
+
pendingRunType: null,
|
|
493
|
+
pendingRunContextJson: null,
|
|
494
|
+
});
|
|
495
|
+
if (!updated)
|
|
496
|
+
return issue;
|
|
497
|
+
return this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
|
|
498
|
+
}
|
|
122
499
|
// ─── Run ────────────────────────────────────────────────────────
|
|
123
500
|
async run(item) {
|
|
124
501
|
const project = this.config.projects.find((p) => p.id === item.projectId);
|
|
125
502
|
if (!project)
|
|
126
503
|
return;
|
|
504
|
+
if (this.activeSessionLeases.has(this.issueSessionLeaseKey(item.projectId, item.issueId))) {
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
127
507
|
const issue = this.db.getIssue(item.projectId, item.issueId);
|
|
128
|
-
if (!issue
|
|
508
|
+
if (!issue || issue.activeRunId !== undefined)
|
|
509
|
+
return;
|
|
510
|
+
const issueSession = this.db.getIssueSession(item.projectId, item.issueId);
|
|
511
|
+
const leaseId = this.acquireIssueSessionLease(item.projectId, item.issueId);
|
|
512
|
+
if (!leaseId) {
|
|
513
|
+
this.logger.info({ issueKey: issue.issueKey, projectId: item.projectId }, "Skipped run because another worker holds the session lease");
|
|
129
514
|
return;
|
|
515
|
+
}
|
|
130
516
|
if (issue.prState === "merged") {
|
|
131
|
-
this.db.
|
|
517
|
+
this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
|
|
518
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
132
519
|
return;
|
|
133
520
|
}
|
|
134
|
-
const
|
|
135
|
-
const
|
|
136
|
-
|
|
521
|
+
const wakeIssue = this.materializeLegacyPendingWake(issue, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
|
|
522
|
+
const wake = this.resolveRunWake(wakeIssue);
|
|
523
|
+
if (!wake) {
|
|
524
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
const { runType, context, resumeThread } = wake;
|
|
528
|
+
const effectiveContext = runType === "review_fix"
|
|
529
|
+
? await this.resolveReviewFixWakeContext(issue, context, project)
|
|
530
|
+
: context;
|
|
531
|
+
const isReviewFixBranchUpkeep = runType === "review_fix" && isBranchUpkeepRequired(effectiveContext);
|
|
137
532
|
// Check repair budgets
|
|
138
533
|
if (runType === "ci_repair" && issue.ciRepairAttempts >= DEFAULT_CI_REPAIR_BUDGET) {
|
|
139
534
|
this.escalate(issue, runType, `CI repair budget exhausted (${DEFAULT_CI_REPAIR_BUDGET} attempts)`);
|
|
@@ -143,22 +538,34 @@ export class RunOrchestrator {
|
|
|
143
538
|
this.escalate(issue, runType, `Queue repair budget exhausted (${DEFAULT_QUEUE_REPAIR_BUDGET} attempts)`);
|
|
144
539
|
return;
|
|
145
540
|
}
|
|
146
|
-
if (runType === "review_fix" && issue.reviewFixAttempts >= DEFAULT_REVIEW_FIX_BUDGET) {
|
|
541
|
+
if (runType === "review_fix" && !isReviewFixBranchUpkeep && issue.reviewFixAttempts >= DEFAULT_REVIEW_FIX_BUDGET) {
|
|
147
542
|
this.escalate(issue, runType, `Review fix budget exhausted (${DEFAULT_REVIEW_FIX_BUDGET} attempts)`);
|
|
148
543
|
return;
|
|
149
544
|
}
|
|
150
545
|
// Increment repair counters
|
|
151
546
|
if (runType === "ci_repair") {
|
|
152
|
-
this.db.
|
|
547
|
+
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, ciRepairAttempts: issue.ciRepairAttempts + 1 });
|
|
548
|
+
if (!updated) {
|
|
549
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
153
552
|
}
|
|
154
553
|
if (runType === "queue_repair") {
|
|
155
|
-
this.db.
|
|
554
|
+
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueRepairAttempts: issue.queueRepairAttempts + 1 });
|
|
555
|
+
if (!updated) {
|
|
556
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
156
559
|
}
|
|
157
|
-
if (runType === "review_fix") {
|
|
158
|
-
this.db.
|
|
560
|
+
if (runType === "review_fix" && !isReviewFixBranchUpkeep) {
|
|
561
|
+
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, reviewFixAttempts: issue.reviewFixAttempts + 1 });
|
|
562
|
+
if (!updated) {
|
|
563
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
159
566
|
}
|
|
160
567
|
// Build prompt
|
|
161
|
-
const prompt = buildRunPrompt(issue, runType, project.repoPath,
|
|
568
|
+
const prompt = buildRunPrompt(issue, runType, project.repoPath, effectiveContext);
|
|
162
569
|
// Resolve workspace
|
|
163
570
|
const issueRef = sanitizePathSegment(issue.issueKey ?? issue.linearIssueId);
|
|
164
571
|
const slug = issue.title ? slugify(issue.title) : "";
|
|
@@ -166,9 +573,13 @@ export class RunOrchestrator {
|
|
|
166
573
|
const branchName = issue.branchName ?? `${project.branchPrefix}/${branchSuffix}`;
|
|
167
574
|
const worktreePath = issue.worktreePath ?? `${project.worktreeRoot}/${issueRef}`;
|
|
168
575
|
// Claim the run atomically
|
|
169
|
-
const run = this.db.
|
|
576
|
+
const run = this.db.withIssueSessionLease(item.projectId, item.issueId, leaseId, () => {
|
|
170
577
|
const fresh = this.db.getIssue(item.projectId, item.issueId);
|
|
171
|
-
if (!fresh
|
|
578
|
+
if (!fresh || fresh.activeRunId !== undefined)
|
|
579
|
+
return undefined;
|
|
580
|
+
const wakeIssue = this.materializeLegacyPendingWake(fresh, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
|
|
581
|
+
const freshWake = this.resolveRunWake(wakeIssue);
|
|
582
|
+
if (!freshWake || freshWake.runType !== runType)
|
|
172
583
|
return undefined;
|
|
173
584
|
const created = this.db.createRun({
|
|
174
585
|
issueId: fresh.id,
|
|
@@ -177,10 +588,10 @@ export class RunOrchestrator {
|
|
|
177
588
|
runType,
|
|
178
589
|
promptText: prompt,
|
|
179
590
|
});
|
|
180
|
-
const failureHeadSha = typeof
|
|
181
|
-
?
|
|
182
|
-
: typeof
|
|
183
|
-
const failureSignature = typeof
|
|
591
|
+
const failureHeadSha = typeof effectiveContext?.failureHeadSha === "string"
|
|
592
|
+
? effectiveContext.failureHeadSha
|
|
593
|
+
: typeof effectiveContext?.headSha === "string" ? effectiveContext.headSha : undefined;
|
|
594
|
+
const failureSignature = typeof effectiveContext?.failureSignature === "string" ? effectiveContext.failureSignature : undefined;
|
|
184
595
|
this.db.upsertIssue({
|
|
185
596
|
projectId: item.projectId,
|
|
186
597
|
linearIssueId: item.issueId,
|
|
@@ -201,11 +612,15 @@ export class RunOrchestrator {
|
|
|
201
612
|
}
|
|
202
613
|
: {}),
|
|
203
614
|
});
|
|
204
|
-
this.db.
|
|
615
|
+
this.db.consumeIssueSessionEvents(item.projectId, item.issueId, freshWake.eventIds, created.id);
|
|
616
|
+
this.db.setIssueSessionLastWakeReason(item.projectId, item.issueId, freshWake.wakeReason ?? null);
|
|
617
|
+
this.db.setBranchOwnerWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, "patchrelay");
|
|
205
618
|
return created;
|
|
206
619
|
});
|
|
207
|
-
if (!run)
|
|
620
|
+
if (!run) {
|
|
621
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
208
622
|
return;
|
|
623
|
+
}
|
|
209
624
|
this.feed?.publish({
|
|
210
625
|
level: "info",
|
|
211
626
|
kind: "stage",
|
|
@@ -217,6 +632,7 @@ export class RunOrchestrator {
|
|
|
217
632
|
});
|
|
218
633
|
let threadId;
|
|
219
634
|
let turnId;
|
|
635
|
+
let parentThreadId;
|
|
220
636
|
try {
|
|
221
637
|
// Ensure worktree
|
|
222
638
|
await this.worktreeManager.ensureIssueWorktree(project.repoPath, project.worktreeRoot, worktreePath, branchName, { allowExistingOutsideRoot: issue.branchName !== undefined });
|
|
@@ -232,6 +648,7 @@ export class RunOrchestrator {
|
|
|
232
648
|
const credentialHelper = `!f() { echo "username=x-access-token"; echo "password=$(cat ${this.botIdentity.tokenFile})"; }; f`;
|
|
233
649
|
await execCommand(gitBin, ["-C", worktreePath, "config", "credential.helper", credentialHelper], { timeoutMs: 5_000 });
|
|
234
650
|
}
|
|
651
|
+
await this.resetWorktreeToTrackedBranch(worktreePath, branchName, issue);
|
|
235
652
|
// Freshen the worktree: fetch + rebase onto latest base branch.
|
|
236
653
|
// This prevents branch contamination when local main has drifted
|
|
237
654
|
// and avoids scope-bundling review rejections from stale commits.
|
|
@@ -245,15 +662,22 @@ export class RunOrchestrator {
|
|
|
245
662
|
if (prepareResult.ran && prepareResult.exitCode !== 0) {
|
|
246
663
|
throw new Error(`prepare-worktree hook failed (exit ${prepareResult.exitCode}): ${prepareResult.stderr?.slice(0, 500) ?? ""}`);
|
|
247
664
|
}
|
|
248
|
-
|
|
249
|
-
//
|
|
250
|
-
|
|
665
|
+
this.assertLaunchLease(run, "before starting the Codex turn");
|
|
666
|
+
// Reuse the existing thread when the wake source is an additive follow-up
|
|
667
|
+
// or when review-fix work benefits from carrying reviewer context forward.
|
|
668
|
+
// If the thread has accumulated many resumptions and batched follow-ups,
|
|
669
|
+
// compact by starting a fresh main thread while keeping a parent link.
|
|
670
|
+
const compactThread = shouldCompactThread(issue, issueSession?.threadGeneration, effectiveContext);
|
|
671
|
+
if (compactThread && issue.threadId) {
|
|
672
|
+
parentThreadId = issue.threadId;
|
|
673
|
+
}
|
|
674
|
+
if (issue.threadId && !compactThread && (resumeThread || runType === "review_fix")) {
|
|
251
675
|
threadId = issue.threadId;
|
|
252
676
|
}
|
|
253
677
|
else {
|
|
254
678
|
const thread = await this.codex.startThread({ cwd: worktreePath });
|
|
255
679
|
threadId = thread.id;
|
|
256
|
-
this.db.
|
|
680
|
+
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
|
|
257
681
|
}
|
|
258
682
|
try {
|
|
259
683
|
const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
|
|
@@ -266,7 +690,7 @@ export class RunOrchestrator {
|
|
|
266
690
|
this.logger.info({ issueKey: issue.issueKey, staleThreadId: threadId }, "Thread is stale, retrying with fresh thread");
|
|
267
691
|
const thread = await this.codex.startThread({ cwd: worktreePath });
|
|
268
692
|
threadId = thread.id;
|
|
269
|
-
this.db.
|
|
693
|
+
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
|
|
270
694
|
const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
|
|
271
695
|
turnId = turn.turnId;
|
|
272
696
|
}
|
|
@@ -274,26 +698,39 @@ export class RunOrchestrator {
|
|
|
274
698
|
throw turnError;
|
|
275
699
|
}
|
|
276
700
|
}
|
|
701
|
+
this.assertLaunchLease(run, "after starting the Codex turn");
|
|
277
702
|
}
|
|
278
703
|
catch (error) {
|
|
279
704
|
const message = error instanceof Error ? error.message : String(error);
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
projectId: item.projectId,
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
705
|
+
const lostLease = error instanceof Error && error.name === "IssueSessionLeaseLostError";
|
|
706
|
+
if (!lostLease) {
|
|
707
|
+
this.db.finishRunWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, run.id, {
|
|
708
|
+
status: "failed",
|
|
709
|
+
failureReason: message,
|
|
710
|
+
});
|
|
711
|
+
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
712
|
+
projectId: item.projectId,
|
|
713
|
+
linearIssueId: item.issueId,
|
|
714
|
+
activeRunId: null,
|
|
715
|
+
factoryState: "failed",
|
|
716
|
+
});
|
|
717
|
+
}
|
|
287
718
|
this.logger.error({ issueKey: issue.issueKey, runType, error: message }, `Failed to launch ${runType} run`);
|
|
288
719
|
const failedIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
289
720
|
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(runType, `Failed to start ${lowerCaseFirst(message)}`));
|
|
290
721
|
void this.linearSync.syncSession(failedIssue, { activeRunType: runType });
|
|
722
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
291
723
|
throw error;
|
|
292
724
|
}
|
|
293
|
-
this.
|
|
725
|
+
this.assertLaunchLease(run, "before recording the active thread");
|
|
726
|
+
if (!this.db.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
|
|
727
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping run thread update after losing issue-session lease");
|
|
728
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
729
|
+
return;
|
|
730
|
+
}
|
|
294
731
|
// Reset zombie recovery counter — this run started successfully
|
|
295
732
|
if (issue.zombieRecoveryAttempts > 0) {
|
|
296
|
-
this.db.
|
|
733
|
+
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
297
734
|
projectId: item.projectId,
|
|
298
735
|
linearIssueId: item.issueId,
|
|
299
736
|
zombieRecoveryAttempts: 0,
|
|
@@ -357,6 +794,44 @@ export class RunOrchestrator {
|
|
|
357
794
|
if (didStash)
|
|
358
795
|
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
359
796
|
}
|
|
797
|
+
async resetWorktreeToTrackedBranch(worktreePath, branchName, issue) {
|
|
798
|
+
const gitBin = this.config.runner.gitBin;
|
|
799
|
+
const branchFetch = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", branchName], { timeoutMs: 60_000 });
|
|
800
|
+
const hasRemoteBranch = branchFetch.exitCode === 0;
|
|
801
|
+
await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
|
|
802
|
+
await execCommand(gitBin, ["-C", worktreePath, "merge", "--abort"], { timeoutMs: 10_000 });
|
|
803
|
+
await execCommand(gitBin, ["-C", worktreePath, "cherry-pick", "--abort"], { timeoutMs: 10_000 });
|
|
804
|
+
await execCommand(gitBin, ["-C", worktreePath, "am", "--abort"], { timeoutMs: 10_000 });
|
|
805
|
+
await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", "HEAD"], { timeoutMs: 30_000 });
|
|
806
|
+
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
807
|
+
const checkoutTarget = hasRemoteBranch ? `origin/${branchName}` : branchName;
|
|
808
|
+
const checkoutResult = await execCommand(gitBin, ["-C", worktreePath, "checkout", "-B", branchName, checkoutTarget], { timeoutMs: 30_000 });
|
|
809
|
+
if (checkoutResult.exitCode !== 0) {
|
|
810
|
+
throw new Error(`Failed to restore ${branchName} worktree state: ${checkoutResult.stderr?.slice(0, 300) ?? "git checkout failed"}`);
|
|
811
|
+
}
|
|
812
|
+
const resetTarget = hasRemoteBranch ? `origin/${branchName}` : "HEAD";
|
|
813
|
+
const resetResult = await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", resetTarget], { timeoutMs: 30_000 });
|
|
814
|
+
if (resetResult.exitCode !== 0) {
|
|
815
|
+
throw new Error(`Failed to reset ${branchName} worktree state: ${resetResult.stderr?.slice(0, 300) ?? "git reset failed"}`);
|
|
816
|
+
}
|
|
817
|
+
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
818
|
+
this.logger.debug({ issueKey: issue.issueKey, branchName, hasRemoteBranch }, "Reset issue worktree to tracked branch state");
|
|
819
|
+
}
|
|
820
|
+
async restoreIdleWorktree(issue) {
|
|
821
|
+
if (!issue.worktreePath || !issue.branchName)
|
|
822
|
+
return;
|
|
823
|
+
try {
|
|
824
|
+
await this.resetWorktreeToTrackedBranch(issue.worktreePath, issue.branchName, issue);
|
|
825
|
+
}
|
|
826
|
+
catch (error) {
|
|
827
|
+
this.logger.warn({
|
|
828
|
+
issueKey: issue.issueKey,
|
|
829
|
+
branchName: issue.branchName,
|
|
830
|
+
worktreePath: issue.worktreePath,
|
|
831
|
+
error: error instanceof Error ? error.message : String(error),
|
|
832
|
+
}, "Failed to restore idle worktree after interrupted run");
|
|
833
|
+
}
|
|
834
|
+
}
|
|
360
835
|
// ─── Notification handler ─────────────────────────────────────────
|
|
361
836
|
async handleCodexNotification(notification) {
|
|
362
837
|
// threadId is present on turn-level notifications but NOT on item-level ones.
|
|
@@ -374,6 +849,10 @@ export class RunOrchestrator {
|
|
|
374
849
|
const run = this.db.getRunByThreadId(threadId);
|
|
375
850
|
if (!run)
|
|
376
851
|
return;
|
|
852
|
+
if (!this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
853
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Ignoring Codex notification after losing issue-session lease");
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
377
856
|
const turnId = typeof notification.params.turnId === "string" ? notification.params.turnId : undefined;
|
|
378
857
|
if (this.config.runner.codex.persistExtendedHistory) {
|
|
379
858
|
this.db.saveThreadEvent({
|
|
@@ -402,18 +881,26 @@ export class RunOrchestrator {
|
|
|
402
881
|
const completedTurnId = extractTurnId(notification.params);
|
|
403
882
|
const status = resolveRunCompletionStatus(notification.params);
|
|
404
883
|
if (status === "failed") {
|
|
405
|
-
this.
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
884
|
+
const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
885
|
+
this.db.finishRunWithLease(lease, run.id, {
|
|
886
|
+
status: "failed",
|
|
887
|
+
threadId,
|
|
888
|
+
...(completedTurnId ? { turnId: completedTurnId } : {}),
|
|
889
|
+
failureReason: "Codex reported the turn completed in a failed state",
|
|
890
|
+
});
|
|
891
|
+
this.db.upsertIssueWithLease(lease, {
|
|
892
|
+
projectId: run.projectId,
|
|
893
|
+
linearIssueId: run.linearIssueId,
|
|
894
|
+
activeRunId: null,
|
|
895
|
+
factoryState: "failed",
|
|
896
|
+
});
|
|
897
|
+
return true;
|
|
416
898
|
});
|
|
899
|
+
if (!updated) {
|
|
900
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping failed-turn cleanup after losing issue-session lease");
|
|
901
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
902
|
+
return;
|
|
903
|
+
}
|
|
417
904
|
this.feed?.publish({
|
|
418
905
|
level: "error",
|
|
419
906
|
kind: "turn",
|
|
@@ -428,11 +915,12 @@ export class RunOrchestrator {
|
|
|
428
915
|
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
429
916
|
this.linearSync.clearProgress(run.id);
|
|
430
917
|
this.activeThreadId = undefined;
|
|
918
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
431
919
|
return;
|
|
432
920
|
}
|
|
433
921
|
// Complete the run
|
|
434
922
|
const trackedIssue = this.db.issueToTrackedIssue(issue);
|
|
435
|
-
const report = buildStageReport(run, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
|
|
923
|
+
const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
|
|
436
924
|
// Determine post-run state based on current PR metadata.
|
|
437
925
|
const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
438
926
|
const verifiedRepairError = await this.verifyReactiveRunAdvancedBranch(run, freshIssue);
|
|
@@ -453,10 +941,32 @@ export class RunOrchestrator {
|
|
|
453
941
|
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
454
942
|
this.linearSync.clearProgress(run.id);
|
|
455
943
|
this.activeThreadId = undefined;
|
|
944
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
456
945
|
return;
|
|
457
946
|
}
|
|
458
|
-
const
|
|
459
|
-
|
|
947
|
+
const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
|
|
948
|
+
if (publishedOutcomeError) {
|
|
949
|
+
this.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
950
|
+
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
951
|
+
this.feed?.publish({
|
|
952
|
+
level: "warn",
|
|
953
|
+
kind: "turn",
|
|
954
|
+
issueKey: freshIssue.issueKey,
|
|
955
|
+
projectId: run.projectId,
|
|
956
|
+
stage: run.runType,
|
|
957
|
+
status: "publish_incomplete",
|
|
958
|
+
summary: publishedOutcomeError,
|
|
959
|
+
});
|
|
960
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
961
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
962
|
+
this.linearSync.clearProgress(run.id);
|
|
963
|
+
this.activeThreadId = undefined;
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
967
|
+
const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
|
|
968
|
+
const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
|
|
969
|
+
const completed = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
460
970
|
this.db.finishRun(run.id, {
|
|
461
971
|
status: "completed",
|
|
462
972
|
threadId,
|
|
@@ -468,9 +978,10 @@ export class RunOrchestrator {
|
|
|
468
978
|
projectId: run.projectId,
|
|
469
979
|
linearIssueId: run.linearIssueId,
|
|
470
980
|
activeRunId: null,
|
|
471
|
-
...(postRunState === "awaiting_queue" ? { queueLabelApplied: false } : {}),
|
|
472
981
|
...(postRunState ? { factoryState: postRunState } : {}),
|
|
473
|
-
|
|
982
|
+
pendingRunType: null,
|
|
983
|
+
pendingRunContextJson: null,
|
|
984
|
+
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
474
985
|
? {
|
|
475
986
|
lastGitHubFailureSource: null,
|
|
476
987
|
lastGitHubFailureHeadSha: null,
|
|
@@ -483,15 +994,31 @@ export class RunOrchestrator {
|
|
|
483
994
|
lastAttemptedFailureHeadSha: null,
|
|
484
995
|
lastAttemptedFailureSignature: null,
|
|
485
996
|
}
|
|
486
|
-
: {}),
|
|
997
|
+
: {})),
|
|
487
998
|
});
|
|
488
|
-
if (
|
|
489
|
-
this.
|
|
999
|
+
if (postRunFollowUp) {
|
|
1000
|
+
return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
490
1001
|
}
|
|
1002
|
+
return true;
|
|
491
1003
|
});
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
this.
|
|
1004
|
+
if (!completed) {
|
|
1005
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping completion writes after losing issue-session lease");
|
|
1006
|
+
this.linearSync.clearProgress(run.id);
|
|
1007
|
+
this.activeThreadId = undefined;
|
|
1008
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
if (postRunFollowUp) {
|
|
1012
|
+
this.feed?.publish({
|
|
1013
|
+
level: "info",
|
|
1014
|
+
kind: "stage",
|
|
1015
|
+
issueKey: issue.issueKey,
|
|
1016
|
+
projectId: run.projectId,
|
|
1017
|
+
stage: postRunFollowUp.factoryState,
|
|
1018
|
+
status: "follow_up_queued",
|
|
1019
|
+
summary: postRunFollowUp.summary,
|
|
1020
|
+
});
|
|
1021
|
+
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
495
1022
|
}
|
|
496
1023
|
this.feed?.publish({
|
|
497
1024
|
level: "info",
|
|
@@ -504,7 +1031,7 @@ export class RunOrchestrator {
|
|
|
504
1031
|
detail: summarizeCurrentThread(thread).latestAgentMessage,
|
|
505
1032
|
});
|
|
506
1033
|
// Emit Linear completion activity + plan
|
|
507
|
-
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ??
|
|
1034
|
+
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
508
1035
|
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
509
1036
|
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
510
1037
|
runType: run.runType,
|
|
@@ -515,6 +1042,7 @@ export class RunOrchestrator {
|
|
|
515
1042
|
void this.linearSync.syncSession(updatedIssue);
|
|
516
1043
|
this.linearSync.clearProgress(run.id);
|
|
517
1044
|
this.activeThreadId = undefined;
|
|
1045
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
518
1046
|
}
|
|
519
1047
|
// ─── Active status for query ──────────────────────────────────────
|
|
520
1048
|
async getActiveRunStatus(issueKey) {
|
|
@@ -543,6 +1071,44 @@ export class RunOrchestrator {
|
|
|
543
1071
|
// Advance issues stuck in pr_open whose stored PR metadata already
|
|
544
1072
|
// shows they should transition (e.g. approved PR, missed webhook).
|
|
545
1073
|
await this.idleReconciler.reconcile();
|
|
1074
|
+
await this.reconcileMergedLinearCompletion();
|
|
1075
|
+
}
|
|
1076
|
+
async reconcileMergedLinearCompletion() {
|
|
1077
|
+
for (const issue of this.db.listIssues()) {
|
|
1078
|
+
if (issue.prState !== "merged")
|
|
1079
|
+
continue;
|
|
1080
|
+
if (issue.currentLinearStateType?.trim().toLowerCase() === "completed")
|
|
1081
|
+
continue;
|
|
1082
|
+
const linear = await this.linearProvider.forProject(issue.projectId).catch(() => undefined);
|
|
1083
|
+
if (!linear)
|
|
1084
|
+
continue;
|
|
1085
|
+
try {
|
|
1086
|
+
const liveIssue = await linear.getIssue(issue.linearIssueId);
|
|
1087
|
+
const targetState = resolvePreferredCompletedLinearState(liveIssue);
|
|
1088
|
+
if (!targetState)
|
|
1089
|
+
continue;
|
|
1090
|
+
const normalizedCurrent = liveIssue.stateName?.trim().toLowerCase();
|
|
1091
|
+
if (normalizedCurrent === targetState.trim().toLowerCase()) {
|
|
1092
|
+
this.db.upsertIssue({
|
|
1093
|
+
projectId: issue.projectId,
|
|
1094
|
+
linearIssueId: issue.linearIssueId,
|
|
1095
|
+
...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
|
|
1096
|
+
...(liveIssue.stateType ? { currentLinearStateType: liveIssue.stateType } : {}),
|
|
1097
|
+
});
|
|
1098
|
+
continue;
|
|
1099
|
+
}
|
|
1100
|
+
const updated = await linear.setIssueState(issue.linearIssueId, targetState);
|
|
1101
|
+
this.db.upsertIssue({
|
|
1102
|
+
projectId: issue.projectId,
|
|
1103
|
+
linearIssueId: issue.linearIssueId,
|
|
1104
|
+
...(updated.stateName ? { currentLinearState: updated.stateName } : {}),
|
|
1105
|
+
...(updated.stateType ? { currentLinearStateType: updated.stateType } : {}),
|
|
1106
|
+
});
|
|
1107
|
+
}
|
|
1108
|
+
catch (error) {
|
|
1109
|
+
this.logger.warn({ issueKey: issue.issueKey, error: error instanceof Error ? error.message : String(error) }, "Failed to reconcile merged issue to a completed Linear state");
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
546
1112
|
}
|
|
547
1113
|
// advanceIdleIssue is now on IdleIssueReconciler — delegate for internal callers
|
|
548
1114
|
advanceIdleIssue(issue, newState, options) {
|
|
@@ -560,24 +1126,41 @@ export class RunOrchestrator {
|
|
|
560
1126
|
return;
|
|
561
1127
|
// If PR already merged, transition to done — no retry needed
|
|
562
1128
|
if (fresh.prState === "merged") {
|
|
563
|
-
this.
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
1129
|
+
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
1130
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1131
|
+
projectId: fresh.projectId,
|
|
1132
|
+
linearIssueId: fresh.linearIssueId,
|
|
1133
|
+
factoryState: "done",
|
|
1134
|
+
zombieRecoveryAttempts: 0,
|
|
1135
|
+
lastZombieRecoveryAt: null,
|
|
1136
|
+
});
|
|
1137
|
+
return true;
|
|
569
1138
|
});
|
|
1139
|
+
if (!updated) {
|
|
1140
|
+
this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping merged recovery completion after losing issue-session lease");
|
|
1141
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
1142
|
+
return;
|
|
1143
|
+
}
|
|
570
1144
|
this.logger.info({ issueKey: fresh.issueKey, reason }, "Recovery: PR already merged — transitioning to done");
|
|
1145
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
571
1146
|
return;
|
|
572
1147
|
}
|
|
573
1148
|
// Budget check
|
|
574
1149
|
const attempts = fresh.zombieRecoveryAttempts + 1;
|
|
575
1150
|
if (attempts > DEFAULT_ZOMBIE_RECOVERY_BUDGET) {
|
|
576
|
-
this.
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
1151
|
+
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
1152
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1153
|
+
projectId: fresh.projectId,
|
|
1154
|
+
linearIssueId: fresh.linearIssueId,
|
|
1155
|
+
factoryState: "escalated",
|
|
1156
|
+
});
|
|
1157
|
+
return true;
|
|
580
1158
|
});
|
|
1159
|
+
if (!updated) {
|
|
1160
|
+
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery escalation after losing issue-session lease");
|
|
1161
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
1162
|
+
return;
|
|
1163
|
+
}
|
|
581
1164
|
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: budget exhausted — escalating");
|
|
582
1165
|
this.feed?.publish({
|
|
583
1166
|
level: "error",
|
|
@@ -588,6 +1171,7 @@ export class RunOrchestrator {
|
|
|
588
1171
|
status: "budget_exhausted",
|
|
589
1172
|
summary: `${reason} recovery failed after ${DEFAULT_ZOMBIE_RECOVERY_BUDGET} attempts`,
|
|
590
1173
|
});
|
|
1174
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
591
1175
|
return;
|
|
592
1176
|
}
|
|
593
1177
|
// Exponential backoff — skip if delay hasn't elapsed
|
|
@@ -600,14 +1184,22 @@ export class RunOrchestrator {
|
|
|
600
1184
|
}
|
|
601
1185
|
}
|
|
602
1186
|
// Re-enqueue with backoff tracking
|
|
603
|
-
this.
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
1187
|
+
const requeued = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
1188
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1189
|
+
projectId: fresh.projectId,
|
|
1190
|
+
linearIssueId: fresh.linearIssueId,
|
|
1191
|
+
pendingRunType: null,
|
|
1192
|
+
pendingRunContextJson: null,
|
|
1193
|
+
zombieRecoveryAttempts: attempts,
|
|
1194
|
+
lastZombieRecoveryAt: new Date().toISOString(),
|
|
1195
|
+
});
|
|
1196
|
+
return this.appendWakeEventWithLease(lease, fresh, runType, undefined, `recovery:${attempts}`);
|
|
610
1197
|
});
|
|
1198
|
+
if (!requeued) {
|
|
1199
|
+
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery re-enqueue after losing issue-session lease");
|
|
1200
|
+
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
1201
|
+
return;
|
|
1202
|
+
}
|
|
611
1203
|
this.enqueueIssue(fresh.projectId, fresh.linearIssueId);
|
|
612
1204
|
this.logger.info({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: re-enqueued with backoff");
|
|
613
1205
|
}
|
|
@@ -615,24 +1207,38 @@ export class RunOrchestrator {
|
|
|
615
1207
|
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
616
1208
|
if (!issue)
|
|
617
1209
|
return;
|
|
1210
|
+
let recoveryLease = this.claimLeaseForReconciliation(run.projectId, run.linearIssueId);
|
|
1211
|
+
if (recoveryLease === "skip" && await this.reclaimForeignRecoveryLeaseIfSafe(run, issue)) {
|
|
1212
|
+
recoveryLease = true;
|
|
1213
|
+
}
|
|
1214
|
+
if (recoveryLease === "skip")
|
|
1215
|
+
return;
|
|
1216
|
+
const acquiredRecoveryLease = recoveryLease === true;
|
|
618
1217
|
// If the issue reached a terminal state while this run was active
|
|
619
1218
|
// (e.g. pr_merged processed, DB manually edited), just release the run.
|
|
620
1219
|
if (TERMINAL_STATES.has(issue.factoryState)) {
|
|
621
|
-
this.
|
|
1220
|
+
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
622
1221
|
this.db.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
623
1222
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
624
1223
|
});
|
|
625
1224
|
this.logger.info({ issueKey: issue.issueKey, runId: run.id, factoryState: issue.factoryState }, "Reconciliation: released run on terminal issue");
|
|
1225
|
+
const releasedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1226
|
+
void this.linearSync.syncSession(releasedIssue, { activeRunType: run.runType });
|
|
1227
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
626
1228
|
return;
|
|
627
1229
|
}
|
|
628
1230
|
// Zombie run: claimed in DB but Codex never started (no thread).
|
|
629
1231
|
if (!run.threadId) {
|
|
630
1232
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
|
|
631
|
-
this.
|
|
1233
|
+
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
632
1234
|
this.db.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
633
1235
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
634
1236
|
});
|
|
635
1237
|
this.recoverOrEscalate(issue, run.runType, "zombie");
|
|
1238
|
+
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1239
|
+
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "The Codex turn never started before PatchRelay restarted."));
|
|
1240
|
+
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
1241
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
636
1242
|
return;
|
|
637
1243
|
}
|
|
638
1244
|
// Read Codex state — thread may not exist after app-server restart.
|
|
@@ -642,11 +1248,15 @@ export class RunOrchestrator {
|
|
|
642
1248
|
}
|
|
643
1249
|
catch {
|
|
644
1250
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
|
|
645
|
-
this.
|
|
1251
|
+
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
646
1252
|
this.db.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
647
1253
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
648
1254
|
});
|
|
649
1255
|
this.recoverOrEscalate(issue, run.runType, "stale_thread");
|
|
1256
|
+
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1257
|
+
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "PatchRelay lost the active Codex thread after restart and needs to recover."));
|
|
1258
|
+
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
1259
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
650
1260
|
return;
|
|
651
1261
|
}
|
|
652
1262
|
// Check Linear state (non-fatal — token refresh may fail)
|
|
@@ -656,7 +1266,7 @@ export class RunOrchestrator {
|
|
|
656
1266
|
if (linearIssue) {
|
|
657
1267
|
const stopState = resolveAuthoritativeLinearStopState(linearIssue);
|
|
658
1268
|
if (stopState?.isFinal) {
|
|
659
|
-
this.
|
|
1269
|
+
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
660
1270
|
this.db.finishRun(run.id, { status: "released" });
|
|
661
1271
|
this.db.upsertIssue({
|
|
662
1272
|
projectId: run.projectId,
|
|
@@ -675,28 +1285,60 @@ export class RunOrchestrator {
|
|
|
675
1285
|
status: "reconciled",
|
|
676
1286
|
summary: `Linear state ${stopState.stateName} \u2192 done`,
|
|
677
1287
|
});
|
|
1288
|
+
const doneIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1289
|
+
void this.linearSync.syncSession(doneIssue, { activeRunType: run.runType });
|
|
1290
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
678
1291
|
return;
|
|
679
1292
|
}
|
|
680
1293
|
}
|
|
681
1294
|
}
|
|
682
|
-
const latestTurn = thread.
|
|
1295
|
+
const latestTurn = getThreadTurns(thread).at(-1);
|
|
683
1296
|
// Handle interrupted turn — fail the run rather than retrying indefinitely.
|
|
684
1297
|
// The agent may have partially completed work (commits, PR) before interruption.
|
|
685
1298
|
// Reactive loops (CI repair, review fix) will handle follow-up if needed.
|
|
686
1299
|
if (latestTurn?.status === "interrupted") {
|
|
687
1300
|
this.logger.warn({ issueKey: issue.issueKey, runType: run.runType, threadId: run.threadId }, "Run has interrupted turn — marking as failed");
|
|
688
1301
|
// Interrupted runs are not real failures — undo the budget increment.
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
1302
|
+
const repairedCounters = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
1303
|
+
if (run.runType === "ci_repair" && issue.ciRepairAttempts > 0) {
|
|
1304
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1305
|
+
projectId: issue.projectId,
|
|
1306
|
+
linearIssueId: issue.linearIssueId,
|
|
1307
|
+
ciRepairAttempts: issue.ciRepairAttempts - 1,
|
|
1308
|
+
});
|
|
1309
|
+
}
|
|
1310
|
+
else if (run.runType === "queue_repair" && issue.queueRepairAttempts > 0) {
|
|
1311
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1312
|
+
projectId: issue.projectId,
|
|
1313
|
+
linearIssueId: issue.linearIssueId,
|
|
1314
|
+
queueRepairAttempts: issue.queueRepairAttempts - 1,
|
|
1315
|
+
});
|
|
1316
|
+
}
|
|
1317
|
+
else if (run.runType === "review_fix" && issue.reviewFixAttempts > 0) {
|
|
1318
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1319
|
+
projectId: issue.projectId,
|
|
1320
|
+
linearIssueId: issue.linearIssueId,
|
|
1321
|
+
reviewFixAttempts: issue.reviewFixAttempts - 1,
|
|
1322
|
+
});
|
|
1323
|
+
}
|
|
1324
|
+
if (run.runType === "ci_repair" || run.runType === "queue_repair") {
|
|
1325
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1326
|
+
projectId: issue.projectId,
|
|
1327
|
+
linearIssueId: issue.linearIssueId,
|
|
1328
|
+
lastAttemptedFailureHeadSha: null,
|
|
1329
|
+
lastAttemptedFailureSignature: null,
|
|
1330
|
+
});
|
|
1331
|
+
}
|
|
1332
|
+
return true;
|
|
1333
|
+
});
|
|
1334
|
+
if (!repairedCounters) {
|
|
1335
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping interrupted-run recovery after losing issue-session lease");
|
|
1336
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1337
|
+
return;
|
|
697
1338
|
}
|
|
698
1339
|
const recoveredState = resolveRecoverablePostRunState(this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
|
|
699
1340
|
this.failRunAndClear(run, "Codex turn was interrupted", recoveredState);
|
|
1341
|
+
await this.restoreIdleWorktree(issue);
|
|
700
1342
|
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
701
1343
|
if (recoveredState) {
|
|
702
1344
|
this.feed?.publish({
|
|
@@ -713,12 +1355,13 @@ export class RunOrchestrator {
|
|
|
713
1355
|
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, "The Codex turn was interrupted."));
|
|
714
1356
|
}
|
|
715
1357
|
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1358
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
716
1359
|
return;
|
|
717
1360
|
}
|
|
718
1361
|
// Handle completed turn discovered during reconciliation
|
|
719
1362
|
if (latestTurn?.status === "completed") {
|
|
720
1363
|
const trackedIssue = this.db.issueToTrackedIssue(issue);
|
|
721
|
-
const report = buildStageReport(run, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
|
|
1364
|
+
const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
|
|
722
1365
|
const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
723
1366
|
const verifiedRepairError = await this.verifyReactiveRunAdvancedBranch(run, freshIssue);
|
|
724
1367
|
if (verifiedRepairError) {
|
|
@@ -733,10 +1376,34 @@ export class RunOrchestrator {
|
|
|
733
1376
|
status: "branch_not_advanced",
|
|
734
1377
|
summary: verifiedRepairError,
|
|
735
1378
|
});
|
|
1379
|
+
const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1380
|
+
void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
|
|
1381
|
+
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
1382
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
|
|
1386
|
+
if (publishedOutcomeError) {
|
|
1387
|
+
this.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
1388
|
+
this.feed?.publish({
|
|
1389
|
+
level: "warn",
|
|
1390
|
+
kind: "turn",
|
|
1391
|
+
issueKey: issue.issueKey,
|
|
1392
|
+
projectId: run.projectId,
|
|
1393
|
+
stage: run.runType,
|
|
1394
|
+
status: "publish_incomplete",
|
|
1395
|
+
summary: publishedOutcomeError,
|
|
1396
|
+
});
|
|
1397
|
+
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1398
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
1399
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1400
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
736
1401
|
return;
|
|
737
1402
|
}
|
|
738
|
-
const
|
|
739
|
-
this.
|
|
1403
|
+
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
1404
|
+
const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
|
|
1405
|
+
const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
|
|
1406
|
+
const reconciled = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
740
1407
|
this.db.finishRun(run.id, {
|
|
741
1408
|
status: "completed",
|
|
742
1409
|
...(run.threadId ? { threadId: run.threadId } : {}),
|
|
@@ -748,9 +1415,10 @@ export class RunOrchestrator {
|
|
|
748
1415
|
projectId: run.projectId,
|
|
749
1416
|
linearIssueId: run.linearIssueId,
|
|
750
1417
|
activeRunId: null,
|
|
751
|
-
...(postRunState === "awaiting_queue" ? { queueLabelApplied: false } : {}),
|
|
752
1418
|
...(postRunState ? { factoryState: postRunState } : {}),
|
|
753
|
-
|
|
1419
|
+
pendingRunType: null,
|
|
1420
|
+
pendingRunContextJson: null,
|
|
1421
|
+
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
754
1422
|
? {
|
|
755
1423
|
lastGitHubFailureSource: null,
|
|
756
1424
|
lastGitHubFailureHeadSha: null,
|
|
@@ -763,12 +1431,30 @@ export class RunOrchestrator {
|
|
|
763
1431
|
lastAttemptedFailureHeadSha: null,
|
|
764
1432
|
lastAttemptedFailureSignature: null,
|
|
765
1433
|
}
|
|
766
|
-
: {}),
|
|
1434
|
+
: {})),
|
|
767
1435
|
});
|
|
768
|
-
if (
|
|
769
|
-
this.
|
|
1436
|
+
if (postRunFollowUp) {
|
|
1437
|
+
return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
770
1438
|
}
|
|
1439
|
+
return true;
|
|
771
1440
|
});
|
|
1441
|
+
if (!reconciled) {
|
|
1442
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping reconciled completion writes after losing issue-session lease");
|
|
1443
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1444
|
+
return;
|
|
1445
|
+
}
|
|
1446
|
+
if (postRunFollowUp) {
|
|
1447
|
+
this.feed?.publish({
|
|
1448
|
+
level: "info",
|
|
1449
|
+
kind: "stage",
|
|
1450
|
+
issueKey: issue.issueKey,
|
|
1451
|
+
projectId: run.projectId,
|
|
1452
|
+
stage: postRunFollowUp.factoryState,
|
|
1453
|
+
status: "follow_up_queued",
|
|
1454
|
+
summary: postRunFollowUp.summary,
|
|
1455
|
+
});
|
|
1456
|
+
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
1457
|
+
}
|
|
772
1458
|
if (postRunState) {
|
|
773
1459
|
this.feed?.publish({
|
|
774
1460
|
level: "info",
|
|
@@ -780,25 +1466,43 @@ export class RunOrchestrator {
|
|
|
780
1466
|
summary: `Reconciliation: ${run.runType} completed \u2192 ${postRunState}`,
|
|
781
1467
|
});
|
|
782
1468
|
}
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
1469
|
+
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
1470
|
+
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
1471
|
+
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
1472
|
+
runType: run.runType,
|
|
1473
|
+
completionSummary,
|
|
1474
|
+
postRunState: updatedIssue.factoryState,
|
|
1475
|
+
...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
|
|
1476
|
+
}));
|
|
1477
|
+
void this.linearSync.syncSession(updatedIssue);
|
|
1478
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1479
|
+
return;
|
|
786
1480
|
}
|
|
1481
|
+
if (acquiredRecoveryLease)
|
|
1482
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
787
1483
|
}
|
|
788
1484
|
// ─── Internal helpers ─────────────────────────────────────────────
|
|
789
1485
|
escalate(issue, runType, reason) {
|
|
790
1486
|
this.logger.warn({ issueKey: issue.issueKey, runType, reason }, "Escalating to human");
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
1487
|
+
const escalated = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
1488
|
+
if (issue.activeRunId) {
|
|
1489
|
+
this.db.finishRunWithLease(lease, issue.activeRunId, { status: "released" });
|
|
1490
|
+
}
|
|
1491
|
+
this.db.upsertIssueWithLease(lease, {
|
|
1492
|
+
projectId: issue.projectId,
|
|
1493
|
+
linearIssueId: issue.linearIssueId,
|
|
1494
|
+
pendingRunType: null,
|
|
1495
|
+
pendingRunContextJson: null,
|
|
1496
|
+
activeRunId: null,
|
|
1497
|
+
factoryState: "escalated",
|
|
1498
|
+
});
|
|
1499
|
+
return true;
|
|
801
1500
|
});
|
|
1501
|
+
if (!escalated) {
|
|
1502
|
+
this.logger.warn({ issueKey: issue.issueKey, runType }, "Skipping escalation write after losing issue-session lease");
|
|
1503
|
+
this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
1504
|
+
return;
|
|
1505
|
+
}
|
|
802
1506
|
this.feed?.publish({
|
|
803
1507
|
level: "error",
|
|
804
1508
|
kind: "workflow",
|
|
@@ -814,23 +1518,10 @@ export class RunOrchestrator {
|
|
|
814
1518
|
body: `PatchRelay needs human help to continue.\n\n${reason}`,
|
|
815
1519
|
});
|
|
816
1520
|
void this.linearSync.syncSession(escalatedIssue);
|
|
817
|
-
|
|
818
|
-
/** Add the merge queue admission label for external-queue projects (best-effort). */
|
|
819
|
-
async requestMergeQueueAdmission(issue, projectId) {
|
|
820
|
-
const project = this.config.projects.find((p) => p.id === projectId);
|
|
821
|
-
const protocol = resolveMergeQueueProtocol(project);
|
|
822
|
-
const applied = await requestMergeQueueAdmission({
|
|
823
|
-
issue,
|
|
824
|
-
protocol,
|
|
825
|
-
logger: this.logger,
|
|
826
|
-
feed: this.feed,
|
|
827
|
-
});
|
|
828
|
-
if (applied) {
|
|
829
|
-
this.db.upsertIssue({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueLabelApplied: true });
|
|
830
|
-
}
|
|
1521
|
+
this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
831
1522
|
}
|
|
832
1523
|
failRunAndClear(run, message, nextState = "failed") {
|
|
833
|
-
this.
|
|
1524
|
+
const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
834
1525
|
this.db.finishRun(run.id, { status: "failed", failureReason: message });
|
|
835
1526
|
this.db.upsertIssue({
|
|
836
1527
|
projectId: run.projectId,
|
|
@@ -840,9 +1531,17 @@ export class RunOrchestrator {
|
|
|
840
1531
|
});
|
|
841
1532
|
const branchOwner = this.resolveBranchOwnerForStateTransition(nextState);
|
|
842
1533
|
if (branchOwner) {
|
|
843
|
-
this.
|
|
1534
|
+
const lease = this.getHeldIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1535
|
+
if (lease) {
|
|
1536
|
+
this.db.setBranchOwnerWithLease(lease, branchOwner);
|
|
1537
|
+
}
|
|
844
1538
|
}
|
|
1539
|
+
return true;
|
|
845
1540
|
});
|
|
1541
|
+
if (!updated) {
|
|
1542
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping failure cleanup after losing issue-session lease");
|
|
1543
|
+
}
|
|
1544
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
846
1545
|
}
|
|
847
1546
|
resolveBranchOwnerForStateTransition(newState, pendingRunType) {
|
|
848
1547
|
return resolveBranchOwnerForStateTransition(newState, pendingRunType);
|
|
@@ -859,15 +1558,8 @@ export class RunOrchestrator {
|
|
|
859
1558
|
return undefined;
|
|
860
1559
|
}
|
|
861
1560
|
try {
|
|
862
|
-
const
|
|
863
|
-
|
|
864
|
-
"--repo", project.github.repoFullName,
|
|
865
|
-
"--json", "headRefOid,state",
|
|
866
|
-
], { timeoutMs: 10_000 });
|
|
867
|
-
if (exitCode !== 0)
|
|
868
|
-
return undefined;
|
|
869
|
-
const pr = JSON.parse(stdout);
|
|
870
|
-
if (pr.state?.toUpperCase() !== "OPEN")
|
|
1561
|
+
const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
|
|
1562
|
+
if (!pr || pr.state?.toUpperCase() !== "OPEN")
|
|
871
1563
|
return undefined;
|
|
872
1564
|
if (!pr.headRefOid || pr.headRefOid !== issue.lastGitHubFailureHeadSha)
|
|
873
1565
|
return undefined;
|
|
@@ -882,6 +1574,271 @@ export class RunOrchestrator {
|
|
|
882
1574
|
return undefined;
|
|
883
1575
|
}
|
|
884
1576
|
}
|
|
1577
|
+
async refreshIssueAfterReactivePublish(run, issue) {
|
|
1578
|
+
if (run.runType !== "ci_repair" && run.runType !== "queue_repair") {
|
|
1579
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1580
|
+
}
|
|
1581
|
+
if (!issue.prNumber) {
|
|
1582
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1583
|
+
}
|
|
1584
|
+
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1585
|
+
const repoFullName = project?.github?.repoFullName;
|
|
1586
|
+
if (!repoFullName) {
|
|
1587
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1588
|
+
}
|
|
1589
|
+
try {
|
|
1590
|
+
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1591
|
+
if (!pr) {
|
|
1592
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1593
|
+
}
|
|
1594
|
+
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1595
|
+
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1596
|
+
const gateCheckName = project?.gateChecks?.find((entry) => entry.trim())?.trim() ?? "verify";
|
|
1597
|
+
const headAdvanced = Boolean(pr.headRefOid && pr.headRefOid !== issue.lastGitHubFailureHeadSha);
|
|
1598
|
+
this.upsertIssueIfLeaseHeld(run.projectId, run.linearIssueId, {
|
|
1599
|
+
projectId: run.projectId,
|
|
1600
|
+
linearIssueId: run.linearIssueId,
|
|
1601
|
+
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1602
|
+
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1603
|
+
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1604
|
+
...(headAdvanced
|
|
1605
|
+
? {
|
|
1606
|
+
prCheckStatus: "pending",
|
|
1607
|
+
lastGitHubFailureSource: null,
|
|
1608
|
+
lastGitHubFailureHeadSha: null,
|
|
1609
|
+
lastGitHubFailureSignature: null,
|
|
1610
|
+
lastGitHubFailureCheckName: null,
|
|
1611
|
+
lastGitHubFailureCheckUrl: null,
|
|
1612
|
+
lastGitHubFailureContextJson: null,
|
|
1613
|
+
lastGitHubFailureAt: null,
|
|
1614
|
+
lastQueueIncidentJson: null,
|
|
1615
|
+
lastAttemptedFailureHeadSha: null,
|
|
1616
|
+
lastAttemptedFailureSignature: null,
|
|
1617
|
+
lastGitHubCiSnapshotHeadSha: pr.headRefOid ?? null,
|
|
1618
|
+
lastGitHubCiSnapshotGateCheckName: gateCheckName,
|
|
1619
|
+
lastGitHubCiSnapshotGateCheckStatus: "pending",
|
|
1620
|
+
lastGitHubCiSnapshotJson: null,
|
|
1621
|
+
lastGitHubCiSnapshotSettledAt: null,
|
|
1622
|
+
}
|
|
1623
|
+
: {}),
|
|
1624
|
+
}, "reactive publish refresh");
|
|
1625
|
+
}
|
|
1626
|
+
catch (error) {
|
|
1627
|
+
this.logger.debug({
|
|
1628
|
+
issueKey: issue.issueKey,
|
|
1629
|
+
prNumber: issue.prNumber,
|
|
1630
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1631
|
+
}, "Failed to refresh PR state after reactive publish");
|
|
1632
|
+
}
|
|
1633
|
+
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1634
|
+
}
|
|
1635
|
+
async loadRemotePrState(repoFullName, prNumber) {
|
|
1636
|
+
const { stdout, exitCode } = await execCommand("gh", [
|
|
1637
|
+
"pr", "view", String(prNumber),
|
|
1638
|
+
"--repo", repoFullName,
|
|
1639
|
+
"--json", "headRefOid,state,reviewDecision,mergeStateStatus",
|
|
1640
|
+
], { timeoutMs: 10_000 });
|
|
1641
|
+
if (exitCode !== 0)
|
|
1642
|
+
return undefined;
|
|
1643
|
+
return JSON.parse(stdout);
|
|
1644
|
+
}
|
|
1645
|
+
async resolveReviewFixWakeContext(issue, context, project) {
|
|
1646
|
+
if (isBranchUpkeepRequired(context)) {
|
|
1647
|
+
return context;
|
|
1648
|
+
}
|
|
1649
|
+
if (!issue.prNumber || issue.prState !== "open" || issue.prReviewState !== "changes_requested") {
|
|
1650
|
+
return context;
|
|
1651
|
+
}
|
|
1652
|
+
const repoFullName = project.github?.repoFullName;
|
|
1653
|
+
if (!repoFullName) {
|
|
1654
|
+
return context;
|
|
1655
|
+
}
|
|
1656
|
+
try {
|
|
1657
|
+
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1658
|
+
if (!pr)
|
|
1659
|
+
return context;
|
|
1660
|
+
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1661
|
+
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1662
|
+
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1663
|
+
projectId: issue.projectId,
|
|
1664
|
+
linearIssueId: issue.linearIssueId,
|
|
1665
|
+
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1666
|
+
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1667
|
+
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1668
|
+
}, "review-fix wake refresh");
|
|
1669
|
+
if (nextPrState !== "open")
|
|
1670
|
+
return context;
|
|
1671
|
+
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
1672
|
+
return context;
|
|
1673
|
+
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
1674
|
+
return context;
|
|
1675
|
+
return buildReviewFixBranchUpkeepContext(issue.prNumber, project.github?.baseBranch ?? "main", pr, context);
|
|
1676
|
+
}
|
|
1677
|
+
catch (error) {
|
|
1678
|
+
this.logger.debug({
|
|
1679
|
+
issueKey: issue.issueKey,
|
|
1680
|
+
prNumber: issue.prNumber,
|
|
1681
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1682
|
+
}, "Failed to resolve review-fix wake context");
|
|
1683
|
+
return context;
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
async resolvePostRunFollowUp(run, issue, projectOverride) {
|
|
1687
|
+
if (run.runType !== "review_fix") {
|
|
1688
|
+
return undefined;
|
|
1689
|
+
}
|
|
1690
|
+
if (!issue.prNumber || issue.prState !== "open") {
|
|
1691
|
+
return undefined;
|
|
1692
|
+
}
|
|
1693
|
+
if (issue.prReviewState !== "changes_requested") {
|
|
1694
|
+
return undefined;
|
|
1695
|
+
}
|
|
1696
|
+
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1697
|
+
const repoFullName = project?.github?.repoFullName;
|
|
1698
|
+
if (!repoFullName) {
|
|
1699
|
+
return undefined;
|
|
1700
|
+
}
|
|
1701
|
+
try {
|
|
1702
|
+
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1703
|
+
if (!pr)
|
|
1704
|
+
return undefined;
|
|
1705
|
+
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1706
|
+
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1707
|
+
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1708
|
+
projectId: issue.projectId,
|
|
1709
|
+
linearIssueId: issue.linearIssueId,
|
|
1710
|
+
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1711
|
+
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1712
|
+
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1713
|
+
}, "post-run follow-up refresh");
|
|
1714
|
+
if (nextPrState !== "open")
|
|
1715
|
+
return undefined;
|
|
1716
|
+
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
1717
|
+
return undefined;
|
|
1718
|
+
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
1719
|
+
return undefined;
|
|
1720
|
+
return {
|
|
1721
|
+
pendingRunType: "review_fix",
|
|
1722
|
+
factoryState: "changes_requested",
|
|
1723
|
+
context: buildReviewFixBranchUpkeepContext(issue.prNumber, project?.github?.baseBranch ?? "main", pr),
|
|
1724
|
+
summary: `PR #${issue.prNumber} is still dirty after review fix; queued branch upkeep`,
|
|
1725
|
+
};
|
|
1726
|
+
}
|
|
1727
|
+
catch (error) {
|
|
1728
|
+
this.logger.debug({
|
|
1729
|
+
issueKey: issue.issueKey,
|
|
1730
|
+
prNumber: issue.prNumber,
|
|
1731
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1732
|
+
}, "Failed to resolve post-run PR upkeep");
|
|
1733
|
+
return undefined;
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
async verifyPublishedRunOutcome(run, issue, projectOverride) {
|
|
1737
|
+
if (run.runType !== "implementation") {
|
|
1738
|
+
return undefined;
|
|
1739
|
+
}
|
|
1740
|
+
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1741
|
+
const baseBranch = project?.github?.baseBranch ?? "main";
|
|
1742
|
+
const deliveryMode = resolveImplementationDeliveryMode(issue, undefined, run.promptText);
|
|
1743
|
+
if (deliveryMode === "linear_only") {
|
|
1744
|
+
if (issue.prNumber !== undefined) {
|
|
1745
|
+
return `Planning-only implementation should not open a PR, but PR #${issue.prNumber} was observed`;
|
|
1746
|
+
}
|
|
1747
|
+
return this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
1748
|
+
}
|
|
1749
|
+
if (issue.prNumber && issue.prState && issue.prState !== "closed") {
|
|
1750
|
+
return undefined;
|
|
1751
|
+
}
|
|
1752
|
+
if (project?.github?.repoFullName && issue.branchName) {
|
|
1753
|
+
try {
|
|
1754
|
+
const { stdout, exitCode } = await execCommand("gh", [
|
|
1755
|
+
"pr",
|
|
1756
|
+
"list",
|
|
1757
|
+
"--repo",
|
|
1758
|
+
project.github.repoFullName,
|
|
1759
|
+
"--head",
|
|
1760
|
+
issue.branchName,
|
|
1761
|
+
"--state",
|
|
1762
|
+
"all",
|
|
1763
|
+
"--json",
|
|
1764
|
+
"number,url,state,author,headRefOid",
|
|
1765
|
+
], { timeoutMs: 10_000 });
|
|
1766
|
+
if (exitCode === 0) {
|
|
1767
|
+
const matches = JSON.parse(stdout);
|
|
1768
|
+
const pr = matches[0];
|
|
1769
|
+
if (pr?.number) {
|
|
1770
|
+
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1771
|
+
projectId: issue.projectId,
|
|
1772
|
+
linearIssueId: issue.linearIssueId,
|
|
1773
|
+
prNumber: pr.number,
|
|
1774
|
+
...(pr.url ? { prUrl: pr.url } : {}),
|
|
1775
|
+
...(pr.state ? { prState: pr.state.toLowerCase() } : {}),
|
|
1776
|
+
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1777
|
+
...(pr.author?.login ? { prAuthorLogin: pr.author.login } : {}),
|
|
1778
|
+
}, "published PR verification refresh");
|
|
1779
|
+
return undefined;
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
catch (error) {
|
|
1784
|
+
this.logger.debug({
|
|
1785
|
+
issueKey: issue.issueKey,
|
|
1786
|
+
branchName: issue.branchName,
|
|
1787
|
+
repoFullName: project.github.repoFullName,
|
|
1788
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1789
|
+
}, "Failed to verify published PR state after implementation");
|
|
1790
|
+
}
|
|
1791
|
+
}
|
|
1792
|
+
const details = await this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
1793
|
+
return details ?? `Implementation completed without opening a PR for branch ${issue.branchName ?? issue.linearIssueId}`;
|
|
1794
|
+
}
|
|
1795
|
+
async describeLocalImplementationOutcome(issue, baseBranch, deliveryMode = "publish_pr") {
|
|
1796
|
+
if (!issue.worktreePath) {
|
|
1797
|
+
return undefined;
|
|
1798
|
+
}
|
|
1799
|
+
try {
|
|
1800
|
+
const status = await execCommand(this.config.runner.gitBin, [
|
|
1801
|
+
"-C",
|
|
1802
|
+
issue.worktreePath,
|
|
1803
|
+
"status",
|
|
1804
|
+
"--short",
|
|
1805
|
+
], { timeoutMs: 10_000 });
|
|
1806
|
+
const dirtyEntries = status.exitCode === 0
|
|
1807
|
+
? status.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
1808
|
+
: [];
|
|
1809
|
+
if (dirtyEntries.length > 0) {
|
|
1810
|
+
if (deliveryMode === "linear_only") {
|
|
1811
|
+
return `Planning-only implementation should not modify the repo; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1812
|
+
}
|
|
1813
|
+
return `Implementation completed without opening a PR; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1814
|
+
}
|
|
1815
|
+
}
|
|
1816
|
+
catch {
|
|
1817
|
+
// Best effort only.
|
|
1818
|
+
}
|
|
1819
|
+
try {
|
|
1820
|
+
const ahead = await execCommand(this.config.runner.gitBin, [
|
|
1821
|
+
"-C",
|
|
1822
|
+
issue.worktreePath,
|
|
1823
|
+
"rev-list",
|
|
1824
|
+
"--count",
|
|
1825
|
+
`origin/${baseBranch}..HEAD`,
|
|
1826
|
+
], { timeoutMs: 10_000 });
|
|
1827
|
+
if (ahead.exitCode === 0) {
|
|
1828
|
+
const count = Number(ahead.stdout.trim());
|
|
1829
|
+
if (Number.isFinite(count) && count > 0) {
|
|
1830
|
+
if (deliveryMode === "linear_only") {
|
|
1831
|
+
return `Planning-only implementation should not create repo commits; worktree is ${count} local commit(s) ahead of origin/${baseBranch}`;
|
|
1832
|
+
}
|
|
1833
|
+
return `Implementation completed with ${count} local commit(s) ahead of origin/${baseBranch} but no PR was observed`;
|
|
1834
|
+
}
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
catch {
|
|
1838
|
+
// Best effort only.
|
|
1839
|
+
}
|
|
1840
|
+
return undefined;
|
|
1841
|
+
}
|
|
885
1842
|
async readThreadWithRetry(threadId, maxRetries = 3) {
|
|
886
1843
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
887
1844
|
try {
|
|
@@ -895,6 +1852,151 @@ export class RunOrchestrator {
|
|
|
895
1852
|
}
|
|
896
1853
|
throw new Error(`Failed to read thread ${threadId}`);
|
|
897
1854
|
}
|
|
1855
|
+
issueSessionLeaseKey(projectId, linearIssueId) {
|
|
1856
|
+
return `${projectId}:${linearIssueId}`;
|
|
1857
|
+
}
|
|
1858
|
+
getHeldIssueSessionLease(projectId, linearIssueId) {
|
|
1859
|
+
const leaseId = this.activeSessionLeases.get(this.issueSessionLeaseKey(projectId, linearIssueId));
|
|
1860
|
+
if (!leaseId)
|
|
1861
|
+
return undefined;
|
|
1862
|
+
return { projectId, linearIssueId, leaseId };
|
|
1863
|
+
}
|
|
1864
|
+
withHeldIssueSessionLease(projectId, linearIssueId, fn) {
|
|
1865
|
+
const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
|
|
1866
|
+
if (!lease)
|
|
1867
|
+
return undefined;
|
|
1868
|
+
return this.db.withIssueSessionLease(projectId, linearIssueId, lease.leaseId, () => fn(lease));
|
|
1869
|
+
}
|
|
1870
|
+
upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
|
|
1871
|
+
const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
|
|
1872
|
+
if (!lease) {
|
|
1873
|
+
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write without a held issue-session lease");
|
|
1874
|
+
return undefined;
|
|
1875
|
+
}
|
|
1876
|
+
const updated = this.db.upsertIssueWithLease(lease, params);
|
|
1877
|
+
if (!updated) {
|
|
1878
|
+
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
|
|
1879
|
+
}
|
|
1880
|
+
return updated;
|
|
1881
|
+
}
|
|
1882
|
+
assertLaunchLease(run, phase) {
|
|
1883
|
+
if (this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
1884
|
+
return;
|
|
1885
|
+
}
|
|
1886
|
+
const error = new Error(`Lost issue-session lease ${phase}`);
|
|
1887
|
+
error.name = "IssueSessionLeaseLostError";
|
|
1888
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId, phase }, "Aborting run launch after losing issue-session lease");
|
|
1889
|
+
throw error;
|
|
1890
|
+
}
|
|
1891
|
+
acquireIssueSessionLease(projectId, linearIssueId) {
|
|
1892
|
+
const leaseId = randomUUID();
|
|
1893
|
+
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
1894
|
+
const acquired = this.db.acquireIssueSessionLease({
|
|
1895
|
+
projectId,
|
|
1896
|
+
linearIssueId,
|
|
1897
|
+
leaseId,
|
|
1898
|
+
workerId: this.workerId,
|
|
1899
|
+
leasedUntil,
|
|
1900
|
+
});
|
|
1901
|
+
if (!acquired)
|
|
1902
|
+
return undefined;
|
|
1903
|
+
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
1904
|
+
return leaseId;
|
|
1905
|
+
}
|
|
1906
|
+
forceAcquireIssueSessionLease(projectId, linearIssueId) {
|
|
1907
|
+
const leaseId = randomUUID();
|
|
1908
|
+
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
1909
|
+
const acquired = this.db.forceAcquireIssueSessionLease({
|
|
1910
|
+
projectId,
|
|
1911
|
+
linearIssueId,
|
|
1912
|
+
leaseId,
|
|
1913
|
+
workerId: this.workerId,
|
|
1914
|
+
leasedUntil,
|
|
1915
|
+
});
|
|
1916
|
+
if (!acquired)
|
|
1917
|
+
return undefined;
|
|
1918
|
+
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
1919
|
+
return leaseId;
|
|
1920
|
+
}
|
|
1921
|
+
claimLeaseForReconciliation(projectId, linearIssueId) {
|
|
1922
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
1923
|
+
if (this.activeSessionLeases.has(key)) {
|
|
1924
|
+
return "owned";
|
|
1925
|
+
}
|
|
1926
|
+
const session = this.db.getIssueSession(projectId, linearIssueId);
|
|
1927
|
+
if (!session)
|
|
1928
|
+
return "skip";
|
|
1929
|
+
const leasedUntilMs = session.leasedUntil ? Date.parse(session.leasedUntil) : undefined;
|
|
1930
|
+
if (leasedUntilMs !== undefined && Number.isFinite(leasedUntilMs) && leasedUntilMs > Date.now()) {
|
|
1931
|
+
return "skip";
|
|
1932
|
+
}
|
|
1933
|
+
return this.acquireIssueSessionLease(projectId, linearIssueId) ? true : "skip";
|
|
1934
|
+
}
|
|
1935
|
+
async reclaimForeignRecoveryLeaseIfSafe(run, issue) {
|
|
1936
|
+
const key = this.issueSessionLeaseKey(run.projectId, run.linearIssueId);
|
|
1937
|
+
if (this.activeSessionLeases.has(key)) {
|
|
1938
|
+
return false;
|
|
1939
|
+
}
|
|
1940
|
+
const session = this.db.getIssueSession(run.projectId, run.linearIssueId);
|
|
1941
|
+
if (!session?.leaseId || !session.workerId || session.workerId === this.workerId) {
|
|
1942
|
+
return false;
|
|
1943
|
+
}
|
|
1944
|
+
if (issue.activeRunId !== run.id) {
|
|
1945
|
+
return false;
|
|
1946
|
+
}
|
|
1947
|
+
let safeToReclaim = !run.threadId;
|
|
1948
|
+
if (!safeToReclaim && run.threadId) {
|
|
1949
|
+
try {
|
|
1950
|
+
const thread = await this.readThreadWithRetry(run.threadId, 1);
|
|
1951
|
+
const latestTurn = getThreadTurns(thread).at(-1);
|
|
1952
|
+
safeToReclaim = thread.status === "notLoaded"
|
|
1953
|
+
|| latestTurn?.status === "interrupted"
|
|
1954
|
+
|| latestTurn?.status === "completed";
|
|
1955
|
+
}
|
|
1956
|
+
catch {
|
|
1957
|
+
safeToReclaim = true;
|
|
1958
|
+
}
|
|
1959
|
+
}
|
|
1960
|
+
if (!safeToReclaim) {
|
|
1961
|
+
return false;
|
|
1962
|
+
}
|
|
1963
|
+
const leaseId = this.forceAcquireIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1964
|
+
if (!leaseId) {
|
|
1965
|
+
return false;
|
|
1966
|
+
}
|
|
1967
|
+
this.logger.info({
|
|
1968
|
+
issueKey: issue.issueKey,
|
|
1969
|
+
runId: run.id,
|
|
1970
|
+
previousWorkerId: session.workerId,
|
|
1971
|
+
previousLeaseId: session.leaseId,
|
|
1972
|
+
reclaimedLeaseId: leaseId,
|
|
1973
|
+
}, "Reclaimed foreign issue-session lease for active-run recovery");
|
|
1974
|
+
return true;
|
|
1975
|
+
}
|
|
1976
|
+
heartbeatIssueSessionLease(projectId, linearIssueId) {
|
|
1977
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
1978
|
+
const leaseId = this.activeSessionLeases.get(key) ?? this.db.getIssueSession(projectId, linearIssueId)?.leaseId;
|
|
1979
|
+
if (!leaseId)
|
|
1980
|
+
return false;
|
|
1981
|
+
const renewed = this.db.renewIssueSessionLease({
|
|
1982
|
+
projectId,
|
|
1983
|
+
linearIssueId,
|
|
1984
|
+
leaseId,
|
|
1985
|
+
leasedUntil: new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString(),
|
|
1986
|
+
});
|
|
1987
|
+
if (renewed) {
|
|
1988
|
+
this.activeSessionLeases.set(key, leaseId);
|
|
1989
|
+
return true;
|
|
1990
|
+
}
|
|
1991
|
+
this.activeSessionLeases.delete(key);
|
|
1992
|
+
return false;
|
|
1993
|
+
}
|
|
1994
|
+
releaseIssueSessionLease(projectId, linearIssueId) {
|
|
1995
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
1996
|
+
const leaseId = this.activeSessionLeases.get(key);
|
|
1997
|
+
this.db.releaseIssueSessionLease(projectId, linearIssueId, leaseId);
|
|
1998
|
+
this.activeSessionLeases.delete(key);
|
|
1999
|
+
}
|
|
898
2000
|
}
|
|
899
2001
|
/**
|
|
900
2002
|
* Determine post-run factory state from current PR metadata.
|
|
@@ -912,6 +2014,12 @@ function resolvePostRunState(issue) {
|
|
|
912
2014
|
}
|
|
913
2015
|
return undefined;
|
|
914
2016
|
}
|
|
2017
|
+
function resolveCompletedRunState(issue, run) {
|
|
2018
|
+
if (run.runType === "implementation" && resolveImplementationDeliveryMode(issue, undefined, run.promptText) === "linear_only") {
|
|
2019
|
+
return "done";
|
|
2020
|
+
}
|
|
2021
|
+
return resolvePostRunState(issue);
|
|
2022
|
+
}
|
|
915
2023
|
function resolveRecoverablePostRunState(issue) {
|
|
916
2024
|
if (!issue.prNumber) {
|
|
917
2025
|
return resolvePostRunState(issue);
|
|
@@ -919,18 +2027,59 @@ function resolveRecoverablePostRunState(issue) {
|
|
|
919
2027
|
if (issue.prState === "merged")
|
|
920
2028
|
return "done";
|
|
921
2029
|
if (issue.prState === "open") {
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
2030
|
+
const reactiveIntent = deriveIssueSessionReactiveIntent({
|
|
2031
|
+
prNumber: issue.prNumber,
|
|
2032
|
+
prState: issue.prState,
|
|
2033
|
+
prReviewState: issue.prReviewState,
|
|
2034
|
+
prCheckStatus: issue.prCheckStatus,
|
|
2035
|
+
latestFailureSource: issue.lastGitHubFailureSource,
|
|
2036
|
+
});
|
|
2037
|
+
if (reactiveIntent)
|
|
2038
|
+
return reactiveIntent.compatibilityFactoryState;
|
|
928
2039
|
if (issue.prReviewState === "approved")
|
|
929
2040
|
return "awaiting_queue";
|
|
930
2041
|
return "pr_open";
|
|
931
2042
|
}
|
|
932
2043
|
return resolvePostRunState(issue);
|
|
933
2044
|
}
|
|
2045
|
+
function normalizeRemotePrState(value) {
|
|
2046
|
+
const normalized = value?.trim().toUpperCase();
|
|
2047
|
+
if (normalized === "OPEN")
|
|
2048
|
+
return "open";
|
|
2049
|
+
if (normalized === "CLOSED")
|
|
2050
|
+
return "closed";
|
|
2051
|
+
if (normalized === "MERGED")
|
|
2052
|
+
return "merged";
|
|
2053
|
+
return undefined;
|
|
2054
|
+
}
|
|
2055
|
+
function normalizeRemoteReviewDecision(value) {
|
|
2056
|
+
const normalized = value?.trim().toUpperCase();
|
|
2057
|
+
if (normalized === "APPROVED")
|
|
2058
|
+
return "approved";
|
|
2059
|
+
if (normalized === "CHANGES_REQUESTED")
|
|
2060
|
+
return "changes_requested";
|
|
2061
|
+
if (normalized === "REVIEW_REQUIRED")
|
|
2062
|
+
return "commented";
|
|
2063
|
+
return undefined;
|
|
2064
|
+
}
|
|
2065
|
+
function isDirtyMergeStateStatus(value) {
|
|
2066
|
+
return value?.trim().toUpperCase() === "DIRTY";
|
|
2067
|
+
}
|
|
2068
|
+
function buildReviewFixBranchUpkeepContext(prNumber, baseBranch, pr, context) {
|
|
2069
|
+
const promptContext = [
|
|
2070
|
+
`The requested review change is already addressed, but GitHub still reports PR #${prNumber} as ${String(pr.mergeStateStatus)} against latest ${baseBranch}.`,
|
|
2071
|
+
`Before stopping, update the existing PR branch onto latest ${baseBranch}, resolve any conflicts, rerun the narrowest relevant verification, and push again.`,
|
|
2072
|
+
"Do not stop just because the requested code change is already present.",
|
|
2073
|
+
].join(" ");
|
|
2074
|
+
return {
|
|
2075
|
+
...(context ?? {}),
|
|
2076
|
+
branchUpkeepRequired: true,
|
|
2077
|
+
promptContext,
|
|
2078
|
+
...(pr.mergeStateStatus ? { mergeStateStatus: pr.mergeStateStatus } : {}),
|
|
2079
|
+
...(pr.headRefOid ? { failingHeadSha: pr.headRefOid } : {}),
|
|
2080
|
+
baseBranch,
|
|
2081
|
+
};
|
|
2082
|
+
}
|
|
934
2083
|
function appendQueueRepairContext(lines, context) {
|
|
935
2084
|
const incidentTitle = typeof context?.incidentTitle === "string" ? context.incidentTitle.trim() : "";
|
|
936
2085
|
const incidentSummary = typeof context?.incidentSummary === "string" ? context.incidentSummary.trim() : "";
|