patchrelay 0.36.7 → 0.36.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/cluster-health.js +1 -1
- package/dist/cli/data.js +11 -11
- package/dist/db/issue-session-store.js +292 -0
- package/dist/db/run-store.js +127 -0
- package/dist/db/webhook-event-store.js +71 -0
- package/dist/db.js +22 -520
- package/dist/github-webhook-handler.js +25 -25
- package/dist/idle-reconciliation.js +5 -5
- package/dist/issue-query-service.js +9 -9
- package/dist/issue-session-lease-service.js +143 -0
- package/dist/linear-session-sync.js +4 -4
- package/dist/queue-health-monitor.js +2 -2
- package/dist/run-finalizer.js +161 -0
- package/dist/run-launcher.js +193 -0
- package/dist/run-orchestrator.js +148 -856
- package/dist/run-recovery-service.js +203 -0
- package/dist/run-wake-planner.js +101 -0
- package/dist/service.js +24 -24
- package/dist/tracked-issue-projector.js +69 -0
- package/dist/webhook-handler.js +59 -688
- package/dist/webhooks/agent-session-handler.js +212 -0
- package/dist/webhooks/comment-policy.js +41 -0
- package/dist/webhooks/comment-wake-handler.js +133 -0
- package/dist/webhooks/decision-helpers.js +74 -0
- package/dist/webhooks/desired-stage-recorder.js +177 -0
- package/dist/webhooks/issue-removal-handler.js +68 -0
- package/package.json +1 -1
package/dist/run-orchestrator.js
CHANGED
|
@@ -1,37 +1,20 @@
|
|
|
1
|
-
import { randomUUID } from "node:crypto";
|
|
2
1
|
import { ACTIVE_RUN_STATES, TERMINAL_STATES } from "./factory-state.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { buildRunCompletedActivity, buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
2
|
+
import { extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread } from "./run-reporting.js";
|
|
3
|
+
import { buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
6
4
|
import { WorktreeManager } from "./worktree-manager.js";
|
|
7
5
|
import { resolveAuthoritativeLinearStopState, resolvePreferredCompletedLinearState } from "./linear-workflow.js";
|
|
8
6
|
import { execCommand } from "./utils.js";
|
|
9
7
|
import { getThreadTurns } from "./codex-thread-utils.js";
|
|
10
8
|
import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
11
|
-
import { loadPatchRelayRepoPrompting } from "./patchrelay-customization.js";
|
|
12
|
-
import { buildRunPrompt as buildPatchRelayRunPrompt, findDisallowedPatchRelayPromptSectionIds, findUnknownPatchRelayPromptSectionIds, mergePromptCustomizationLayers, resolveImplementationDeliveryMode, resolvePromptLayers, } from "./prompting/patchrelay.js";
|
|
13
|
-
const DEFAULT_CI_REPAIR_BUDGET = 3;
|
|
14
|
-
const DEFAULT_QUEUE_REPAIR_BUDGET = 3;
|
|
15
|
-
// Requested-changes loops can legitimately take more iterations than CI/queue
|
|
16
|
-
// repair when the reviewer is catching nuanced product or timing bugs across
|
|
17
|
-
// successive heads. Keep a hard ceiling to prevent infinite ping-pong, but make
|
|
18
|
-
// it wide enough that real review cycles can continue after multiple successful
|
|
19
|
-
// head advances.
|
|
20
|
-
const DEFAULT_REVIEW_FIX_BUDGET = 12;
|
|
21
|
-
const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
|
|
22
|
-
const ZOMBIE_RECOVERY_BASE_DELAY_MS = 15_000; // 15s, 30s, 60s, 120s, 240s
|
|
23
|
-
const ISSUE_SESSION_LEASE_MS = 10 * 60_000;
|
|
24
|
-
const MAX_THREAD_GENERATION_BEFORE_COMPACTION = 4;
|
|
25
|
-
const MAX_FOLLOW_UPS_BEFORE_COMPACTION = 4;
|
|
26
9
|
import { QueueHealthMonitor } from "./queue-health-monitor.js";
|
|
10
|
+
import { resolveImplementationDeliveryMode, } from "./prompting/patchrelay.js";
|
|
27
11
|
import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
|
|
28
12
|
import { LinearSessionSync } from "./linear-session-sync.js";
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
}
|
|
13
|
+
import { IssueSessionLeaseService } from "./issue-session-lease-service.js";
|
|
14
|
+
import { RunFinalizer } from "./run-finalizer.js";
|
|
15
|
+
import { RunLauncher } from "./run-launcher.js";
|
|
16
|
+
import { RunRecoveryService } from "./run-recovery-service.js";
|
|
17
|
+
import { RunWakePlanner } from "./run-wake-planner.js";
|
|
35
18
|
function lowerCaseFirst(value) {
|
|
36
19
|
return value ? `${value.slice(0, 1).toLowerCase()}${value.slice(1)}` : value;
|
|
37
20
|
}
|
|
@@ -46,15 +29,6 @@ function resolveRequestedChangesMode(runType, context) {
|
|
|
46
29
|
? "branch_upkeep"
|
|
47
30
|
: "address_review_feedback";
|
|
48
31
|
}
|
|
49
|
-
function shouldCompactThread(issue, threadGeneration, context) {
|
|
50
|
-
const followUpCount = typeof context?.followUpCount === "number" ? context.followUpCount : 0;
|
|
51
|
-
return issue.threadId !== undefined
|
|
52
|
-
&& (threadGeneration ?? 0) >= MAX_THREAD_GENERATION_BEFORE_COMPACTION
|
|
53
|
-
&& followUpCount >= MAX_FOLLOW_UPS_BEFORE_COMPACTION;
|
|
54
|
-
}
|
|
55
|
-
export function shouldReuseIssueThread(params) {
|
|
56
|
-
return Boolean(params.existingThreadId) && !params.compactThread && params.resumeThread;
|
|
57
|
-
}
|
|
58
32
|
function isBranchUpkeepRequired(context) {
|
|
59
33
|
return context?.branchUpkeepRequired === true;
|
|
60
34
|
}
|
|
@@ -73,7 +47,12 @@ export class RunOrchestrator {
|
|
|
73
47
|
linearSync;
|
|
74
48
|
activeThreadId;
|
|
75
49
|
workerId = `patchrelay:${process.pid}`;
|
|
76
|
-
|
|
50
|
+
leaseService;
|
|
51
|
+
runFinalizer;
|
|
52
|
+
runLauncher;
|
|
53
|
+
runRecovery;
|
|
54
|
+
runWakePlanner;
|
|
55
|
+
activeSessionLeases;
|
|
77
56
|
botIdentity;
|
|
78
57
|
constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
|
|
79
58
|
this.config = config;
|
|
@@ -85,6 +64,12 @@ export class RunOrchestrator {
|
|
|
85
64
|
this.feed = feed;
|
|
86
65
|
this.worktreeManager = new WorktreeManager(config);
|
|
87
66
|
this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
|
|
67
|
+
this.leaseService = new IssueSessionLeaseService(db, logger, this.workerId, (threadId, maxRetries) => this.readThreadWithRetry(threadId, maxRetries));
|
|
68
|
+
this.activeSessionLeases = this.leaseService.activeSessionLeases;
|
|
69
|
+
this.runFinalizer = new RunFinalizer(db, logger, this.linearSync, this.enqueueIssue, feed);
|
|
70
|
+
this.runLauncher = new RunLauncher(config, db, codex, logger, this.worktreeManager);
|
|
71
|
+
this.runRecovery = new RunRecoveryService(db, logger, this.linearSync, (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (projectId, issueId) => this.enqueueIssue(projectId, issueId), (newState, pendingRunType) => this.resolveBranchOwnerForStateTransition(newState, pendingRunType), feed);
|
|
72
|
+
this.runWakePlanner = new RunWakePlanner(db);
|
|
88
73
|
this.idleReconciler = new IdleIssueReconciler(db, config, {
|
|
89
74
|
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
90
75
|
}, logger, feed);
|
|
@@ -94,88 +79,40 @@ export class RunOrchestrator {
|
|
|
94
79
|
}, logger, feed);
|
|
95
80
|
}
|
|
96
81
|
resolveRunWake(issue) {
|
|
97
|
-
|
|
98
|
-
if (sessionWake) {
|
|
99
|
-
return {
|
|
100
|
-
runType: sessionWake.runType,
|
|
101
|
-
context: sessionWake.context,
|
|
102
|
-
wakeReason: sessionWake.wakeReason,
|
|
103
|
-
resumeThread: sessionWake.resumeThread,
|
|
104
|
-
eventIds: sessionWake.eventIds,
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
return undefined;
|
|
82
|
+
return this.runWakePlanner.resolveRunWake(issue);
|
|
108
83
|
}
|
|
109
84
|
appendWakeEventWithLease(lease, issue, runType, context, dedupeScope) {
|
|
110
|
-
|
|
111
|
-
let dedupeKey;
|
|
112
|
-
if (runType === "queue_repair") {
|
|
113
|
-
eventType = "merge_steward_incident";
|
|
114
|
-
dedupeKey = `${dedupeScope ?? "wake"}:queue_repair:${issue.linearIssueId}:${issue.prHeadSha ?? issue.lastGitHubFailureHeadSha ?? "unknown-sha"}`;
|
|
115
|
-
}
|
|
116
|
-
else if (runType === "ci_repair") {
|
|
117
|
-
eventType = "settled_red_ci";
|
|
118
|
-
dedupeKey = `${dedupeScope ?? "wake"}:ci_repair:${issue.linearIssueId}:${issue.lastGitHubFailureSignature ?? issue.prHeadSha ?? "unknown-sha"}`;
|
|
119
|
-
}
|
|
120
|
-
else if (runType === "review_fix" || runType === "branch_upkeep") {
|
|
121
|
-
eventType = "review_changes_requested";
|
|
122
|
-
dedupeKey = `${dedupeScope ?? "wake"}:${runType}:${issue.linearIssueId}:${issue.prHeadSha ?? "unknown-sha"}`;
|
|
123
|
-
}
|
|
124
|
-
else {
|
|
125
|
-
eventType = "delegated";
|
|
126
|
-
dedupeKey = `${dedupeScope ?? "wake"}:implementation:${issue.linearIssueId}`;
|
|
127
|
-
}
|
|
128
|
-
return Boolean(this.db.appendIssueSessionEventWithLease(lease, {
|
|
129
|
-
projectId: issue.projectId,
|
|
130
|
-
linearIssueId: issue.linearIssueId,
|
|
131
|
-
eventType,
|
|
132
|
-
...(context ? { eventJson: JSON.stringify(context) } : {}),
|
|
133
|
-
dedupeKey,
|
|
134
|
-
}));
|
|
85
|
+
return this.runWakePlanner.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope);
|
|
135
86
|
}
|
|
136
87
|
materializeLegacyPendingWake(issue, lease) {
|
|
137
|
-
|
|
138
|
-
return issue;
|
|
139
|
-
const context = issue.pendingRunContextJson
|
|
140
|
-
? JSON.parse(issue.pendingRunContextJson)
|
|
141
|
-
: undefined;
|
|
142
|
-
this.appendWakeEventWithLease(lease, issue, issue.pendingRunType, context, "legacy_pending");
|
|
143
|
-
const updated = this.db.upsertIssueWithLease(lease, {
|
|
144
|
-
projectId: issue.projectId,
|
|
145
|
-
linearIssueId: issue.linearIssueId,
|
|
146
|
-
pendingRunType: null,
|
|
147
|
-
pendingRunContextJson: null,
|
|
148
|
-
});
|
|
149
|
-
if (!updated)
|
|
150
|
-
return issue;
|
|
151
|
-
return this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
|
|
88
|
+
return this.runWakePlanner.materializeLegacyPendingWake(issue, lease);
|
|
152
89
|
}
|
|
153
90
|
// ─── Run ────────────────────────────────────────────────────────
|
|
154
91
|
async run(item) {
|
|
155
92
|
const project = this.config.projects.find((p) => p.id === item.projectId);
|
|
156
93
|
if (!project)
|
|
157
94
|
return;
|
|
158
|
-
if (this.
|
|
95
|
+
if (this.leaseService.hasLocalLease(item.projectId, item.issueId)) {
|
|
159
96
|
return;
|
|
160
97
|
}
|
|
161
98
|
const issue = this.db.getIssue(item.projectId, item.issueId);
|
|
162
99
|
if (!issue || issue.activeRunId !== undefined)
|
|
163
100
|
return;
|
|
164
|
-
const issueSession = this.db.getIssueSession(item.projectId, item.issueId);
|
|
165
|
-
const leaseId = this.
|
|
101
|
+
const issueSession = this.db.issueSessions.getIssueSession(item.projectId, item.issueId);
|
|
102
|
+
const leaseId = this.leaseService.acquire(item.projectId, item.issueId);
|
|
166
103
|
if (!leaseId) {
|
|
167
104
|
this.logger.info({ issueKey: issue.issueKey, projectId: item.projectId }, "Skipped run because another worker holds the session lease");
|
|
168
105
|
return;
|
|
169
106
|
}
|
|
170
107
|
if (issue.prState === "merged") {
|
|
171
|
-
this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
|
|
172
|
-
this.
|
|
108
|
+
this.db.issueSessions.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
|
|
109
|
+
this.leaseService.release(item.projectId, item.issueId);
|
|
173
110
|
return;
|
|
174
111
|
}
|
|
175
112
|
const wakeIssue = this.materializeLegacyPendingWake(issue, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
|
|
176
113
|
const wake = this.resolveRunWake(wakeIssue);
|
|
177
114
|
if (!wake) {
|
|
178
|
-
this.
|
|
115
|
+
this.leaseService.release(item.projectId, item.issueId);
|
|
179
116
|
return;
|
|
180
117
|
}
|
|
181
118
|
const { runType, context, resumeThread } = wake;
|
|
@@ -187,112 +124,33 @@ export class RunOrchestrator {
|
|
|
187
124
|
: typeof effectiveContext?.headSha === "string"
|
|
188
125
|
? effectiveContext.headSha
|
|
189
126
|
: issue.prHeadSha;
|
|
190
|
-
|
|
191
|
-
if (
|
|
192
|
-
this.escalate(issue, runType,
|
|
193
|
-
return;
|
|
194
|
-
}
|
|
195
|
-
if (runType === "queue_repair" && issue.queueRepairAttempts >= DEFAULT_QUEUE_REPAIR_BUDGET) {
|
|
196
|
-
this.escalate(issue, runType, `Queue repair budget exhausted (${DEFAULT_QUEUE_REPAIR_BUDGET} attempts)`);
|
|
127
|
+
const budgetExceeded = this.runWakePlanner.budgetExceeded(issue, runType, isRequestedChangesRunType);
|
|
128
|
+
if (budgetExceeded) {
|
|
129
|
+
this.escalate(issue, runType, budgetExceeded);
|
|
197
130
|
return;
|
|
198
131
|
}
|
|
199
|
-
if (
|
|
200
|
-
this.
|
|
132
|
+
if (!this.runWakePlanner.incrementAttemptCounters(issue, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, runType, isRequestedChangesRunType)) {
|
|
133
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
201
134
|
return;
|
|
202
135
|
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, ciRepairAttempts: issue.ciRepairAttempts + 1 });
|
|
206
|
-
if (!updated) {
|
|
207
|
-
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
208
|
-
return;
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
if (runType === "queue_repair") {
|
|
212
|
-
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueRepairAttempts: issue.queueRepairAttempts + 1 });
|
|
213
|
-
if (!updated) {
|
|
214
|
-
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
215
|
-
return;
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
if (isRequestedChangesRunType(runType)) {
|
|
219
|
-
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, reviewFixAttempts: issue.reviewFixAttempts + 1 });
|
|
220
|
-
if (!updated) {
|
|
221
|
-
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
222
|
-
return;
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
const repoPrompting = loadPatchRelayRepoPrompting({
|
|
226
|
-
repoRoot: project.repoPath,
|
|
227
|
-
logger: this.logger,
|
|
228
|
-
});
|
|
229
|
-
const promptLayer = mergePromptCustomizationLayers(resolvePromptLayers(this.config.prompting, runType), resolvePromptLayers(repoPrompting, runType));
|
|
230
|
-
const unknownPromptSections = findUnknownPatchRelayPromptSectionIds(promptLayer);
|
|
231
|
-
if (unknownPromptSections.length > 0) {
|
|
232
|
-
this.logger.warn({ issueKey: issue.issueKey, runType, unknownPromptSections }, "PatchRelay prompt customization references unknown section ids");
|
|
233
|
-
}
|
|
234
|
-
const disallowedPromptSections = findDisallowedPatchRelayPromptSectionIds(promptLayer);
|
|
235
|
-
if (disallowedPromptSections.length > 0) {
|
|
236
|
-
this.logger.warn({ issueKey: issue.issueKey, runType, disallowedPromptSections }, "PatchRelay prompt customization attempted to replace non-overridable sections");
|
|
237
|
-
}
|
|
238
|
-
const prompt = buildPatchRelayRunPrompt({
|
|
136
|
+
const { prompt, branchName, worktreePath } = this.runLauncher.prepareLaunchPlan({
|
|
137
|
+
project,
|
|
239
138
|
issue,
|
|
240
139
|
runType,
|
|
241
|
-
|
|
242
|
-
...(effectiveContext ? { context: effectiveContext } : {}),
|
|
243
|
-
...(promptLayer ? { promptLayer } : {}),
|
|
140
|
+
...(effectiveContext ? { effectiveContext } : {}),
|
|
244
141
|
});
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
const freshWake = this.resolveRunWake(wakeIssue);
|
|
258
|
-
if (!freshWake || freshWake.runType !== runType)
|
|
259
|
-
return undefined;
|
|
260
|
-
const created = this.db.createRun({
|
|
261
|
-
issueId: fresh.id,
|
|
262
|
-
projectId: item.projectId,
|
|
263
|
-
linearIssueId: item.issueId,
|
|
264
|
-
runType,
|
|
265
|
-
...(sourceHeadSha ? { sourceHeadSha } : {}),
|
|
266
|
-
promptText: prompt,
|
|
267
|
-
});
|
|
268
|
-
const failureHeadSha = typeof effectiveContext?.failureHeadSha === "string"
|
|
269
|
-
? effectiveContext.failureHeadSha
|
|
270
|
-
: typeof effectiveContext?.headSha === "string" ? effectiveContext.headSha : undefined;
|
|
271
|
-
const failureSignature = typeof effectiveContext?.failureSignature === "string" ? effectiveContext.failureSignature : undefined;
|
|
272
|
-
this.db.upsertIssue({
|
|
273
|
-
projectId: item.projectId,
|
|
274
|
-
linearIssueId: item.issueId,
|
|
275
|
-
pendingRunType: null,
|
|
276
|
-
pendingRunContextJson: null,
|
|
277
|
-
activeRunId: created.id,
|
|
278
|
-
branchName,
|
|
279
|
-
worktreePath,
|
|
280
|
-
factoryState: runType === "implementation" ? "implementing"
|
|
281
|
-
: runType === "ci_repair" ? "repairing_ci"
|
|
282
|
-
: runType === "review_fix" || runType === "branch_upkeep" ? "changes_requested"
|
|
283
|
-
: runType === "queue_repair" ? "repairing_queue"
|
|
284
|
-
: "implementing",
|
|
285
|
-
...((runType === "ci_repair" || runType === "queue_repair") && failureSignature
|
|
286
|
-
? {
|
|
287
|
-
lastAttemptedFailureSignature: failureSignature,
|
|
288
|
-
lastAttemptedFailureHeadSha: failureHeadSha ?? null,
|
|
289
|
-
}
|
|
290
|
-
: {}),
|
|
291
|
-
});
|
|
292
|
-
this.db.consumeIssueSessionEvents(item.projectId, item.issueId, freshWake.eventIds, created.id);
|
|
293
|
-
this.db.setIssueSessionLastWakeReason(item.projectId, item.issueId, freshWake.wakeReason ?? null);
|
|
294
|
-
this.db.setBranchOwnerWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, "patchrelay");
|
|
295
|
-
return created;
|
|
142
|
+
const run = this.runLauncher.claimRun({
|
|
143
|
+
item,
|
|
144
|
+
issue,
|
|
145
|
+
leaseId,
|
|
146
|
+
runType,
|
|
147
|
+
prompt,
|
|
148
|
+
...(sourceHeadSha ? { sourceHeadSha } : {}),
|
|
149
|
+
...(effectiveContext ? { effectiveContext } : {}),
|
|
150
|
+
materializeLegacyPendingWake: (targetIssue, lease) => this.materializeLegacyPendingWake(targetIssue, lease),
|
|
151
|
+
resolveRunWake: (targetIssue) => this.resolveRunWake(targetIssue),
|
|
152
|
+
branchName,
|
|
153
|
+
worktreePath,
|
|
296
154
|
});
|
|
297
155
|
if (!run) {
|
|
298
156
|
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
@@ -307,110 +165,36 @@ export class RunOrchestrator {
|
|
|
307
165
|
status: "starting",
|
|
308
166
|
summary: `Starting ${runType} run`,
|
|
309
167
|
});
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
// and avoids scope-bundling review rejections from stale commits.
|
|
332
|
-
// Skip for queue_repair — its entire purpose is to resolve rebase conflicts.
|
|
333
|
-
if (runType !== "queue_repair") {
|
|
334
|
-
await this.freshenWorktree(worktreePath, project, issue);
|
|
335
|
-
}
|
|
336
|
-
// Run prepare-worktree hook
|
|
337
|
-
const hookEnv = buildHookEnv(issue.issueKey ?? issue.linearIssueId, branchName, runType, worktreePath);
|
|
338
|
-
const prepareResult = await runProjectHook(project.repoPath, "prepare-worktree", { cwd: worktreePath, env: hookEnv });
|
|
339
|
-
if (prepareResult.ran && prepareResult.exitCode !== 0) {
|
|
340
|
-
throw new Error(`prepare-worktree hook failed (exit ${prepareResult.exitCode}): ${prepareResult.stderr?.slice(0, 500) ?? ""}`);
|
|
341
|
-
}
|
|
342
|
-
this.assertLaunchLease(run, "before starting the Codex turn");
|
|
343
|
-
// Reuse the existing thread only for additive follow-ups that explicitly
|
|
344
|
-
// request continuity. Fresh review-fix runs now start a new thread so the
|
|
345
|
-
// model is not anchored to the implementation conversation that produced
|
|
346
|
-
// the rejected patch. If the thread has accumulated many resumptions and
|
|
347
|
-
// batched follow-ups, compact by starting a fresh main thread while
|
|
348
|
-
// keeping a parent link.
|
|
349
|
-
const compactThread = shouldCompactThread(issue, issueSession?.threadGeneration, effectiveContext);
|
|
350
|
-
if (compactThread && issue.threadId) {
|
|
351
|
-
parentThreadId = issue.threadId;
|
|
352
|
-
}
|
|
353
|
-
if (shouldReuseIssueThread({ existingThreadId: issue.threadId, compactThread, resumeThread })) {
|
|
354
|
-
threadId = issue.threadId;
|
|
355
|
-
}
|
|
356
|
-
else {
|
|
357
|
-
const thread = await this.codex.startThread({ cwd: worktreePath });
|
|
358
|
-
threadId = thread.id;
|
|
359
|
-
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
|
|
360
|
-
}
|
|
361
|
-
try {
|
|
362
|
-
const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
|
|
363
|
-
turnId = turn.turnId;
|
|
364
|
-
}
|
|
365
|
-
catch (turnError) {
|
|
366
|
-
// If the thread is stale (e.g. after app-server restart), start fresh and retry once.
|
|
367
|
-
const msg = turnError instanceof Error ? turnError.message : String(turnError);
|
|
368
|
-
if (msg.includes("thread not found") || msg.includes("not materialized")) {
|
|
369
|
-
this.logger.info({ issueKey: issue.issueKey, staleThreadId: threadId }, "Thread is stale, retrying with fresh thread");
|
|
370
|
-
const thread = await this.codex.startThread({ cwd: worktreePath });
|
|
371
|
-
threadId = thread.id;
|
|
372
|
-
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
|
|
373
|
-
const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
|
|
374
|
-
turnId = turn.turnId;
|
|
375
|
-
}
|
|
376
|
-
else {
|
|
377
|
-
throw turnError;
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
this.assertLaunchLease(run, "after starting the Codex turn");
|
|
381
|
-
}
|
|
382
|
-
catch (error) {
|
|
383
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
384
|
-
const lostLease = error instanceof Error && error.name === "IssueSessionLeaseLostError";
|
|
385
|
-
if (!lostLease) {
|
|
386
|
-
const nextState = isRequestedChangesRunType(runType) ? "escalated" : "failed";
|
|
387
|
-
this.db.finishRunWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, run.id, {
|
|
388
|
-
status: "failed",
|
|
389
|
-
failureReason: message,
|
|
390
|
-
});
|
|
391
|
-
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
392
|
-
projectId: item.projectId,
|
|
393
|
-
linearIssueId: item.issueId,
|
|
394
|
-
activeRunId: null,
|
|
395
|
-
factoryState: nextState,
|
|
396
|
-
});
|
|
397
|
-
}
|
|
398
|
-
this.logger.error({ issueKey: issue.issueKey, runType, error: message }, `Failed to launch ${runType} run`);
|
|
399
|
-
const failedIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
400
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(runType, `Failed to start ${lowerCaseFirst(message)}`));
|
|
401
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: runType });
|
|
402
|
-
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
403
|
-
throw error;
|
|
404
|
-
}
|
|
168
|
+
const { threadId, turnId, parentThreadId, } = await this.runLauncher.launchTurn({
|
|
169
|
+
project,
|
|
170
|
+
issue,
|
|
171
|
+
...(issueSession ? { issueSession } : {}),
|
|
172
|
+
run,
|
|
173
|
+
runType,
|
|
174
|
+
prompt,
|
|
175
|
+
branchName,
|
|
176
|
+
worktreePath,
|
|
177
|
+
resumeThread,
|
|
178
|
+
...(effectiveContext ? { effectiveContext } : {}),
|
|
179
|
+
leaseId,
|
|
180
|
+
...(this.botIdentity ? { botIdentity: this.botIdentity } : {}),
|
|
181
|
+
assertLaunchLease: (targetRun, phase) => this.assertLaunchLease(targetRun, phase),
|
|
182
|
+
resetWorktreeToTrackedBranch: (targetWorktreePath, targetBranchName, targetIssue) => this.resetWorktreeToTrackedBranch(targetWorktreePath, targetBranchName, targetIssue),
|
|
183
|
+
freshenWorktree: (targetWorktreePath, targetProject, targetIssue) => this.freshenWorktree(targetWorktreePath, targetProject, targetIssue),
|
|
184
|
+
linearSync: this.linearSync,
|
|
185
|
+
releaseLease: (projectId, issueId) => this.releaseIssueSessionLease(projectId, issueId),
|
|
186
|
+
isRequestedChangesRunType,
|
|
187
|
+
lowerCaseFirst,
|
|
188
|
+
});
|
|
405
189
|
this.assertLaunchLease(run, "before recording the active thread");
|
|
406
|
-
if (!this.db.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
|
|
190
|
+
if (!this.db.issueSessions.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
|
|
407
191
|
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping run thread update after losing issue-session lease");
|
|
408
192
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
409
193
|
return;
|
|
410
194
|
}
|
|
411
195
|
// Reset zombie recovery counter — this run started successfully
|
|
412
196
|
if (issue.zombieRecoveryAttempts > 0) {
|
|
413
|
-
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
197
|
+
this.db.issueSessions.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
414
198
|
projectId: item.projectId,
|
|
415
199
|
linearIssueId: item.issueId,
|
|
416
200
|
zombieRecoveryAttempts: 0,
|
|
@@ -526,7 +310,7 @@ export class RunOrchestrator {
|
|
|
526
310
|
if (notification.method === "turn/started" && threadId) {
|
|
527
311
|
this.activeThreadId = threadId;
|
|
528
312
|
}
|
|
529
|
-
const run = this.db.getRunByThreadId(threadId);
|
|
313
|
+
const run = this.db.runs.getRunByThreadId(threadId);
|
|
530
314
|
if (!run)
|
|
531
315
|
return;
|
|
532
316
|
if (!this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
@@ -535,7 +319,7 @@ export class RunOrchestrator {
|
|
|
535
319
|
}
|
|
536
320
|
const turnId = typeof notification.params.turnId === "string" ? notification.params.turnId : undefined;
|
|
537
321
|
if (this.config.runner.codex.persistExtendedHistory) {
|
|
538
|
-
this.db.saveThreadEvent({
|
|
322
|
+
this.db.runs.saveThreadEvent({
|
|
539
323
|
runId: run.id,
|
|
540
324
|
threadId,
|
|
541
325
|
...(turnId ? { turnId } : {}),
|
|
@@ -563,13 +347,13 @@ export class RunOrchestrator {
|
|
|
563
347
|
if (status === "failed") {
|
|
564
348
|
const nextState = isRequestedChangesRunType(run.runType) ? "escalated" : "failed";
|
|
565
349
|
const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
566
|
-
this.db.finishRunWithLease(lease, run.id, {
|
|
350
|
+
this.db.issueSessions.finishRunWithLease(lease, run.id, {
|
|
567
351
|
status: "failed",
|
|
568
352
|
threadId,
|
|
569
353
|
...(completedTurnId ? { turnId: completedTurnId } : {}),
|
|
570
354
|
failureReason: "Codex reported the turn completed in a failed state",
|
|
571
355
|
});
|
|
572
|
-
this.db.upsertIssueWithLease(lease, {
|
|
356
|
+
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
573
357
|
projectId: run.projectId,
|
|
574
358
|
linearIssueId: run.linearIssueId,
|
|
575
359
|
activeRunId: null,
|
|
@@ -599,158 +383,33 @@ export class RunOrchestrator {
|
|
|
599
383
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
600
384
|
return;
|
|
601
385
|
}
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
this.
|
|
611
|
-
|
|
612
|
-
this.
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
});
|
|
621
|
-
void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
|
|
622
|
-
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
623
|
-
this.linearSync.clearProgress(run.id);
|
|
624
|
-
this.activeThreadId = undefined;
|
|
625
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
626
|
-
return;
|
|
627
|
-
}
|
|
628
|
-
const missingReviewFixHeadError = await this.verifyReviewFixAdvancedHead(run, freshIssue);
|
|
629
|
-
if (missingReviewFixHeadError) {
|
|
630
|
-
this.failRunAndClear(run, missingReviewFixHeadError, "escalated");
|
|
631
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
632
|
-
this.feed?.publish({
|
|
633
|
-
level: "error",
|
|
634
|
-
kind: "turn",
|
|
635
|
-
issueKey: freshIssue.issueKey,
|
|
636
|
-
projectId: run.projectId,
|
|
637
|
-
stage: run.runType,
|
|
638
|
-
status: "same_head_review_handoff_blocked",
|
|
639
|
-
summary: missingReviewFixHeadError,
|
|
640
|
-
});
|
|
641
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
|
|
642
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
643
|
-
this.linearSync.clearProgress(run.id);
|
|
644
|
-
this.activeThreadId = undefined;
|
|
645
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
646
|
-
return;
|
|
647
|
-
}
|
|
648
|
-
const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
|
|
649
|
-
if (publishedOutcomeError) {
|
|
650
|
-
this.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
651
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
652
|
-
this.feed?.publish({
|
|
653
|
-
level: "warn",
|
|
654
|
-
kind: "turn",
|
|
655
|
-
issueKey: freshIssue.issueKey,
|
|
656
|
-
projectId: run.projectId,
|
|
657
|
-
stage: run.runType,
|
|
658
|
-
status: "publish_incomplete",
|
|
659
|
-
summary: publishedOutcomeError,
|
|
660
|
-
});
|
|
661
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
662
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
663
|
-
this.linearSync.clearProgress(run.id);
|
|
664
|
-
this.activeThreadId = undefined;
|
|
665
|
-
return;
|
|
666
|
-
}
|
|
667
|
-
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
668
|
-
const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
|
|
669
|
-
const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
|
|
670
|
-
const completed = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
671
|
-
this.db.finishRun(run.id, {
|
|
672
|
-
status: "completed",
|
|
673
|
-
threadId,
|
|
674
|
-
...(completedTurnId ? { turnId: completedTurnId } : {}),
|
|
675
|
-
summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
|
|
676
|
-
reportJson: JSON.stringify(report),
|
|
677
|
-
});
|
|
678
|
-
this.db.upsertIssue({
|
|
679
|
-
projectId: run.projectId,
|
|
680
|
-
linearIssueId: run.linearIssueId,
|
|
681
|
-
activeRunId: null,
|
|
682
|
-
...(postRunState ? { factoryState: postRunState } : {}),
|
|
683
|
-
pendingRunType: null,
|
|
684
|
-
pendingRunContextJson: null,
|
|
685
|
-
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
686
|
-
? {
|
|
687
|
-
lastGitHubFailureSource: null,
|
|
688
|
-
lastGitHubFailureHeadSha: null,
|
|
689
|
-
lastGitHubFailureSignature: null,
|
|
690
|
-
lastGitHubFailureCheckName: null,
|
|
691
|
-
lastGitHubFailureCheckUrl: null,
|
|
692
|
-
lastGitHubFailureContextJson: null,
|
|
693
|
-
lastGitHubFailureAt: null,
|
|
694
|
-
lastQueueIncidentJson: null,
|
|
695
|
-
lastAttemptedFailureHeadSha: null,
|
|
696
|
-
lastAttemptedFailureSignature: null,
|
|
697
|
-
}
|
|
698
|
-
: {})),
|
|
699
|
-
});
|
|
700
|
-
if (postRunFollowUp) {
|
|
701
|
-
return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
702
|
-
}
|
|
703
|
-
return true;
|
|
704
|
-
});
|
|
705
|
-
if (!completed) {
|
|
706
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping completion writes after losing issue-session lease");
|
|
707
|
-
this.linearSync.clearProgress(run.id);
|
|
708
|
-
this.activeThreadId = undefined;
|
|
709
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
710
|
-
return;
|
|
711
|
-
}
|
|
712
|
-
if (postRunFollowUp) {
|
|
713
|
-
this.feed?.publish({
|
|
714
|
-
level: "info",
|
|
715
|
-
kind: "stage",
|
|
716
|
-
issueKey: issue.issueKey,
|
|
717
|
-
projectId: run.projectId,
|
|
718
|
-
stage: postRunFollowUp.factoryState,
|
|
719
|
-
status: "follow_up_queued",
|
|
720
|
-
summary: postRunFollowUp.summary,
|
|
721
|
-
});
|
|
722
|
-
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
723
|
-
}
|
|
724
|
-
this.feed?.publish({
|
|
725
|
-
level: "info",
|
|
726
|
-
kind: "turn",
|
|
727
|
-
issueKey: issue.issueKey,
|
|
728
|
-
projectId: run.projectId,
|
|
729
|
-
stage: run.runType,
|
|
730
|
-
status: "completed",
|
|
731
|
-
summary: `Turn completed for ${run.runType}`,
|
|
732
|
-
detail: summarizeCurrentThread(thread).latestAgentMessage,
|
|
386
|
+
await this.runFinalizer.finalizeCompletedRun({
|
|
387
|
+
source: "notification",
|
|
388
|
+
run,
|
|
389
|
+
issue,
|
|
390
|
+
thread,
|
|
391
|
+
threadId,
|
|
392
|
+
...(completedTurnId ? { completedTurnId } : {}),
|
|
393
|
+
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
394
|
+
releaseLease: (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId),
|
|
395
|
+
failRunAndClear: (targetRun, message, nextState) => this.failRunAndClear(targetRun, message, nextState),
|
|
396
|
+
verifyReactiveRunAdvancedBranch: (targetRun, targetIssue) => this.verifyReactiveRunAdvancedBranch(targetRun, targetIssue),
|
|
397
|
+
verifyReviewFixAdvancedHead: (targetRun, targetIssue) => this.verifyReviewFixAdvancedHead(targetRun, targetIssue),
|
|
398
|
+
verifyPublishedRunOutcome: (targetRun, targetIssue) => this.verifyPublishedRunOutcome(targetRun, targetIssue),
|
|
399
|
+
refreshIssueAfterReactivePublish: (targetRun, targetIssue) => this.refreshIssueAfterReactivePublish(targetRun, targetIssue),
|
|
400
|
+
resolvePostRunFollowUp: (targetRun, targetIssue) => this.resolvePostRunFollowUp(targetRun, targetIssue),
|
|
401
|
+
resolveCompletedRunState,
|
|
402
|
+
resolveRecoverableRunState: resolveRecoverablePostRunState,
|
|
403
|
+
appendWakeEventWithLease: (lease, targetIssue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, runType, context, dedupeScope),
|
|
733
404
|
});
|
|
734
|
-
// Emit Linear completion activity + plan
|
|
735
|
-
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
736
|
-
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
737
|
-
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
738
|
-
runType: run.runType,
|
|
739
|
-
completionSummary,
|
|
740
|
-
postRunState: updatedIssue.factoryState,
|
|
741
|
-
...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
|
|
742
|
-
}));
|
|
743
|
-
void this.linearSync.syncSession(updatedIssue);
|
|
744
|
-
this.linearSync.clearProgress(run.id);
|
|
745
405
|
this.activeThreadId = undefined;
|
|
746
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
747
406
|
}
|
|
748
407
|
// ─── Active status for query ──────────────────────────────────────
|
|
749
408
|
async getActiveRunStatus(issueKey) {
|
|
750
409
|
const issue = this.db.getIssueByKey(issueKey);
|
|
751
410
|
if (!issue?.activeRunId)
|
|
752
411
|
return undefined;
|
|
753
|
-
const run = this.db.
|
|
412
|
+
const run = this.db.runs.getRunById(issue.activeRunId);
|
|
754
413
|
if (!run?.threadId)
|
|
755
414
|
return undefined;
|
|
756
415
|
const trackedIssue = this.db.issueToTrackedIssue(issue);
|
|
@@ -763,7 +422,7 @@ export class RunOrchestrator {
|
|
|
763
422
|
}
|
|
764
423
|
// ─── Reconciliation ───────────────────────────────────────────────
|
|
765
424
|
async reconcileActiveRuns() {
|
|
766
|
-
for (const run of this.db.listRunningRuns()) {
|
|
425
|
+
for (const run of this.db.runs.listRunningRuns()) {
|
|
767
426
|
await this.reconcileRun(run);
|
|
768
427
|
}
|
|
769
428
|
// Preemptively detect stuck merge-queue PRs (conflicts visible on
|
|
@@ -821,118 +480,14 @@ export class RunOrchestrator {
|
|
|
821
480
|
* escalate; backoff delay not elapsed → skip.
|
|
822
481
|
*/
|
|
823
482
|
recoverOrEscalate(issue, runType, reason) {
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
this.db.upsertIssueWithLease(lease, {
|
|
832
|
-
projectId: fresh.projectId,
|
|
833
|
-
linearIssueId: fresh.linearIssueId,
|
|
834
|
-
pendingRunType: null,
|
|
835
|
-
pendingRunContextJson: null,
|
|
836
|
-
factoryState: "escalated",
|
|
837
|
-
});
|
|
838
|
-
return true;
|
|
839
|
-
});
|
|
840
|
-
if (!updated) {
|
|
841
|
-
this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping review-fix recovery escalation after losing issue-session lease");
|
|
842
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
843
|
-
return;
|
|
844
|
-
}
|
|
845
|
-
this.logger.warn({ issueKey: fresh.issueKey, reason }, "Requested-changes run failed before a new head was published — escalating");
|
|
846
|
-
this.feed?.publish({
|
|
847
|
-
level: "error",
|
|
848
|
-
kind: "workflow",
|
|
849
|
-
issueKey: fresh.issueKey,
|
|
850
|
-
projectId: fresh.projectId,
|
|
851
|
-
stage: runType,
|
|
852
|
-
status: "escalated",
|
|
853
|
-
summary: `Requested-changes run failed before publishing a new head (${reason})`,
|
|
854
|
-
});
|
|
855
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
856
|
-
return;
|
|
857
|
-
}
|
|
858
|
-
// If PR already merged, transition to done — no retry needed
|
|
859
|
-
if (fresh.prState === "merged") {
|
|
860
|
-
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
861
|
-
this.db.upsertIssueWithLease(lease, {
|
|
862
|
-
projectId: fresh.projectId,
|
|
863
|
-
linearIssueId: fresh.linearIssueId,
|
|
864
|
-
factoryState: "done",
|
|
865
|
-
zombieRecoveryAttempts: 0,
|
|
866
|
-
lastZombieRecoveryAt: null,
|
|
867
|
-
});
|
|
868
|
-
return true;
|
|
869
|
-
});
|
|
870
|
-
if (!updated) {
|
|
871
|
-
this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping merged recovery completion after losing issue-session lease");
|
|
872
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
873
|
-
return;
|
|
874
|
-
}
|
|
875
|
-
this.logger.info({ issueKey: fresh.issueKey, reason }, "Recovery: PR already merged — transitioning to done");
|
|
876
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
877
|
-
return;
|
|
878
|
-
}
|
|
879
|
-
// Budget check
|
|
880
|
-
const attempts = fresh.zombieRecoveryAttempts + 1;
|
|
881
|
-
if (attempts > DEFAULT_ZOMBIE_RECOVERY_BUDGET) {
|
|
882
|
-
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
883
|
-
this.db.upsertIssueWithLease(lease, {
|
|
884
|
-
projectId: fresh.projectId,
|
|
885
|
-
linearIssueId: fresh.linearIssueId,
|
|
886
|
-
factoryState: "escalated",
|
|
887
|
-
});
|
|
888
|
-
return true;
|
|
889
|
-
});
|
|
890
|
-
if (!updated) {
|
|
891
|
-
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery escalation after losing issue-session lease");
|
|
892
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
893
|
-
return;
|
|
894
|
-
}
|
|
895
|
-
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: budget exhausted — escalating");
|
|
896
|
-
this.feed?.publish({
|
|
897
|
-
level: "error",
|
|
898
|
-
kind: "workflow",
|
|
899
|
-
issueKey: fresh.issueKey,
|
|
900
|
-
projectId: fresh.projectId,
|
|
901
|
-
stage: "escalated",
|
|
902
|
-
status: "budget_exhausted",
|
|
903
|
-
summary: `${reason} recovery failed after ${DEFAULT_ZOMBIE_RECOVERY_BUDGET} attempts`,
|
|
904
|
-
});
|
|
905
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
906
|
-
return;
|
|
907
|
-
}
|
|
908
|
-
// Exponential backoff — skip if delay hasn't elapsed
|
|
909
|
-
if (fresh.lastZombieRecoveryAt) {
|
|
910
|
-
const elapsed = Date.now() - new Date(fresh.lastZombieRecoveryAt).getTime();
|
|
911
|
-
const delay = ZOMBIE_RECOVERY_BASE_DELAY_MS * Math.pow(2, fresh.zombieRecoveryAttempts);
|
|
912
|
-
if (elapsed < delay) {
|
|
913
|
-
this.logger.debug({ issueKey: fresh.issueKey, attempts: fresh.zombieRecoveryAttempts, delay, elapsed }, "Recovery: backoff not elapsed, skipping");
|
|
914
|
-
return;
|
|
915
|
-
}
|
|
916
|
-
}
|
|
917
|
-
// Re-enqueue with backoff tracking
|
|
918
|
-
const requeued = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
919
|
-
this.db.upsertIssueWithLease(lease, {
|
|
920
|
-
projectId: fresh.projectId,
|
|
921
|
-
linearIssueId: fresh.linearIssueId,
|
|
922
|
-
pendingRunType: null,
|
|
923
|
-
pendingRunContextJson: null,
|
|
924
|
-
zombieRecoveryAttempts: attempts,
|
|
925
|
-
lastZombieRecoveryAt: new Date().toISOString(),
|
|
926
|
-
});
|
|
927
|
-
return this.appendWakeEventWithLease(lease, fresh, runType, undefined, `recovery:${attempts}`);
|
|
483
|
+
this.runRecovery.recoverOrEscalate({
|
|
484
|
+
issue,
|
|
485
|
+
runType,
|
|
486
|
+
reason,
|
|
487
|
+
isRequestedChangesRunType,
|
|
488
|
+
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
489
|
+
appendWakeEventWithLease: (lease, targetIssue, pendingRunType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, pendingRunType, context, dedupeScope),
|
|
928
490
|
});
|
|
929
|
-
if (!requeued) {
|
|
930
|
-
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery re-enqueue after losing issue-session lease");
|
|
931
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
932
|
-
return;
|
|
933
|
-
}
|
|
934
|
-
this.enqueueIssue(fresh.projectId, fresh.linearIssueId);
|
|
935
|
-
this.logger.info({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: re-enqueued with backoff");
|
|
936
491
|
}
|
|
937
492
|
async reconcileRun(run) {
|
|
938
493
|
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
@@ -949,7 +504,7 @@ export class RunOrchestrator {
|
|
|
949
504
|
// (e.g. pr_merged processed, DB manually edited), just release the run.
|
|
950
505
|
if (TERMINAL_STATES.has(issue.factoryState)) {
|
|
951
506
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
952
|
-
this.db.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
507
|
+
this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
953
508
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
954
509
|
});
|
|
955
510
|
this.logger.info({ issueKey: issue.issueKey, runId: run.id, factoryState: issue.factoryState }, "Reconciliation: released run on terminal issue");
|
|
@@ -962,7 +517,7 @@ export class RunOrchestrator {
|
|
|
962
517
|
if (!run.threadId) {
|
|
963
518
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
|
|
964
519
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
965
|
-
this.db.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
520
|
+
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
966
521
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
967
522
|
});
|
|
968
523
|
this.recoverOrEscalate(issue, run.runType, "zombie");
|
|
@@ -980,7 +535,7 @@ export class RunOrchestrator {
|
|
|
980
535
|
catch {
|
|
981
536
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
|
|
982
537
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
983
|
-
this.db.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
538
|
+
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
984
539
|
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
985
540
|
});
|
|
986
541
|
this.recoverOrEscalate(issue, run.runType, "stale_thread");
|
|
@@ -998,7 +553,7 @@ export class RunOrchestrator {
|
|
|
998
553
|
const stopState = resolveAuthoritativeLinearStopState(linearIssue);
|
|
999
554
|
if (stopState?.isFinal) {
|
|
1000
555
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
1001
|
-
this.db.finishRun(run.id, { status: "released" });
|
|
556
|
+
this.db.runs.finishRun(run.id, { status: "released" });
|
|
1002
557
|
this.db.upsertIssue({
|
|
1003
558
|
projectId: run.projectId,
|
|
1004
559
|
linearIssueId: run.linearIssueId,
|
|
@@ -1032,21 +587,21 @@ export class RunOrchestrator {
|
|
|
1032
587
|
// Interrupted runs are not real failures — undo the budget increment.
|
|
1033
588
|
const repairedCounters = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
1034
589
|
if (run.runType === "ci_repair" && issue.ciRepairAttempts > 0) {
|
|
1035
|
-
this.db.upsertIssueWithLease(lease, {
|
|
590
|
+
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
1036
591
|
projectId: issue.projectId,
|
|
1037
592
|
linearIssueId: issue.linearIssueId,
|
|
1038
593
|
ciRepairAttempts: issue.ciRepairAttempts - 1,
|
|
1039
594
|
});
|
|
1040
595
|
}
|
|
1041
596
|
else if (run.runType === "queue_repair" && issue.queueRepairAttempts > 0) {
|
|
1042
|
-
this.db.upsertIssueWithLease(lease, {
|
|
597
|
+
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
1043
598
|
projectId: issue.projectId,
|
|
1044
599
|
linearIssueId: issue.linearIssueId,
|
|
1045
600
|
queueRepairAttempts: issue.queueRepairAttempts - 1,
|
|
1046
601
|
});
|
|
1047
602
|
}
|
|
1048
603
|
if (run.runType === "ci_repair" || run.runType === "queue_repair") {
|
|
1049
|
-
this.db.upsertIssueWithLease(lease, {
|
|
604
|
+
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
1050
605
|
projectId: issue.projectId,
|
|
1051
606
|
linearIssueId: issue.linearIssueId,
|
|
1052
607
|
lastAttemptedFailureHeadSha: null,
|
|
@@ -1138,140 +693,25 @@ export class RunOrchestrator {
|
|
|
1138
693
|
}
|
|
1139
694
|
// Handle completed turn discovered during reconciliation
|
|
1140
695
|
if (latestTurn?.status === "completed") {
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
this.
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
1160
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1161
|
-
return;
|
|
1162
|
-
}
|
|
1163
|
-
const missingReviewFixHeadError = await this.verifyReviewFixAdvancedHead(run, freshIssue);
|
|
1164
|
-
if (missingReviewFixHeadError) {
|
|
1165
|
-
this.failRunAndClear(run, missingReviewFixHeadError, "escalated");
|
|
1166
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1167
|
-
this.feed?.publish({
|
|
1168
|
-
level: "error",
|
|
1169
|
-
kind: "turn",
|
|
1170
|
-
issueKey: freshIssue.issueKey,
|
|
1171
|
-
projectId: run.projectId,
|
|
1172
|
-
stage: run.runType,
|
|
1173
|
-
status: "same_head_review_handoff_blocked",
|
|
1174
|
-
summary: missingReviewFixHeadError,
|
|
1175
|
-
});
|
|
1176
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
|
|
1177
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1178
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1179
|
-
return;
|
|
1180
|
-
}
|
|
1181
|
-
const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
|
|
1182
|
-
if (publishedOutcomeError) {
|
|
1183
|
-
this.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
1184
|
-
this.feed?.publish({
|
|
1185
|
-
level: "warn",
|
|
1186
|
-
kind: "turn",
|
|
1187
|
-
issueKey: issue.issueKey,
|
|
1188
|
-
projectId: run.projectId,
|
|
1189
|
-
stage: run.runType,
|
|
1190
|
-
status: "publish_incomplete",
|
|
1191
|
-
summary: publishedOutcomeError,
|
|
1192
|
-
});
|
|
1193
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1194
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
1195
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1196
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1197
|
-
return;
|
|
1198
|
-
}
|
|
1199
|
-
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
1200
|
-
const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
|
|
1201
|
-
const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
|
|
1202
|
-
const reconciled = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
1203
|
-
this.db.finishRun(run.id, {
|
|
1204
|
-
status: "completed",
|
|
1205
|
-
...(run.threadId ? { threadId: run.threadId } : {}),
|
|
1206
|
-
...(latestTurn.id ? { turnId: latestTurn.id } : {}),
|
|
1207
|
-
summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
|
|
1208
|
-
reportJson: JSON.stringify(report),
|
|
1209
|
-
});
|
|
1210
|
-
this.db.upsertIssue({
|
|
1211
|
-
projectId: run.projectId,
|
|
1212
|
-
linearIssueId: run.linearIssueId,
|
|
1213
|
-
activeRunId: null,
|
|
1214
|
-
...(postRunState ? { factoryState: postRunState } : {}),
|
|
1215
|
-
pendingRunType: null,
|
|
1216
|
-
pendingRunContextJson: null,
|
|
1217
|
-
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
1218
|
-
? {
|
|
1219
|
-
lastGitHubFailureSource: null,
|
|
1220
|
-
lastGitHubFailureHeadSha: null,
|
|
1221
|
-
lastGitHubFailureSignature: null,
|
|
1222
|
-
lastGitHubFailureCheckName: null,
|
|
1223
|
-
lastGitHubFailureCheckUrl: null,
|
|
1224
|
-
lastGitHubFailureContextJson: null,
|
|
1225
|
-
lastGitHubFailureAt: null,
|
|
1226
|
-
lastQueueIncidentJson: null,
|
|
1227
|
-
lastAttemptedFailureHeadSha: null,
|
|
1228
|
-
lastAttemptedFailureSignature: null,
|
|
1229
|
-
}
|
|
1230
|
-
: {})),
|
|
1231
|
-
});
|
|
1232
|
-
if (postRunFollowUp) {
|
|
1233
|
-
return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
1234
|
-
}
|
|
1235
|
-
return true;
|
|
696
|
+
await this.runFinalizer.finalizeCompletedRun({
|
|
697
|
+
source: "reconciliation",
|
|
698
|
+
run,
|
|
699
|
+
issue,
|
|
700
|
+
thread,
|
|
701
|
+
threadId: run.threadId,
|
|
702
|
+
...(latestTurn.id ? { completedTurnId: latestTurn.id } : {}),
|
|
703
|
+
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
704
|
+
releaseLease: (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId),
|
|
705
|
+
failRunAndClear: (targetRun, message, nextState) => this.failRunAndClear(targetRun, message, nextState),
|
|
706
|
+
verifyReactiveRunAdvancedBranch: (targetRun, targetIssue) => this.verifyReactiveRunAdvancedBranch(targetRun, targetIssue),
|
|
707
|
+
verifyReviewFixAdvancedHead: (targetRun, targetIssue) => this.verifyReviewFixAdvancedHead(targetRun, targetIssue),
|
|
708
|
+
verifyPublishedRunOutcome: (targetRun, targetIssue) => this.verifyPublishedRunOutcome(targetRun, targetIssue),
|
|
709
|
+
refreshIssueAfterReactivePublish: (targetRun, targetIssue) => this.refreshIssueAfterReactivePublish(targetRun, targetIssue),
|
|
710
|
+
resolvePostRunFollowUp: (targetRun, targetIssue) => this.resolvePostRunFollowUp(targetRun, targetIssue),
|
|
711
|
+
resolveCompletedRunState,
|
|
712
|
+
resolveRecoverableRunState: resolveRecoverablePostRunState,
|
|
713
|
+
appendWakeEventWithLease: (lease, targetIssue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, runType, context, dedupeScope),
|
|
1236
714
|
});
|
|
1237
|
-
if (!reconciled) {
|
|
1238
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping reconciled completion writes after losing issue-session lease");
|
|
1239
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1240
|
-
return;
|
|
1241
|
-
}
|
|
1242
|
-
if (postRunFollowUp) {
|
|
1243
|
-
this.feed?.publish({
|
|
1244
|
-
level: "info",
|
|
1245
|
-
kind: "stage",
|
|
1246
|
-
issueKey: issue.issueKey,
|
|
1247
|
-
projectId: run.projectId,
|
|
1248
|
-
stage: postRunFollowUp.factoryState,
|
|
1249
|
-
status: "follow_up_queued",
|
|
1250
|
-
summary: postRunFollowUp.summary,
|
|
1251
|
-
});
|
|
1252
|
-
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
1253
|
-
}
|
|
1254
|
-
if (postRunState) {
|
|
1255
|
-
this.feed?.publish({
|
|
1256
|
-
level: "info",
|
|
1257
|
-
kind: "turn",
|
|
1258
|
-
issueKey: issue.issueKey,
|
|
1259
|
-
projectId: run.projectId,
|
|
1260
|
-
stage: run.runType,
|
|
1261
|
-
status: "completed",
|
|
1262
|
-
summary: `Reconciliation: ${run.runType} completed \u2192 ${postRunState}`,
|
|
1263
|
-
});
|
|
1264
|
-
}
|
|
1265
|
-
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
1266
|
-
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
1267
|
-
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
1268
|
-
runType: run.runType,
|
|
1269
|
-
completionSummary,
|
|
1270
|
-
postRunState: updatedIssue.factoryState,
|
|
1271
|
-
...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
|
|
1272
|
-
}));
|
|
1273
|
-
void this.linearSync.syncSession(updatedIssue);
|
|
1274
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1275
715
|
return;
|
|
1276
716
|
}
|
|
1277
717
|
if (acquiredRecoveryLease)
|
|
@@ -1279,69 +719,21 @@ export class RunOrchestrator {
|
|
|
1279
719
|
}
|
|
1280
720
|
// ─── Internal helpers ─────────────────────────────────────────────
|
|
1281
721
|
escalate(issue, runType, reason) {
|
|
1282
|
-
this.
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
this.db.clearPendingIssueSessionEventsWithLease(lease);
|
|
1288
|
-
this.db.upsertIssueWithLease(lease, {
|
|
1289
|
-
projectId: issue.projectId,
|
|
1290
|
-
linearIssueId: issue.linearIssueId,
|
|
1291
|
-
pendingRunType: null,
|
|
1292
|
-
pendingRunContextJson: null,
|
|
1293
|
-
activeRunId: null,
|
|
1294
|
-
factoryState: "escalated",
|
|
1295
|
-
});
|
|
1296
|
-
return true;
|
|
1297
|
-
});
|
|
1298
|
-
if (!escalated) {
|
|
1299
|
-
this.logger.warn({ issueKey: issue.issueKey, runType }, "Skipping escalation write after losing issue-session lease");
|
|
1300
|
-
this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
1301
|
-
return;
|
|
1302
|
-
}
|
|
1303
|
-
this.feed?.publish({
|
|
1304
|
-
level: "error",
|
|
1305
|
-
kind: "workflow",
|
|
1306
|
-
issueKey: issue.issueKey,
|
|
1307
|
-
projectId: issue.projectId,
|
|
1308
|
-
stage: runType,
|
|
1309
|
-
status: "escalated",
|
|
1310
|
-
summary: `Escalated: ${reason}`,
|
|
1311
|
-
});
|
|
1312
|
-
const escalatedIssue = this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
|
|
1313
|
-
void this.linearSync.emitActivity(escalatedIssue, {
|
|
1314
|
-
type: "error",
|
|
1315
|
-
body: `PatchRelay needs human help to continue.\n\n${reason}`,
|
|
722
|
+
this.runRecovery.escalate({
|
|
723
|
+
issue,
|
|
724
|
+
runType,
|
|
725
|
+
reason,
|
|
726
|
+
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
1316
727
|
});
|
|
1317
|
-
void this.linearSync.syncSession(escalatedIssue);
|
|
1318
|
-
this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
1319
728
|
}
|
|
1320
729
|
failRunAndClear(run, message, nextState = "failed") {
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
this.
|
|
1327
|
-
projectId: run.projectId,
|
|
1328
|
-
linearIssueId: run.linearIssueId,
|
|
1329
|
-
activeRunId: null,
|
|
1330
|
-
factoryState: nextState,
|
|
1331
|
-
});
|
|
1332
|
-
const branchOwner = this.resolveBranchOwnerForStateTransition(nextState);
|
|
1333
|
-
if (branchOwner) {
|
|
1334
|
-
const lease = this.getHeldIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1335
|
-
if (lease) {
|
|
1336
|
-
this.db.setBranchOwnerWithLease(lease, branchOwner);
|
|
1337
|
-
}
|
|
1338
|
-
}
|
|
1339
|
-
return true;
|
|
730
|
+
this.runRecovery.failRunAndClear({
|
|
731
|
+
run,
|
|
732
|
+
message,
|
|
733
|
+
nextState,
|
|
734
|
+
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
735
|
+
getHeldLease: (projectId, linearIssueId) => this.getHeldIssueSessionLease(projectId, linearIssueId),
|
|
1340
736
|
});
|
|
1341
|
-
if (!updated) {
|
|
1342
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping failure cleanup after losing issue-session lease");
|
|
1343
|
-
}
|
|
1344
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1345
737
|
}
|
|
1346
738
|
resolveBranchOwnerForStateTransition(newState, pendingRunType) {
|
|
1347
739
|
return resolveBranchOwnerForStateTransition(newState, pendingRunType);
|
|
@@ -1689,20 +1081,11 @@ export class RunOrchestrator {
|
|
|
1689
1081
|
}
|
|
1690
1082
|
throw new Error(`Failed to read thread ${threadId}`);
|
|
1691
1083
|
}
|
|
1692
|
-
issueSessionLeaseKey(projectId, linearIssueId) {
|
|
1693
|
-
return `${projectId}:${linearIssueId}`;
|
|
1694
|
-
}
|
|
1695
1084
|
getHeldIssueSessionLease(projectId, linearIssueId) {
|
|
1696
|
-
|
|
1697
|
-
if (!leaseId)
|
|
1698
|
-
return undefined;
|
|
1699
|
-
return { projectId, linearIssueId, leaseId };
|
|
1085
|
+
return this.leaseService.getHeldLease(projectId, linearIssueId);
|
|
1700
1086
|
}
|
|
1701
1087
|
withHeldIssueSessionLease(projectId, linearIssueId, fn) {
|
|
1702
|
-
|
|
1703
|
-
if (!lease)
|
|
1704
|
-
return undefined;
|
|
1705
|
-
return this.db.withIssueSessionLease(projectId, linearIssueId, lease.leaseId, () => fn(lease));
|
|
1088
|
+
return this.leaseService.withHeldLease(projectId, linearIssueId, fn);
|
|
1706
1089
|
}
|
|
1707
1090
|
upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
|
|
1708
1091
|
const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
|
|
@@ -1710,7 +1093,7 @@ export class RunOrchestrator {
|
|
|
1710
1093
|
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write without a held issue-session lease");
|
|
1711
1094
|
return undefined;
|
|
1712
1095
|
}
|
|
1713
|
-
const updated = this.db.upsertIssueWithLease(lease, params);
|
|
1096
|
+
const updated = this.db.issueSessions.upsertIssueWithLease(lease, params);
|
|
1714
1097
|
if (!updated) {
|
|
1715
1098
|
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
|
|
1716
1099
|
}
|
|
@@ -1726,113 +1109,22 @@ export class RunOrchestrator {
|
|
|
1726
1109
|
throw error;
|
|
1727
1110
|
}
|
|
1728
1111
|
acquireIssueSessionLease(projectId, linearIssueId) {
|
|
1729
|
-
|
|
1730
|
-
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
1731
|
-
const acquired = this.db.acquireIssueSessionLease({
|
|
1732
|
-
projectId,
|
|
1733
|
-
linearIssueId,
|
|
1734
|
-
leaseId,
|
|
1735
|
-
workerId: this.workerId,
|
|
1736
|
-
leasedUntil,
|
|
1737
|
-
});
|
|
1738
|
-
if (!acquired)
|
|
1739
|
-
return undefined;
|
|
1740
|
-
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
1741
|
-
return leaseId;
|
|
1112
|
+
return this.leaseService.acquire(projectId, linearIssueId);
|
|
1742
1113
|
}
|
|
1743
1114
|
forceAcquireIssueSessionLease(projectId, linearIssueId) {
|
|
1744
|
-
|
|
1745
|
-
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
1746
|
-
const acquired = this.db.forceAcquireIssueSessionLease({
|
|
1747
|
-
projectId,
|
|
1748
|
-
linearIssueId,
|
|
1749
|
-
leaseId,
|
|
1750
|
-
workerId: this.workerId,
|
|
1751
|
-
leasedUntil,
|
|
1752
|
-
});
|
|
1753
|
-
if (!acquired)
|
|
1754
|
-
return undefined;
|
|
1755
|
-
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
1756
|
-
return leaseId;
|
|
1115
|
+
return this.leaseService.forceAcquire(projectId, linearIssueId);
|
|
1757
1116
|
}
|
|
1758
1117
|
claimLeaseForReconciliation(projectId, linearIssueId) {
|
|
1759
|
-
|
|
1760
|
-
if (this.activeSessionLeases.has(key)) {
|
|
1761
|
-
return "owned";
|
|
1762
|
-
}
|
|
1763
|
-
const session = this.db.getIssueSession(projectId, linearIssueId);
|
|
1764
|
-
if (!session)
|
|
1765
|
-
return "skip";
|
|
1766
|
-
const leasedUntilMs = session.leasedUntil ? Date.parse(session.leasedUntil) : undefined;
|
|
1767
|
-
if (leasedUntilMs !== undefined && Number.isFinite(leasedUntilMs) && leasedUntilMs > Date.now()) {
|
|
1768
|
-
return "skip";
|
|
1769
|
-
}
|
|
1770
|
-
return this.acquireIssueSessionLease(projectId, linearIssueId) ? true : "skip";
|
|
1118
|
+
return this.leaseService.claimForReconciliation(projectId, linearIssueId);
|
|
1771
1119
|
}
|
|
1772
1120
|
async reclaimForeignRecoveryLeaseIfSafe(run, issue) {
|
|
1773
|
-
|
|
1774
|
-
if (this.activeSessionLeases.has(key)) {
|
|
1775
|
-
return false;
|
|
1776
|
-
}
|
|
1777
|
-
const session = this.db.getIssueSession(run.projectId, run.linearIssueId);
|
|
1778
|
-
if (!session?.leaseId || !session.workerId || session.workerId === this.workerId) {
|
|
1779
|
-
return false;
|
|
1780
|
-
}
|
|
1781
|
-
if (issue.activeRunId !== run.id) {
|
|
1782
|
-
return false;
|
|
1783
|
-
}
|
|
1784
|
-
let safeToReclaim = !run.threadId;
|
|
1785
|
-
if (!safeToReclaim && run.threadId) {
|
|
1786
|
-
try {
|
|
1787
|
-
const thread = await this.readThreadWithRetry(run.threadId, 1);
|
|
1788
|
-
const latestTurn = getThreadTurns(thread).at(-1);
|
|
1789
|
-
safeToReclaim = thread.status === "notLoaded"
|
|
1790
|
-
|| latestTurn?.status === "interrupted"
|
|
1791
|
-
|| latestTurn?.status === "completed";
|
|
1792
|
-
}
|
|
1793
|
-
catch {
|
|
1794
|
-
safeToReclaim = true;
|
|
1795
|
-
}
|
|
1796
|
-
}
|
|
1797
|
-
if (!safeToReclaim) {
|
|
1798
|
-
return false;
|
|
1799
|
-
}
|
|
1800
|
-
const leaseId = this.forceAcquireIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1801
|
-
if (!leaseId) {
|
|
1802
|
-
return false;
|
|
1803
|
-
}
|
|
1804
|
-
this.logger.info({
|
|
1805
|
-
issueKey: issue.issueKey,
|
|
1806
|
-
runId: run.id,
|
|
1807
|
-
previousWorkerId: session.workerId,
|
|
1808
|
-
previousLeaseId: session.leaseId,
|
|
1809
|
-
reclaimedLeaseId: leaseId,
|
|
1810
|
-
}, "Reclaimed foreign issue-session lease for active-run recovery");
|
|
1811
|
-
return true;
|
|
1121
|
+
return await this.leaseService.reclaimForeignRecoveryLeaseIfSafe(run, issue);
|
|
1812
1122
|
}
|
|
1813
1123
|
heartbeatIssueSessionLease(projectId, linearIssueId) {
|
|
1814
|
-
|
|
1815
|
-
const leaseId = this.activeSessionLeases.get(key) ?? this.db.getIssueSession(projectId, linearIssueId)?.leaseId;
|
|
1816
|
-
if (!leaseId)
|
|
1817
|
-
return false;
|
|
1818
|
-
const renewed = this.db.renewIssueSessionLease({
|
|
1819
|
-
projectId,
|
|
1820
|
-
linearIssueId,
|
|
1821
|
-
leaseId,
|
|
1822
|
-
leasedUntil: new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString(),
|
|
1823
|
-
});
|
|
1824
|
-
if (renewed) {
|
|
1825
|
-
this.activeSessionLeases.set(key, leaseId);
|
|
1826
|
-
return true;
|
|
1827
|
-
}
|
|
1828
|
-
this.activeSessionLeases.delete(key);
|
|
1829
|
-
return false;
|
|
1124
|
+
return this.leaseService.heartbeat(projectId, linearIssueId);
|
|
1830
1125
|
}
|
|
1831
1126
|
releaseIssueSessionLease(projectId, linearIssueId) {
|
|
1832
|
-
|
|
1833
|
-
const leaseId = this.activeSessionLeases.get(key);
|
|
1834
|
-
this.db.releaseIssueSessionLease(projectId, linearIssueId, leaseId);
|
|
1835
|
-
this.activeSessionLeases.delete(key);
|
|
1127
|
+
this.leaseService.release(projectId, linearIssueId);
|
|
1836
1128
|
}
|
|
1837
1129
|
}
|
|
1838
1130
|
/**
|