patchrelay 0.36.7 → 0.36.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/cluster-health.js +2 -2
- package/dist/cli/data.js +20 -20
- package/dist/db/issue-session-store.js +284 -0
- package/dist/db/issue-store.js +559 -0
- package/dist/db/run-store.js +125 -0
- package/dist/db/webhook-event-store.js +71 -0
- package/dist/db.js +52 -1138
- package/dist/github-webhook-handler.js +44 -44
- package/dist/idle-reconciliation.js +20 -20
- package/dist/interrupted-run-recovery.js +176 -0
- package/dist/issue-query-service.js +13 -13
- package/dist/issue-session-lease-service.js +143 -0
- package/dist/issue-session-projector.js +114 -0
- package/dist/linear-session-sync.js +10 -10
- package/dist/queue-health-monitor.js +5 -5
- package/dist/run-completion-policy.js +412 -0
- package/dist/run-finalizer.js +172 -0
- package/dist/run-launcher.js +193 -0
- package/dist/run-orchestrator.js +145 -1505
- package/dist/run-recovery-service.js +209 -0
- package/dist/run-wake-planner.js +101 -0
- package/dist/service.js +33 -33
- package/dist/tracked-issue-projector.js +69 -0
- package/dist/webhook-handler.js +64 -693
- package/dist/webhooks/agent-session-handler.js +212 -0
- package/dist/webhooks/comment-policy.js +41 -0
- package/dist/webhooks/comment-wake-handler.js +133 -0
- package/dist/webhooks/decision-helpers.js +74 -0
- package/dist/webhooks/desired-stage-recorder.js +177 -0
- package/dist/webhooks/issue-removal-handler.js +68 -0
- package/dist/worktree-manager.js +69 -0
- package/package.json +1 -1
package/dist/run-orchestrator.js
CHANGED
|
@@ -1,63 +1,25 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { buildStageReport, countEventMethods, extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread, } from "./run-reporting.js";
|
|
5
|
-
import { buildRunCompletedActivity, buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
1
|
+
import { TERMINAL_STATES } from "./factory-state.js";
|
|
2
|
+
import { extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread } from "./run-reporting.js";
|
|
3
|
+
import { buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
6
4
|
import { WorktreeManager } from "./worktree-manager.js";
|
|
7
5
|
import { resolveAuthoritativeLinearStopState, resolvePreferredCompletedLinearState } from "./linear-workflow.js";
|
|
8
|
-
import { execCommand } from "./utils.js";
|
|
9
6
|
import { getThreadTurns } from "./codex-thread-utils.js";
|
|
10
|
-
import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
11
|
-
import { loadPatchRelayRepoPrompting } from "./patchrelay-customization.js";
|
|
12
|
-
import { buildRunPrompt as buildPatchRelayRunPrompt, findDisallowedPatchRelayPromptSectionIds, findUnknownPatchRelayPromptSectionIds, mergePromptCustomizationLayers, resolveImplementationDeliveryMode, resolvePromptLayers, } from "./prompting/patchrelay.js";
|
|
13
|
-
const DEFAULT_CI_REPAIR_BUDGET = 3;
|
|
14
|
-
const DEFAULT_QUEUE_REPAIR_BUDGET = 3;
|
|
15
|
-
// Requested-changes loops can legitimately take more iterations than CI/queue
|
|
16
|
-
// repair when the reviewer is catching nuanced product or timing bugs across
|
|
17
|
-
// successive heads. Keep a hard ceiling to prevent infinite ping-pong, but make
|
|
18
|
-
// it wide enough that real review cycles can continue after multiple successful
|
|
19
|
-
// head advances.
|
|
20
|
-
const DEFAULT_REVIEW_FIX_BUDGET = 12;
|
|
21
|
-
const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
|
|
22
|
-
const ZOMBIE_RECOVERY_BASE_DELAY_MS = 15_000; // 15s, 30s, 60s, 120s, 240s
|
|
23
|
-
const ISSUE_SESSION_LEASE_MS = 10 * 60_000;
|
|
24
|
-
const MAX_THREAD_GENERATION_BEFORE_COMPACTION = 4;
|
|
25
|
-
const MAX_FOLLOW_UPS_BEFORE_COMPACTION = 4;
|
|
26
7
|
import { QueueHealthMonitor } from "./queue-health-monitor.js";
|
|
27
8
|
import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
|
|
28
9
|
import { LinearSessionSync } from "./linear-session-sync.js";
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
}
|
|
10
|
+
import { IssueSessionLeaseService } from "./issue-session-lease-service.js";
|
|
11
|
+
import { InterruptedRunRecovery, resolveRecoverablePostRunState } from "./interrupted-run-recovery.js";
|
|
12
|
+
import { RunCompletionPolicy } from "./run-completion-policy.js";
|
|
13
|
+
import { RunFinalizer } from "./run-finalizer.js";
|
|
14
|
+
import { RunLauncher } from "./run-launcher.js";
|
|
15
|
+
import { RunRecoveryService } from "./run-recovery-service.js";
|
|
16
|
+
import { RunWakePlanner } from "./run-wake-planner.js";
|
|
35
17
|
function lowerCaseFirst(value) {
|
|
36
18
|
return value ? `${value.slice(0, 1).toLowerCase()}${value.slice(1)}` : value;
|
|
37
19
|
}
|
|
38
20
|
function isRequestedChangesRunType(runType) {
|
|
39
21
|
return runType === "review_fix" || runType === "branch_upkeep";
|
|
40
22
|
}
|
|
41
|
-
function resolveRequestedChangesMode(runType, context) {
|
|
42
|
-
if (runType === "branch_upkeep") {
|
|
43
|
-
return "branch_upkeep";
|
|
44
|
-
}
|
|
45
|
-
return context?.reviewFixMode === "branch_upkeep" || context?.branchUpkeepRequired === true
|
|
46
|
-
? "branch_upkeep"
|
|
47
|
-
: "address_review_feedback";
|
|
48
|
-
}
|
|
49
|
-
function shouldCompactThread(issue, threadGeneration, context) {
|
|
50
|
-
const followUpCount = typeof context?.followUpCount === "number" ? context.followUpCount : 0;
|
|
51
|
-
return issue.threadId !== undefined
|
|
52
|
-
&& (threadGeneration ?? 0) >= MAX_THREAD_GENERATION_BEFORE_COMPACTION
|
|
53
|
-
&& followUpCount >= MAX_FOLLOW_UPS_BEFORE_COMPACTION;
|
|
54
|
-
}
|
|
55
|
-
export function shouldReuseIssueThread(params) {
|
|
56
|
-
return Boolean(params.existingThreadId) && !params.compactThread && params.resumeThread;
|
|
57
|
-
}
|
|
58
|
-
function isBranchUpkeepRequired(context) {
|
|
59
|
-
return context?.branchUpkeepRequired === true;
|
|
60
|
-
}
|
|
61
23
|
export class RunOrchestrator {
|
|
62
24
|
config;
|
|
63
25
|
db;
|
|
@@ -73,7 +35,14 @@ export class RunOrchestrator {
|
|
|
73
35
|
linearSync;
|
|
74
36
|
activeThreadId;
|
|
75
37
|
workerId = `patchrelay:${process.pid}`;
|
|
76
|
-
|
|
38
|
+
leaseService;
|
|
39
|
+
runFinalizer;
|
|
40
|
+
runLauncher;
|
|
41
|
+
runRecovery;
|
|
42
|
+
runWakePlanner;
|
|
43
|
+
interruptedRunRecovery;
|
|
44
|
+
runCompletionPolicy;
|
|
45
|
+
activeSessionLeases;
|
|
77
46
|
botIdentity;
|
|
78
47
|
constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
|
|
79
48
|
this.config = config;
|
|
@@ -85,6 +54,14 @@ export class RunOrchestrator {
|
|
|
85
54
|
this.feed = feed;
|
|
86
55
|
this.worktreeManager = new WorktreeManager(config);
|
|
87
56
|
this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
|
|
57
|
+
this.leaseService = new IssueSessionLeaseService(db, logger, this.workerId, (threadId, maxRetries) => this.readThreadWithRetry(threadId, maxRetries));
|
|
58
|
+
this.activeSessionLeases = this.leaseService.activeSessionLeases;
|
|
59
|
+
this.runCompletionPolicy = new RunCompletionPolicy(config, db, logger, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn));
|
|
60
|
+
this.runFinalizer = new RunFinalizer(db, logger, this.linearSync, this.enqueueIssue, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (lease, issue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope), (run, message, nextState) => this.failRunAndClear(run, message, nextState), this.runCompletionPolicy, feed);
|
|
61
|
+
this.runLauncher = new RunLauncher(config, db, codex, logger, this.worktreeManager);
|
|
62
|
+
this.runRecovery = new RunRecoveryService(db, logger, this.linearSync, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.getHeldIssueSessionLease(projectId, linearIssueId), (lease, issue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (projectId, issueId) => this.enqueueIssue(projectId, issueId), (newState, pendingRunType) => this.resolveBranchOwnerForStateTransition(newState, pendingRunType), feed);
|
|
63
|
+
this.interruptedRunRecovery = new InterruptedRunRecovery(db, logger, this.linearSync, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (run, message, nextState) => this.failRunAndClear(run, message, nextState), (issue) => this.restoreIdleWorktree(issue), this.runCompletionPolicy, (projectId, issueId) => this.enqueueIssue(projectId, issueId), feed);
|
|
64
|
+
this.runWakePlanner = new RunWakePlanner(db);
|
|
88
65
|
this.idleReconciler = new IdleIssueReconciler(db, config, {
|
|
89
66
|
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
90
67
|
}, logger, feed);
|
|
@@ -94,205 +71,78 @@ export class RunOrchestrator {
|
|
|
94
71
|
}, logger, feed);
|
|
95
72
|
}
|
|
96
73
|
resolveRunWake(issue) {
|
|
97
|
-
|
|
98
|
-
if (sessionWake) {
|
|
99
|
-
return {
|
|
100
|
-
runType: sessionWake.runType,
|
|
101
|
-
context: sessionWake.context,
|
|
102
|
-
wakeReason: sessionWake.wakeReason,
|
|
103
|
-
resumeThread: sessionWake.resumeThread,
|
|
104
|
-
eventIds: sessionWake.eventIds,
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
return undefined;
|
|
74
|
+
return this.runWakePlanner.resolveRunWake(issue);
|
|
108
75
|
}
|
|
109
76
|
appendWakeEventWithLease(lease, issue, runType, context, dedupeScope) {
|
|
110
|
-
|
|
111
|
-
let dedupeKey;
|
|
112
|
-
if (runType === "queue_repair") {
|
|
113
|
-
eventType = "merge_steward_incident";
|
|
114
|
-
dedupeKey = `${dedupeScope ?? "wake"}:queue_repair:${issue.linearIssueId}:${issue.prHeadSha ?? issue.lastGitHubFailureHeadSha ?? "unknown-sha"}`;
|
|
115
|
-
}
|
|
116
|
-
else if (runType === "ci_repair") {
|
|
117
|
-
eventType = "settled_red_ci";
|
|
118
|
-
dedupeKey = `${dedupeScope ?? "wake"}:ci_repair:${issue.linearIssueId}:${issue.lastGitHubFailureSignature ?? issue.prHeadSha ?? "unknown-sha"}`;
|
|
119
|
-
}
|
|
120
|
-
else if (runType === "review_fix" || runType === "branch_upkeep") {
|
|
121
|
-
eventType = "review_changes_requested";
|
|
122
|
-
dedupeKey = `${dedupeScope ?? "wake"}:${runType}:${issue.linearIssueId}:${issue.prHeadSha ?? "unknown-sha"}`;
|
|
123
|
-
}
|
|
124
|
-
else {
|
|
125
|
-
eventType = "delegated";
|
|
126
|
-
dedupeKey = `${dedupeScope ?? "wake"}:implementation:${issue.linearIssueId}`;
|
|
127
|
-
}
|
|
128
|
-
return Boolean(this.db.appendIssueSessionEventWithLease(lease, {
|
|
129
|
-
projectId: issue.projectId,
|
|
130
|
-
linearIssueId: issue.linearIssueId,
|
|
131
|
-
eventType,
|
|
132
|
-
...(context ? { eventJson: JSON.stringify(context) } : {}),
|
|
133
|
-
dedupeKey,
|
|
134
|
-
}));
|
|
77
|
+
return this.runWakePlanner.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope);
|
|
135
78
|
}
|
|
136
79
|
materializeLegacyPendingWake(issue, lease) {
|
|
137
|
-
|
|
138
|
-
return issue;
|
|
139
|
-
const context = issue.pendingRunContextJson
|
|
140
|
-
? JSON.parse(issue.pendingRunContextJson)
|
|
141
|
-
: undefined;
|
|
142
|
-
this.appendWakeEventWithLease(lease, issue, issue.pendingRunType, context, "legacy_pending");
|
|
143
|
-
const updated = this.db.upsertIssueWithLease(lease, {
|
|
144
|
-
projectId: issue.projectId,
|
|
145
|
-
linearIssueId: issue.linearIssueId,
|
|
146
|
-
pendingRunType: null,
|
|
147
|
-
pendingRunContextJson: null,
|
|
148
|
-
});
|
|
149
|
-
if (!updated)
|
|
150
|
-
return issue;
|
|
151
|
-
return this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
|
|
80
|
+
return this.runWakePlanner.materializeLegacyPendingWake(issue, lease);
|
|
152
81
|
}
|
|
153
82
|
// ─── Run ────────────────────────────────────────────────────────
|
|
154
83
|
async run(item) {
|
|
155
84
|
const project = this.config.projects.find((p) => p.id === item.projectId);
|
|
156
85
|
if (!project)
|
|
157
86
|
return;
|
|
158
|
-
if (this.
|
|
87
|
+
if (this.leaseService.hasLocalLease(item.projectId, item.issueId)) {
|
|
159
88
|
return;
|
|
160
89
|
}
|
|
161
|
-
const issue = this.db.getIssue(item.projectId, item.issueId);
|
|
90
|
+
const issue = this.db.issues.getIssue(item.projectId, item.issueId);
|
|
162
91
|
if (!issue || issue.activeRunId !== undefined)
|
|
163
92
|
return;
|
|
164
|
-
const issueSession = this.db.getIssueSession(item.projectId, item.issueId);
|
|
165
|
-
const leaseId = this.
|
|
93
|
+
const issueSession = this.db.issueSessions.getIssueSession(item.projectId, item.issueId);
|
|
94
|
+
const leaseId = this.leaseService.acquire(item.projectId, item.issueId);
|
|
166
95
|
if (!leaseId) {
|
|
167
96
|
this.logger.info({ issueKey: issue.issueKey, projectId: item.projectId }, "Skipped run because another worker holds the session lease");
|
|
168
97
|
return;
|
|
169
98
|
}
|
|
170
99
|
if (issue.prState === "merged") {
|
|
171
|
-
this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
|
|
172
|
-
this.
|
|
100
|
+
this.db.issueSessions.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
|
|
101
|
+
this.leaseService.release(item.projectId, item.issueId);
|
|
173
102
|
return;
|
|
174
103
|
}
|
|
175
104
|
const wakeIssue = this.materializeLegacyPendingWake(issue, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
|
|
176
105
|
const wake = this.resolveRunWake(wakeIssue);
|
|
177
106
|
if (!wake) {
|
|
178
|
-
this.
|
|
107
|
+
this.leaseService.release(item.projectId, item.issueId);
|
|
179
108
|
return;
|
|
180
109
|
}
|
|
181
110
|
const { runType, context, resumeThread } = wake;
|
|
182
111
|
const effectiveContext = isRequestedChangesRunType(runType)
|
|
183
|
-
? await this.resolveRequestedChangesWakeContext(issue, runType, context
|
|
112
|
+
? await this.runCompletionPolicy.resolveRequestedChangesWakeContext(issue, runType, context)
|
|
184
113
|
: context;
|
|
185
114
|
const sourceHeadSha = typeof effectiveContext?.failureHeadSha === "string"
|
|
186
115
|
? effectiveContext.failureHeadSha
|
|
187
116
|
: typeof effectiveContext?.headSha === "string"
|
|
188
117
|
? effectiveContext.headSha
|
|
189
118
|
: issue.prHeadSha;
|
|
190
|
-
|
|
191
|
-
if (
|
|
192
|
-
this.escalate(issue, runType,
|
|
193
|
-
return;
|
|
194
|
-
}
|
|
195
|
-
if (runType === "queue_repair" && issue.queueRepairAttempts >= DEFAULT_QUEUE_REPAIR_BUDGET) {
|
|
196
|
-
this.escalate(issue, runType, `Queue repair budget exhausted (${DEFAULT_QUEUE_REPAIR_BUDGET} attempts)`);
|
|
119
|
+
const budgetExceeded = this.runWakePlanner.budgetExceeded(issue, runType, isRequestedChangesRunType);
|
|
120
|
+
if (budgetExceeded) {
|
|
121
|
+
this.escalate(issue, runType, budgetExceeded);
|
|
197
122
|
return;
|
|
198
123
|
}
|
|
199
|
-
if (
|
|
200
|
-
this.
|
|
124
|
+
if (!this.runWakePlanner.incrementAttemptCounters(issue, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, runType, isRequestedChangesRunType)) {
|
|
125
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
201
126
|
return;
|
|
202
127
|
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, ciRepairAttempts: issue.ciRepairAttempts + 1 });
|
|
206
|
-
if (!updated) {
|
|
207
|
-
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
208
|
-
return;
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
if (runType === "queue_repair") {
|
|
212
|
-
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueRepairAttempts: issue.queueRepairAttempts + 1 });
|
|
213
|
-
if (!updated) {
|
|
214
|
-
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
215
|
-
return;
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
if (isRequestedChangesRunType(runType)) {
|
|
219
|
-
const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, reviewFixAttempts: issue.reviewFixAttempts + 1 });
|
|
220
|
-
if (!updated) {
|
|
221
|
-
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
222
|
-
return;
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
const repoPrompting = loadPatchRelayRepoPrompting({
|
|
226
|
-
repoRoot: project.repoPath,
|
|
227
|
-
logger: this.logger,
|
|
228
|
-
});
|
|
229
|
-
const promptLayer = mergePromptCustomizationLayers(resolvePromptLayers(this.config.prompting, runType), resolvePromptLayers(repoPrompting, runType));
|
|
230
|
-
const unknownPromptSections = findUnknownPatchRelayPromptSectionIds(promptLayer);
|
|
231
|
-
if (unknownPromptSections.length > 0) {
|
|
232
|
-
this.logger.warn({ issueKey: issue.issueKey, runType, unknownPromptSections }, "PatchRelay prompt customization references unknown section ids");
|
|
233
|
-
}
|
|
234
|
-
const disallowedPromptSections = findDisallowedPatchRelayPromptSectionIds(promptLayer);
|
|
235
|
-
if (disallowedPromptSections.length > 0) {
|
|
236
|
-
this.logger.warn({ issueKey: issue.issueKey, runType, disallowedPromptSections }, "PatchRelay prompt customization attempted to replace non-overridable sections");
|
|
237
|
-
}
|
|
238
|
-
const prompt = buildPatchRelayRunPrompt({
|
|
128
|
+
const { prompt, branchName, worktreePath } = this.runLauncher.prepareLaunchPlan({
|
|
129
|
+
project,
|
|
239
130
|
issue,
|
|
240
131
|
runType,
|
|
241
|
-
|
|
242
|
-
...(effectiveContext ? { context: effectiveContext } : {}),
|
|
243
|
-
...(promptLayer ? { promptLayer } : {}),
|
|
132
|
+
...(effectiveContext ? { effectiveContext } : {}),
|
|
244
133
|
});
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
const freshWake = this.resolveRunWake(wakeIssue);
|
|
258
|
-
if (!freshWake || freshWake.runType !== runType)
|
|
259
|
-
return undefined;
|
|
260
|
-
const created = this.db.createRun({
|
|
261
|
-
issueId: fresh.id,
|
|
262
|
-
projectId: item.projectId,
|
|
263
|
-
linearIssueId: item.issueId,
|
|
264
|
-
runType,
|
|
265
|
-
...(sourceHeadSha ? { sourceHeadSha } : {}),
|
|
266
|
-
promptText: prompt,
|
|
267
|
-
});
|
|
268
|
-
const failureHeadSha = typeof effectiveContext?.failureHeadSha === "string"
|
|
269
|
-
? effectiveContext.failureHeadSha
|
|
270
|
-
: typeof effectiveContext?.headSha === "string" ? effectiveContext.headSha : undefined;
|
|
271
|
-
const failureSignature = typeof effectiveContext?.failureSignature === "string" ? effectiveContext.failureSignature : undefined;
|
|
272
|
-
this.db.upsertIssue({
|
|
273
|
-
projectId: item.projectId,
|
|
274
|
-
linearIssueId: item.issueId,
|
|
275
|
-
pendingRunType: null,
|
|
276
|
-
pendingRunContextJson: null,
|
|
277
|
-
activeRunId: created.id,
|
|
278
|
-
branchName,
|
|
279
|
-
worktreePath,
|
|
280
|
-
factoryState: runType === "implementation" ? "implementing"
|
|
281
|
-
: runType === "ci_repair" ? "repairing_ci"
|
|
282
|
-
: runType === "review_fix" || runType === "branch_upkeep" ? "changes_requested"
|
|
283
|
-
: runType === "queue_repair" ? "repairing_queue"
|
|
284
|
-
: "implementing",
|
|
285
|
-
...((runType === "ci_repair" || runType === "queue_repair") && failureSignature
|
|
286
|
-
? {
|
|
287
|
-
lastAttemptedFailureSignature: failureSignature,
|
|
288
|
-
lastAttemptedFailureHeadSha: failureHeadSha ?? null,
|
|
289
|
-
}
|
|
290
|
-
: {}),
|
|
291
|
-
});
|
|
292
|
-
this.db.consumeIssueSessionEvents(item.projectId, item.issueId, freshWake.eventIds, created.id);
|
|
293
|
-
this.db.setIssueSessionLastWakeReason(item.projectId, item.issueId, freshWake.wakeReason ?? null);
|
|
294
|
-
this.db.setBranchOwnerWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, "patchrelay");
|
|
295
|
-
return created;
|
|
134
|
+
const run = this.runLauncher.claimRun({
|
|
135
|
+
item,
|
|
136
|
+
issue,
|
|
137
|
+
leaseId,
|
|
138
|
+
runType,
|
|
139
|
+
prompt,
|
|
140
|
+
...(sourceHeadSha ? { sourceHeadSha } : {}),
|
|
141
|
+
...(effectiveContext ? { effectiveContext } : {}),
|
|
142
|
+
materializeLegacyPendingWake: (targetIssue, lease) => this.materializeLegacyPendingWake(targetIssue, lease),
|
|
143
|
+
resolveRunWake: (targetIssue) => this.resolveRunWake(targetIssue),
|
|
144
|
+
branchName,
|
|
145
|
+
worktreePath,
|
|
296
146
|
});
|
|
297
147
|
if (!run) {
|
|
298
148
|
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
@@ -307,110 +157,34 @@ export class RunOrchestrator {
|
|
|
307
157
|
status: "starting",
|
|
308
158
|
summary: `Starting ${runType} run`,
|
|
309
159
|
});
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
// Freshen the worktree: fetch + rebase onto latest base branch.
|
|
330
|
-
// This prevents branch contamination when local main has drifted
|
|
331
|
-
// and avoids scope-bundling review rejections from stale commits.
|
|
332
|
-
// Skip for queue_repair — its entire purpose is to resolve rebase conflicts.
|
|
333
|
-
if (runType !== "queue_repair") {
|
|
334
|
-
await this.freshenWorktree(worktreePath, project, issue);
|
|
335
|
-
}
|
|
336
|
-
// Run prepare-worktree hook
|
|
337
|
-
const hookEnv = buildHookEnv(issue.issueKey ?? issue.linearIssueId, branchName, runType, worktreePath);
|
|
338
|
-
const prepareResult = await runProjectHook(project.repoPath, "prepare-worktree", { cwd: worktreePath, env: hookEnv });
|
|
339
|
-
if (prepareResult.ran && prepareResult.exitCode !== 0) {
|
|
340
|
-
throw new Error(`prepare-worktree hook failed (exit ${prepareResult.exitCode}): ${prepareResult.stderr?.slice(0, 500) ?? ""}`);
|
|
341
|
-
}
|
|
342
|
-
this.assertLaunchLease(run, "before starting the Codex turn");
|
|
343
|
-
// Reuse the existing thread only for additive follow-ups that explicitly
|
|
344
|
-
// request continuity. Fresh review-fix runs now start a new thread so the
|
|
345
|
-
// model is not anchored to the implementation conversation that produced
|
|
346
|
-
// the rejected patch. If the thread has accumulated many resumptions and
|
|
347
|
-
// batched follow-ups, compact by starting a fresh main thread while
|
|
348
|
-
// keeping a parent link.
|
|
349
|
-
const compactThread = shouldCompactThread(issue, issueSession?.threadGeneration, effectiveContext);
|
|
350
|
-
if (compactThread && issue.threadId) {
|
|
351
|
-
parentThreadId = issue.threadId;
|
|
352
|
-
}
|
|
353
|
-
if (shouldReuseIssueThread({ existingThreadId: issue.threadId, compactThread, resumeThread })) {
|
|
354
|
-
threadId = issue.threadId;
|
|
355
|
-
}
|
|
356
|
-
else {
|
|
357
|
-
const thread = await this.codex.startThread({ cwd: worktreePath });
|
|
358
|
-
threadId = thread.id;
|
|
359
|
-
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
|
|
360
|
-
}
|
|
361
|
-
try {
|
|
362
|
-
const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
|
|
363
|
-
turnId = turn.turnId;
|
|
364
|
-
}
|
|
365
|
-
catch (turnError) {
|
|
366
|
-
// If the thread is stale (e.g. after app-server restart), start fresh and retry once.
|
|
367
|
-
const msg = turnError instanceof Error ? turnError.message : String(turnError);
|
|
368
|
-
if (msg.includes("thread not found") || msg.includes("not materialized")) {
|
|
369
|
-
this.logger.info({ issueKey: issue.issueKey, staleThreadId: threadId }, "Thread is stale, retrying with fresh thread");
|
|
370
|
-
const thread = await this.codex.startThread({ cwd: worktreePath });
|
|
371
|
-
threadId = thread.id;
|
|
372
|
-
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
|
|
373
|
-
const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
|
|
374
|
-
turnId = turn.turnId;
|
|
375
|
-
}
|
|
376
|
-
else {
|
|
377
|
-
throw turnError;
|
|
378
|
-
}
|
|
379
|
-
}
|
|
380
|
-
this.assertLaunchLease(run, "after starting the Codex turn");
|
|
381
|
-
}
|
|
382
|
-
catch (error) {
|
|
383
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
384
|
-
const lostLease = error instanceof Error && error.name === "IssueSessionLeaseLostError";
|
|
385
|
-
if (!lostLease) {
|
|
386
|
-
const nextState = isRequestedChangesRunType(runType) ? "escalated" : "failed";
|
|
387
|
-
this.db.finishRunWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, run.id, {
|
|
388
|
-
status: "failed",
|
|
389
|
-
failureReason: message,
|
|
390
|
-
});
|
|
391
|
-
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
392
|
-
projectId: item.projectId,
|
|
393
|
-
linearIssueId: item.issueId,
|
|
394
|
-
activeRunId: null,
|
|
395
|
-
factoryState: nextState,
|
|
396
|
-
});
|
|
397
|
-
}
|
|
398
|
-
this.logger.error({ issueKey: issue.issueKey, runType, error: message }, `Failed to launch ${runType} run`);
|
|
399
|
-
const failedIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
400
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(runType, `Failed to start ${lowerCaseFirst(message)}`));
|
|
401
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: runType });
|
|
402
|
-
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
403
|
-
throw error;
|
|
404
|
-
}
|
|
160
|
+
const { threadId, turnId, parentThreadId, } = await this.runLauncher.launchTurn({
|
|
161
|
+
project,
|
|
162
|
+
issue,
|
|
163
|
+
...(issueSession ? { issueSession } : {}),
|
|
164
|
+
run,
|
|
165
|
+
runType,
|
|
166
|
+
prompt,
|
|
167
|
+
branchName,
|
|
168
|
+
worktreePath,
|
|
169
|
+
resumeThread,
|
|
170
|
+
...(effectiveContext ? { effectiveContext } : {}),
|
|
171
|
+
leaseId,
|
|
172
|
+
...(this.botIdentity ? { botIdentity: this.botIdentity } : {}),
|
|
173
|
+
assertLaunchLease: (targetRun, phase) => this.assertLaunchLease(targetRun, phase),
|
|
174
|
+
linearSync: this.linearSync,
|
|
175
|
+
releaseLease: (projectId, issueId) => this.releaseIssueSessionLease(projectId, issueId),
|
|
176
|
+
isRequestedChangesRunType,
|
|
177
|
+
lowerCaseFirst,
|
|
178
|
+
});
|
|
405
179
|
this.assertLaunchLease(run, "before recording the active thread");
|
|
406
|
-
if (!this.db.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
|
|
180
|
+
if (!this.db.issueSessions.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
|
|
407
181
|
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping run thread update after losing issue-session lease");
|
|
408
182
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
409
183
|
return;
|
|
410
184
|
}
|
|
411
185
|
// Reset zombie recovery counter — this run started successfully
|
|
412
186
|
if (issue.zombieRecoveryAttempts > 0) {
|
|
413
|
-
this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
187
|
+
this.db.issueSessions.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
|
|
414
188
|
projectId: item.projectId,
|
|
415
189
|
linearIssueId: item.issueId,
|
|
416
190
|
zombieRecoveryAttempts: 0,
|
|
@@ -419,98 +193,15 @@ export class RunOrchestrator {
|
|
|
419
193
|
}
|
|
420
194
|
this.logger.info({ issueKey: issue.issueKey, runType, threadId, turnId }, `Started ${runType} run`);
|
|
421
195
|
// Emit Linear activity + plan
|
|
422
|
-
const freshIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
196
|
+
const freshIssue = this.db.issues.getIssue(item.projectId, item.issueId) ?? issue;
|
|
423
197
|
void this.linearSync.emitActivity(freshIssue, buildRunStartedActivity(runType));
|
|
424
198
|
void this.linearSync.syncSession(freshIssue, { activeRunType: runType });
|
|
425
199
|
}
|
|
426
|
-
// ─── Pre-run branch freshening ────────────────────────────────────
|
|
427
|
-
/**
|
|
428
|
-
* Fetch origin and rebase the worktree onto the latest base branch.
|
|
429
|
-
*
|
|
430
|
-
* Risks mitigated:
|
|
431
|
-
* - Dirty worktree from interrupted run → stash before, pop after
|
|
432
|
-
* - Conflicts → abort rebase, throw so the run fails with a clear reason
|
|
433
|
-
* - Already up-to-date → no-op
|
|
434
|
-
* - Keep publishing explicit: the orchestrator updates the local worktree
|
|
435
|
-
* only; the agent/run owns any later branch push.
|
|
436
|
-
*/
|
|
437
|
-
async freshenWorktree(worktreePath, project, issue) {
|
|
438
|
-
const gitBin = this.config.runner.gitBin;
|
|
439
|
-
const baseBranch = project.github?.baseBranch ?? "main";
|
|
440
|
-
// Stash any uncommitted changes from a previous interrupted run
|
|
441
|
-
const stashResult = await execCommand(gitBin, ["-C", worktreePath, "stash"], { timeoutMs: 30_000 });
|
|
442
|
-
const didStash = stashResult.exitCode === 0 && !stashResult.stdout?.includes("No local changes");
|
|
443
|
-
// Fetch latest base
|
|
444
|
-
const fetchResult = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", baseBranch], { timeoutMs: 60_000 });
|
|
445
|
-
if (fetchResult.exitCode !== 0) {
|
|
446
|
-
this.logger.warn({ issueKey: issue.issueKey, stderr: fetchResult.stderr?.slice(0, 300) }, "Pre-run fetch failed, proceeding with current base");
|
|
447
|
-
if (didStash)
|
|
448
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
449
|
-
return;
|
|
450
|
-
}
|
|
451
|
-
// Check if rebase is needed: is HEAD already on top of origin/baseBranch?
|
|
452
|
-
const mergeBaseResult = await execCommand(gitBin, ["-C", worktreePath, "merge-base", "--is-ancestor", `origin/${baseBranch}`, "HEAD"], { timeoutMs: 10_000 });
|
|
453
|
-
if (mergeBaseResult.exitCode === 0) {
|
|
454
|
-
// Already up-to-date — no rebase needed
|
|
455
|
-
this.logger.debug({ issueKey: issue.issueKey }, "Pre-run freshen: branch already up to date");
|
|
456
|
-
if (didStash)
|
|
457
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
458
|
-
return;
|
|
459
|
-
}
|
|
460
|
-
// Rebase onto latest base
|
|
461
|
-
const rebaseResult = await execCommand(gitBin, ["-C", worktreePath, "rebase", `origin/${baseBranch}`], { timeoutMs: 120_000 });
|
|
462
|
-
if (rebaseResult.exitCode !== 0) {
|
|
463
|
-
// Abort the failed rebase and restore state — then let the agent run
|
|
464
|
-
// proceed. The agent can resolve the conflict itself (the workflow
|
|
465
|
-
// prompt tells it to rebase and handle conflicts).
|
|
466
|
-
await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
|
|
467
|
-
if (didStash)
|
|
468
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
469
|
-
this.logger.warn({ issueKey: issue.issueKey, baseBranch }, "Pre-run freshen: rebase conflict, agent will resolve");
|
|
470
|
-
return;
|
|
471
|
-
}
|
|
472
|
-
this.logger.info({ issueKey: issue.issueKey, baseBranch }, "Pre-run freshen: rebased locally onto latest base");
|
|
473
|
-
// Restore stashed changes
|
|
474
|
-
if (didStash)
|
|
475
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
476
|
-
}
|
|
477
200
|
async resetWorktreeToTrackedBranch(worktreePath, branchName, issue) {
|
|
478
|
-
|
|
479
|
-
const branchFetch = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", branchName], { timeoutMs: 60_000 });
|
|
480
|
-
const hasRemoteBranch = branchFetch.exitCode === 0;
|
|
481
|
-
await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
|
|
482
|
-
await execCommand(gitBin, ["-C", worktreePath, "merge", "--abort"], { timeoutMs: 10_000 });
|
|
483
|
-
await execCommand(gitBin, ["-C", worktreePath, "cherry-pick", "--abort"], { timeoutMs: 10_000 });
|
|
484
|
-
await execCommand(gitBin, ["-C", worktreePath, "am", "--abort"], { timeoutMs: 10_000 });
|
|
485
|
-
await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", "HEAD"], { timeoutMs: 30_000 });
|
|
486
|
-
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
487
|
-
const checkoutTarget = hasRemoteBranch ? `origin/${branchName}` : branchName;
|
|
488
|
-
const checkoutResult = await execCommand(gitBin, ["-C", worktreePath, "checkout", "-B", branchName, checkoutTarget], { timeoutMs: 30_000 });
|
|
489
|
-
if (checkoutResult.exitCode !== 0) {
|
|
490
|
-
throw new Error(`Failed to restore ${branchName} worktree state: ${checkoutResult.stderr?.slice(0, 300) ?? "git checkout failed"}`);
|
|
491
|
-
}
|
|
492
|
-
const resetTarget = hasRemoteBranch ? `origin/${branchName}` : "HEAD";
|
|
493
|
-
const resetResult = await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", resetTarget], { timeoutMs: 30_000 });
|
|
494
|
-
if (resetResult.exitCode !== 0) {
|
|
495
|
-
throw new Error(`Failed to reset ${branchName} worktree state: ${resetResult.stderr?.slice(0, 300) ?? "git reset failed"}`);
|
|
496
|
-
}
|
|
497
|
-
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
498
|
-
this.logger.debug({ issueKey: issue.issueKey, branchName, hasRemoteBranch }, "Reset issue worktree to tracked branch state");
|
|
201
|
+
await this.worktreeManager.resetWorktreeToTrackedBranch(worktreePath, branchName, issue, this.logger);
|
|
499
202
|
}
|
|
500
203
|
async restoreIdleWorktree(issue) {
|
|
501
|
-
|
|
502
|
-
return;
|
|
503
|
-
try {
|
|
504
|
-
await this.resetWorktreeToTrackedBranch(issue.worktreePath, issue.branchName, issue);
|
|
505
|
-
}
|
|
506
|
-
catch (error) {
|
|
507
|
-
this.logger.warn({
|
|
508
|
-
issueKey: issue.issueKey,
|
|
509
|
-
branchName: issue.branchName,
|
|
510
|
-
worktreePath: issue.worktreePath,
|
|
511
|
-
error: error instanceof Error ? error.message : String(error),
|
|
512
|
-
}, "Failed to restore idle worktree after interrupted run");
|
|
513
|
-
}
|
|
204
|
+
await this.worktreeManager.restoreIdleWorktree(issue, this.logger);
|
|
514
205
|
}
|
|
515
206
|
// ─── Notification handler ─────────────────────────────────────────
|
|
516
207
|
async handleCodexNotification(notification) {
|
|
@@ -526,7 +217,7 @@ export class RunOrchestrator {
|
|
|
526
217
|
if (notification.method === "turn/started" && threadId) {
|
|
527
218
|
this.activeThreadId = threadId;
|
|
528
219
|
}
|
|
529
|
-
const run = this.db.getRunByThreadId(threadId);
|
|
220
|
+
const run = this.db.runs.getRunByThreadId(threadId);
|
|
530
221
|
if (!run)
|
|
531
222
|
return;
|
|
532
223
|
if (!this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
@@ -535,7 +226,7 @@ export class RunOrchestrator {
|
|
|
535
226
|
}
|
|
536
227
|
const turnId = typeof notification.params.turnId === "string" ? notification.params.turnId : undefined;
|
|
537
228
|
if (this.config.runner.codex.persistExtendedHistory) {
|
|
538
|
-
this.db.saveThreadEvent({
|
|
229
|
+
this.db.runs.saveThreadEvent({
|
|
539
230
|
runId: run.id,
|
|
540
231
|
threadId,
|
|
541
232
|
...(turnId ? { turnId } : {}),
|
|
@@ -547,7 +238,7 @@ export class RunOrchestrator {
|
|
|
547
238
|
this.linearSync.maybeEmitProgress(notification, run);
|
|
548
239
|
// Sync codex plan to Linear session when it updates
|
|
549
240
|
if (notification.method === "turn/plan/updated") {
|
|
550
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
241
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
551
242
|
if (issue) {
|
|
552
243
|
void this.linearSync.syncCodexPlan(issue, notification.params);
|
|
553
244
|
}
|
|
@@ -555,7 +246,7 @@ export class RunOrchestrator {
|
|
|
555
246
|
if (notification.method !== "turn/completed")
|
|
556
247
|
return;
|
|
557
248
|
const thread = await this.readThreadWithRetry(threadId);
|
|
558
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
249
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
559
250
|
if (!issue)
|
|
560
251
|
return;
|
|
561
252
|
const completedTurnId = extractTurnId(notification.params);
|
|
@@ -563,13 +254,13 @@ export class RunOrchestrator {
|
|
|
563
254
|
if (status === "failed") {
|
|
564
255
|
const nextState = isRequestedChangesRunType(run.runType) ? "escalated" : "failed";
|
|
565
256
|
const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
566
|
-
this.db.finishRunWithLease(lease, run.id, {
|
|
257
|
+
this.db.issueSessions.finishRunWithLease(lease, run.id, {
|
|
567
258
|
status: "failed",
|
|
568
259
|
threadId,
|
|
569
260
|
...(completedTurnId ? { turnId: completedTurnId } : {}),
|
|
570
261
|
failureReason: "Codex reported the turn completed in a failed state",
|
|
571
262
|
});
|
|
572
|
-
this.db.upsertIssueWithLease(lease, {
|
|
263
|
+
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
573
264
|
projectId: run.projectId,
|
|
574
265
|
linearIssueId: run.linearIssueId,
|
|
575
266
|
activeRunId: null,
|
|
@@ -591,7 +282,7 @@ export class RunOrchestrator {
|
|
|
591
282
|
status: "failed",
|
|
592
283
|
summary: `Turn failed for ${run.runType}`,
|
|
593
284
|
});
|
|
594
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
285
|
+
const failedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
595
286
|
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType));
|
|
596
287
|
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
597
288
|
this.linearSync.clearProgress(run.id);
|
|
@@ -599,158 +290,23 @@ export class RunOrchestrator {
|
|
|
599
290
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
600
291
|
return;
|
|
601
292
|
}
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
this.failRunAndClear(run, verifiedRepairError, holdState);
|
|
611
|
-
const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
612
|
-
this.feed?.publish({
|
|
613
|
-
level: "warn",
|
|
614
|
-
kind: "turn",
|
|
615
|
-
issueKey: freshIssue.issueKey,
|
|
616
|
-
projectId: run.projectId,
|
|
617
|
-
stage: run.runType,
|
|
618
|
-
status: "branch_not_advanced",
|
|
619
|
-
summary: verifiedRepairError,
|
|
620
|
-
});
|
|
621
|
-
void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
|
|
622
|
-
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
623
|
-
this.linearSync.clearProgress(run.id);
|
|
624
|
-
this.activeThreadId = undefined;
|
|
625
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
626
|
-
return;
|
|
627
|
-
}
|
|
628
|
-
const missingReviewFixHeadError = await this.verifyReviewFixAdvancedHead(run, freshIssue);
|
|
629
|
-
if (missingReviewFixHeadError) {
|
|
630
|
-
this.failRunAndClear(run, missingReviewFixHeadError, "escalated");
|
|
631
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
632
|
-
this.feed?.publish({
|
|
633
|
-
level: "error",
|
|
634
|
-
kind: "turn",
|
|
635
|
-
issueKey: freshIssue.issueKey,
|
|
636
|
-
projectId: run.projectId,
|
|
637
|
-
stage: run.runType,
|
|
638
|
-
status: "same_head_review_handoff_blocked",
|
|
639
|
-
summary: missingReviewFixHeadError,
|
|
640
|
-
});
|
|
641
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
|
|
642
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
643
|
-
this.linearSync.clearProgress(run.id);
|
|
644
|
-
this.activeThreadId = undefined;
|
|
645
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
646
|
-
return;
|
|
647
|
-
}
|
|
648
|
-
const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
|
|
649
|
-
if (publishedOutcomeError) {
|
|
650
|
-
this.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
651
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
652
|
-
this.feed?.publish({
|
|
653
|
-
level: "warn",
|
|
654
|
-
kind: "turn",
|
|
655
|
-
issueKey: freshIssue.issueKey,
|
|
656
|
-
projectId: run.projectId,
|
|
657
|
-
stage: run.runType,
|
|
658
|
-
status: "publish_incomplete",
|
|
659
|
-
summary: publishedOutcomeError,
|
|
660
|
-
});
|
|
661
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
662
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
663
|
-
this.linearSync.clearProgress(run.id);
|
|
664
|
-
this.activeThreadId = undefined;
|
|
665
|
-
return;
|
|
666
|
-
}
|
|
667
|
-
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
668
|
-
const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
|
|
669
|
-
const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
|
|
670
|
-
const completed = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
671
|
-
this.db.finishRun(run.id, {
|
|
672
|
-
status: "completed",
|
|
673
|
-
threadId,
|
|
674
|
-
...(completedTurnId ? { turnId: completedTurnId } : {}),
|
|
675
|
-
summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
|
|
676
|
-
reportJson: JSON.stringify(report),
|
|
677
|
-
});
|
|
678
|
-
this.db.upsertIssue({
|
|
679
|
-
projectId: run.projectId,
|
|
680
|
-
linearIssueId: run.linearIssueId,
|
|
681
|
-
activeRunId: null,
|
|
682
|
-
...(postRunState ? { factoryState: postRunState } : {}),
|
|
683
|
-
pendingRunType: null,
|
|
684
|
-
pendingRunContextJson: null,
|
|
685
|
-
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
686
|
-
? {
|
|
687
|
-
lastGitHubFailureSource: null,
|
|
688
|
-
lastGitHubFailureHeadSha: null,
|
|
689
|
-
lastGitHubFailureSignature: null,
|
|
690
|
-
lastGitHubFailureCheckName: null,
|
|
691
|
-
lastGitHubFailureCheckUrl: null,
|
|
692
|
-
lastGitHubFailureContextJson: null,
|
|
693
|
-
lastGitHubFailureAt: null,
|
|
694
|
-
lastQueueIncidentJson: null,
|
|
695
|
-
lastAttemptedFailureHeadSha: null,
|
|
696
|
-
lastAttemptedFailureSignature: null,
|
|
697
|
-
}
|
|
698
|
-
: {})),
|
|
699
|
-
});
|
|
700
|
-
if (postRunFollowUp) {
|
|
701
|
-
return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
702
|
-
}
|
|
703
|
-
return true;
|
|
704
|
-
});
|
|
705
|
-
if (!completed) {
|
|
706
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping completion writes after losing issue-session lease");
|
|
707
|
-
this.linearSync.clearProgress(run.id);
|
|
708
|
-
this.activeThreadId = undefined;
|
|
709
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
710
|
-
return;
|
|
711
|
-
}
|
|
712
|
-
if (postRunFollowUp) {
|
|
713
|
-
this.feed?.publish({
|
|
714
|
-
level: "info",
|
|
715
|
-
kind: "stage",
|
|
716
|
-
issueKey: issue.issueKey,
|
|
717
|
-
projectId: run.projectId,
|
|
718
|
-
stage: postRunFollowUp.factoryState,
|
|
719
|
-
status: "follow_up_queued",
|
|
720
|
-
summary: postRunFollowUp.summary,
|
|
721
|
-
});
|
|
722
|
-
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
723
|
-
}
|
|
724
|
-
this.feed?.publish({
|
|
725
|
-
level: "info",
|
|
726
|
-
kind: "turn",
|
|
727
|
-
issueKey: issue.issueKey,
|
|
728
|
-
projectId: run.projectId,
|
|
729
|
-
stage: run.runType,
|
|
730
|
-
status: "completed",
|
|
731
|
-
summary: `Turn completed for ${run.runType}`,
|
|
732
|
-
detail: summarizeCurrentThread(thread).latestAgentMessage,
|
|
293
|
+
await this.runFinalizer.finalizeCompletedRun({
|
|
294
|
+
source: "notification",
|
|
295
|
+
run,
|
|
296
|
+
issue,
|
|
297
|
+
thread,
|
|
298
|
+
threadId,
|
|
299
|
+
...(completedTurnId ? { completedTurnId } : {}),
|
|
300
|
+
resolveRecoverableRunState: resolveRecoverablePostRunState,
|
|
733
301
|
});
|
|
734
|
-
// Emit Linear completion activity + plan
|
|
735
|
-
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
736
|
-
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
737
|
-
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
738
|
-
runType: run.runType,
|
|
739
|
-
completionSummary,
|
|
740
|
-
postRunState: updatedIssue.factoryState,
|
|
741
|
-
...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
|
|
742
|
-
}));
|
|
743
|
-
void this.linearSync.syncSession(updatedIssue);
|
|
744
|
-
this.linearSync.clearProgress(run.id);
|
|
745
302
|
this.activeThreadId = undefined;
|
|
746
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
747
303
|
}
|
|
748
304
|
// ─── Active status for query ──────────────────────────────────────
|
|
749
305
|
async getActiveRunStatus(issueKey) {
|
|
750
|
-
const issue = this.db.getIssueByKey(issueKey);
|
|
306
|
+
const issue = this.db.issues.getIssueByKey(issueKey);
|
|
751
307
|
if (!issue?.activeRunId)
|
|
752
308
|
return undefined;
|
|
753
|
-
const run = this.db.
|
|
309
|
+
const run = this.db.runs.getRunById(issue.activeRunId);
|
|
754
310
|
if (!run?.threadId)
|
|
755
311
|
return undefined;
|
|
756
312
|
const trackedIssue = this.db.issueToTrackedIssue(issue);
|
|
@@ -763,7 +319,7 @@ export class RunOrchestrator {
|
|
|
763
319
|
}
|
|
764
320
|
// ─── Reconciliation ───────────────────────────────────────────────
|
|
765
321
|
async reconcileActiveRuns() {
|
|
766
|
-
for (const run of this.db.listRunningRuns()) {
|
|
322
|
+
for (const run of this.db.runs.listRunningRuns()) {
|
|
767
323
|
await this.reconcileRun(run);
|
|
768
324
|
}
|
|
769
325
|
// Preemptively detect stuck merge-queue PRs (conflicts visible on
|
|
@@ -775,7 +331,7 @@ export class RunOrchestrator {
|
|
|
775
331
|
await this.reconcileMergedLinearCompletion();
|
|
776
332
|
}
|
|
777
333
|
async reconcileMergedLinearCompletion() {
|
|
778
|
-
for (const issue of this.db.listIssues()) {
|
|
334
|
+
for (const issue of this.db.issues.listIssues()) {
|
|
779
335
|
if (issue.prState !== "merged")
|
|
780
336
|
continue;
|
|
781
337
|
if (issue.currentLinearStateType?.trim().toLowerCase() === "completed")
|
|
@@ -790,7 +346,7 @@ export class RunOrchestrator {
|
|
|
790
346
|
continue;
|
|
791
347
|
const normalizedCurrent = liveIssue.stateName?.trim().toLowerCase();
|
|
792
348
|
if (normalizedCurrent === targetState.trim().toLowerCase()) {
|
|
793
|
-
this.db.upsertIssue({
|
|
349
|
+
this.db.issues.upsertIssue({
|
|
794
350
|
projectId: issue.projectId,
|
|
795
351
|
linearIssueId: issue.linearIssueId,
|
|
796
352
|
...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
|
|
@@ -799,7 +355,7 @@ export class RunOrchestrator {
|
|
|
799
355
|
continue;
|
|
800
356
|
}
|
|
801
357
|
const updated = await linear.setIssueState(issue.linearIssueId, targetState);
|
|
802
|
-
this.db.upsertIssue({
|
|
358
|
+
this.db.issues.upsertIssue({
|
|
803
359
|
projectId: issue.projectId,
|
|
804
360
|
linearIssueId: issue.linearIssueId,
|
|
805
361
|
...(updated.stateName ? { currentLinearState: updated.stateName } : {}),
|
|
@@ -821,121 +377,15 @@ export class RunOrchestrator {
|
|
|
821
377
|
* escalate; backoff delay not elapsed → skip.
|
|
822
378
|
*/
|
|
823
379
|
recoverOrEscalate(issue, runType, reason) {
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
830
|
-
this.db.clearPendingIssueSessionEventsWithLease(lease);
|
|
831
|
-
this.db.upsertIssueWithLease(lease, {
|
|
832
|
-
projectId: fresh.projectId,
|
|
833
|
-
linearIssueId: fresh.linearIssueId,
|
|
834
|
-
pendingRunType: null,
|
|
835
|
-
pendingRunContextJson: null,
|
|
836
|
-
factoryState: "escalated",
|
|
837
|
-
});
|
|
838
|
-
return true;
|
|
839
|
-
});
|
|
840
|
-
if (!updated) {
|
|
841
|
-
this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping review-fix recovery escalation after losing issue-session lease");
|
|
842
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
843
|
-
return;
|
|
844
|
-
}
|
|
845
|
-
this.logger.warn({ issueKey: fresh.issueKey, reason }, "Requested-changes run failed before a new head was published — escalating");
|
|
846
|
-
this.feed?.publish({
|
|
847
|
-
level: "error",
|
|
848
|
-
kind: "workflow",
|
|
849
|
-
issueKey: fresh.issueKey,
|
|
850
|
-
projectId: fresh.projectId,
|
|
851
|
-
stage: runType,
|
|
852
|
-
status: "escalated",
|
|
853
|
-
summary: `Requested-changes run failed before publishing a new head (${reason})`,
|
|
854
|
-
});
|
|
855
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
856
|
-
return;
|
|
857
|
-
}
|
|
858
|
-
// If PR already merged, transition to done — no retry needed
|
|
859
|
-
if (fresh.prState === "merged") {
|
|
860
|
-
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
861
|
-
this.db.upsertIssueWithLease(lease, {
|
|
862
|
-
projectId: fresh.projectId,
|
|
863
|
-
linearIssueId: fresh.linearIssueId,
|
|
864
|
-
factoryState: "done",
|
|
865
|
-
zombieRecoveryAttempts: 0,
|
|
866
|
-
lastZombieRecoveryAt: null,
|
|
867
|
-
});
|
|
868
|
-
return true;
|
|
869
|
-
});
|
|
870
|
-
if (!updated) {
|
|
871
|
-
this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping merged recovery completion after losing issue-session lease");
|
|
872
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
873
|
-
return;
|
|
874
|
-
}
|
|
875
|
-
this.logger.info({ issueKey: fresh.issueKey, reason }, "Recovery: PR already merged — transitioning to done");
|
|
876
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
877
|
-
return;
|
|
878
|
-
}
|
|
879
|
-
// Budget check
|
|
880
|
-
const attempts = fresh.zombieRecoveryAttempts + 1;
|
|
881
|
-
if (attempts > DEFAULT_ZOMBIE_RECOVERY_BUDGET) {
|
|
882
|
-
const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
883
|
-
this.db.upsertIssueWithLease(lease, {
|
|
884
|
-
projectId: fresh.projectId,
|
|
885
|
-
linearIssueId: fresh.linearIssueId,
|
|
886
|
-
factoryState: "escalated",
|
|
887
|
-
});
|
|
888
|
-
return true;
|
|
889
|
-
});
|
|
890
|
-
if (!updated) {
|
|
891
|
-
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery escalation after losing issue-session lease");
|
|
892
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
893
|
-
return;
|
|
894
|
-
}
|
|
895
|
-
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: budget exhausted — escalating");
|
|
896
|
-
this.feed?.publish({
|
|
897
|
-
level: "error",
|
|
898
|
-
kind: "workflow",
|
|
899
|
-
issueKey: fresh.issueKey,
|
|
900
|
-
projectId: fresh.projectId,
|
|
901
|
-
stage: "escalated",
|
|
902
|
-
status: "budget_exhausted",
|
|
903
|
-
summary: `${reason} recovery failed after ${DEFAULT_ZOMBIE_RECOVERY_BUDGET} attempts`,
|
|
904
|
-
});
|
|
905
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
906
|
-
return;
|
|
907
|
-
}
|
|
908
|
-
// Exponential backoff — skip if delay hasn't elapsed
|
|
909
|
-
if (fresh.lastZombieRecoveryAt) {
|
|
910
|
-
const elapsed = Date.now() - new Date(fresh.lastZombieRecoveryAt).getTime();
|
|
911
|
-
const delay = ZOMBIE_RECOVERY_BASE_DELAY_MS * Math.pow(2, fresh.zombieRecoveryAttempts);
|
|
912
|
-
if (elapsed < delay) {
|
|
913
|
-
this.logger.debug({ issueKey: fresh.issueKey, attempts: fresh.zombieRecoveryAttempts, delay, elapsed }, "Recovery: backoff not elapsed, skipping");
|
|
914
|
-
return;
|
|
915
|
-
}
|
|
916
|
-
}
|
|
917
|
-
// Re-enqueue with backoff tracking
|
|
918
|
-
const requeued = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
919
|
-
this.db.upsertIssueWithLease(lease, {
|
|
920
|
-
projectId: fresh.projectId,
|
|
921
|
-
linearIssueId: fresh.linearIssueId,
|
|
922
|
-
pendingRunType: null,
|
|
923
|
-
pendingRunContextJson: null,
|
|
924
|
-
zombieRecoveryAttempts: attempts,
|
|
925
|
-
lastZombieRecoveryAt: new Date().toISOString(),
|
|
926
|
-
});
|
|
927
|
-
return this.appendWakeEventWithLease(lease, fresh, runType, undefined, `recovery:${attempts}`);
|
|
380
|
+
this.runRecovery.recoverOrEscalate({
|
|
381
|
+
issue,
|
|
382
|
+
runType,
|
|
383
|
+
reason,
|
|
384
|
+
isRequestedChangesRunType,
|
|
928
385
|
});
|
|
929
|
-
if (!requeued) {
|
|
930
|
-
this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery re-enqueue after losing issue-session lease");
|
|
931
|
-
this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
|
|
932
|
-
return;
|
|
933
|
-
}
|
|
934
|
-
this.enqueueIssue(fresh.projectId, fresh.linearIssueId);
|
|
935
|
-
this.logger.info({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: re-enqueued with backoff");
|
|
936
386
|
}
|
|
937
387
|
async reconcileRun(run) {
|
|
938
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
388
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
939
389
|
if (!issue)
|
|
940
390
|
return;
|
|
941
391
|
let recoveryLease = this.claimLeaseForReconciliation(run.projectId, run.linearIssueId);
|
|
@@ -949,11 +399,11 @@ export class RunOrchestrator {
|
|
|
949
399
|
// (e.g. pr_merged processed, DB manually edited), just release the run.
|
|
950
400
|
if (TERMINAL_STATES.has(issue.factoryState)) {
|
|
951
401
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
952
|
-
this.db.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
953
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
402
|
+
this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
403
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
954
404
|
});
|
|
955
405
|
this.logger.info({ issueKey: issue.issueKey, runId: run.id, factoryState: issue.factoryState }, "Reconciliation: released run on terminal issue");
|
|
956
|
-
const releasedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
406
|
+
const releasedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
957
407
|
void this.linearSync.syncSession(releasedIssue, { activeRunType: run.runType });
|
|
958
408
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
959
409
|
return;
|
|
@@ -962,11 +412,11 @@ export class RunOrchestrator {
|
|
|
962
412
|
if (!run.threadId) {
|
|
963
413
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
|
|
964
414
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
965
|
-
this.db.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
966
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
415
|
+
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
416
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
967
417
|
});
|
|
968
418
|
this.recoverOrEscalate(issue, run.runType, "zombie");
|
|
969
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
419
|
+
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
970
420
|
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "The Codex turn never started before PatchRelay restarted."));
|
|
971
421
|
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
972
422
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
@@ -980,11 +430,11 @@ export class RunOrchestrator {
|
|
|
980
430
|
catch {
|
|
981
431
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
|
|
982
432
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
983
|
-
this.db.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
984
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
433
|
+
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
434
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
985
435
|
});
|
|
986
436
|
this.recoverOrEscalate(issue, run.runType, "stale_thread");
|
|
987
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
437
|
+
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
988
438
|
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "PatchRelay lost the active Codex thread after restart and needs to recover."));
|
|
989
439
|
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
990
440
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
@@ -998,8 +448,8 @@ export class RunOrchestrator {
|
|
|
998
448
|
const stopState = resolveAuthoritativeLinearStopState(linearIssue);
|
|
999
449
|
if (stopState?.isFinal) {
|
|
1000
450
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
1001
|
-
this.db.finishRun(run.id, { status: "released" });
|
|
1002
|
-
this.db.upsertIssue({
|
|
451
|
+
this.db.runs.finishRun(run.id, { status: "released" });
|
|
452
|
+
this.db.issues.upsertIssue({
|
|
1003
453
|
projectId: run.projectId,
|
|
1004
454
|
linearIssueId: run.linearIssueId,
|
|
1005
455
|
activeRunId: null,
|
|
@@ -1016,7 +466,7 @@ export class RunOrchestrator {
|
|
|
1016
466
|
status: "reconciled",
|
|
1017
467
|
summary: `Linear state ${stopState.stateName} \u2192 done`,
|
|
1018
468
|
});
|
|
1019
|
-
const doneIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
469
|
+
const doneIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1020
470
|
void this.linearSync.syncSession(doneIssue, { activeRunType: run.runType });
|
|
1021
471
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1022
472
|
return;
|
|
@@ -1028,250 +478,20 @@ export class RunOrchestrator {
|
|
|
1028
478
|
// The agent may have partially completed work (commits, PR) before interruption.
|
|
1029
479
|
// Reactive loops (CI repair, review fix) will handle follow-up if needed.
|
|
1030
480
|
if (latestTurn?.status === "interrupted") {
|
|
1031
|
-
this.
|
|
1032
|
-
// Interrupted runs are not real failures — undo the budget increment.
|
|
1033
|
-
const repairedCounters = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
1034
|
-
if (run.runType === "ci_repair" && issue.ciRepairAttempts > 0) {
|
|
1035
|
-
this.db.upsertIssueWithLease(lease, {
|
|
1036
|
-
projectId: issue.projectId,
|
|
1037
|
-
linearIssueId: issue.linearIssueId,
|
|
1038
|
-
ciRepairAttempts: issue.ciRepairAttempts - 1,
|
|
1039
|
-
});
|
|
1040
|
-
}
|
|
1041
|
-
else if (run.runType === "queue_repair" && issue.queueRepairAttempts > 0) {
|
|
1042
|
-
this.db.upsertIssueWithLease(lease, {
|
|
1043
|
-
projectId: issue.projectId,
|
|
1044
|
-
linearIssueId: issue.linearIssueId,
|
|
1045
|
-
queueRepairAttempts: issue.queueRepairAttempts - 1,
|
|
1046
|
-
});
|
|
1047
|
-
}
|
|
1048
|
-
if (run.runType === "ci_repair" || run.runType === "queue_repair") {
|
|
1049
|
-
this.db.upsertIssueWithLease(lease, {
|
|
1050
|
-
projectId: issue.projectId,
|
|
1051
|
-
linearIssueId: issue.linearIssueId,
|
|
1052
|
-
lastAttemptedFailureHeadSha: null,
|
|
1053
|
-
lastAttemptedFailureSignature: null,
|
|
1054
|
-
});
|
|
1055
|
-
}
|
|
1056
|
-
return true;
|
|
1057
|
-
});
|
|
1058
|
-
if (!repairedCounters) {
|
|
1059
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping interrupted-run recovery after losing issue-session lease");
|
|
1060
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1061
|
-
return;
|
|
1062
|
-
}
|
|
1063
|
-
if (isRequestedChangesRunType(run.runType)) {
|
|
1064
|
-
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
|
|
1065
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1066
|
-
const retryContext = project
|
|
1067
|
-
? await this.resolveRequestedChangesWakeContext(refreshedIssue, run.runType, run.runType === "branch_upkeep"
|
|
1068
|
-
? {
|
|
1069
|
-
branchUpkeepRequired: true,
|
|
1070
|
-
reviewFixMode: "branch_upkeep",
|
|
1071
|
-
wakeReason: "branch_upkeep",
|
|
1072
|
-
}
|
|
1073
|
-
: undefined, project)
|
|
1074
|
-
: undefined;
|
|
1075
|
-
const retryRunType = resolveRequestedChangesMode(run.runType, retryContext) === "branch_upkeep"
|
|
1076
|
-
? "branch_upkeep"
|
|
1077
|
-
: "review_fix";
|
|
1078
|
-
const recoveredState = resolveRecoverablePostRunState(refreshedIssue) ?? "failed";
|
|
1079
|
-
const interruptedMessage = "Requested-changes run was interrupted before PatchRelay could verify that a new PR head was published";
|
|
1080
|
-
this.failRunAndClear(run, interruptedMessage, recoveredState);
|
|
1081
|
-
await this.restoreIdleWorktree(issue);
|
|
1082
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
1083
|
-
if (recoveredState === "changes_requested") {
|
|
1084
|
-
this.db.upsertIssue({
|
|
1085
|
-
projectId: run.projectId,
|
|
1086
|
-
linearIssueId: run.linearIssueId,
|
|
1087
|
-
pendingRunType: retryRunType,
|
|
1088
|
-
pendingRunContextJson: retryContext ? JSON.stringify(retryContext) : null,
|
|
1089
|
-
});
|
|
1090
|
-
this.feed?.publish({
|
|
1091
|
-
level: "warn",
|
|
1092
|
-
kind: "workflow",
|
|
1093
|
-
issueKey: issue.issueKey,
|
|
1094
|
-
projectId: run.projectId,
|
|
1095
|
-
stage: run.runType,
|
|
1096
|
-
status: "retry_queued",
|
|
1097
|
-
summary: "Requested-changes run was interrupted; PatchRelay will retry from fresh GitHub truth",
|
|
1098
|
-
});
|
|
1099
|
-
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
1100
|
-
}
|
|
1101
|
-
else {
|
|
1102
|
-
this.feed?.publish({
|
|
1103
|
-
level: "error",
|
|
1104
|
-
kind: "workflow",
|
|
1105
|
-
issueKey: issue.issueKey,
|
|
1106
|
-
projectId: run.projectId,
|
|
1107
|
-
stage: run.runType,
|
|
1108
|
-
status: "escalated",
|
|
1109
|
-
summary: interruptedMessage,
|
|
1110
|
-
});
|
|
1111
|
-
}
|
|
1112
|
-
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, interruptedMessage));
|
|
1113
|
-
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
1114
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1115
|
-
return;
|
|
1116
|
-
}
|
|
1117
|
-
const recoveredState = resolveRecoverablePostRunState(this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
|
|
1118
|
-
this.failRunAndClear(run, "Codex turn was interrupted", recoveredState);
|
|
1119
|
-
await this.restoreIdleWorktree(issue);
|
|
1120
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1121
|
-
if (recoveredState) {
|
|
1122
|
-
this.feed?.publish({
|
|
1123
|
-
level: "info",
|
|
1124
|
-
kind: "stage",
|
|
1125
|
-
issueKey: issue.issueKey,
|
|
1126
|
-
projectId: run.projectId,
|
|
1127
|
-
stage: recoveredState,
|
|
1128
|
-
status: "reconciled",
|
|
1129
|
-
summary: `Interrupted ${run.runType} recovered \u2192 ${recoveredState}`,
|
|
1130
|
-
});
|
|
1131
|
-
}
|
|
1132
|
-
else {
|
|
1133
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, "The Codex turn was interrupted."));
|
|
1134
|
-
}
|
|
1135
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1136
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
481
|
+
await this.interruptedRunRecovery.handle(run, issue);
|
|
1137
482
|
return;
|
|
1138
483
|
}
|
|
1139
484
|
// Handle completed turn discovered during reconciliation
|
|
1140
485
|
if (latestTurn?.status === "completed") {
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
level: "warn",
|
|
1150
|
-
kind: "turn",
|
|
1151
|
-
issueKey: issue.issueKey,
|
|
1152
|
-
projectId: run.projectId,
|
|
1153
|
-
stage: run.runType,
|
|
1154
|
-
status: "branch_not_advanced",
|
|
1155
|
-
summary: verifiedRepairError,
|
|
1156
|
-
});
|
|
1157
|
-
const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1158
|
-
void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
|
|
1159
|
-
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
1160
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1161
|
-
return;
|
|
1162
|
-
}
|
|
1163
|
-
const missingReviewFixHeadError = await this.verifyReviewFixAdvancedHead(run, freshIssue);
|
|
1164
|
-
if (missingReviewFixHeadError) {
|
|
1165
|
-
this.failRunAndClear(run, missingReviewFixHeadError, "escalated");
|
|
1166
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1167
|
-
this.feed?.publish({
|
|
1168
|
-
level: "error",
|
|
1169
|
-
kind: "turn",
|
|
1170
|
-
issueKey: freshIssue.issueKey,
|
|
1171
|
-
projectId: run.projectId,
|
|
1172
|
-
stage: run.runType,
|
|
1173
|
-
status: "same_head_review_handoff_blocked",
|
|
1174
|
-
summary: missingReviewFixHeadError,
|
|
1175
|
-
});
|
|
1176
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
|
|
1177
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1178
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1179
|
-
return;
|
|
1180
|
-
}
|
|
1181
|
-
const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
|
|
1182
|
-
if (publishedOutcomeError) {
|
|
1183
|
-
this.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
1184
|
-
this.feed?.publish({
|
|
1185
|
-
level: "warn",
|
|
1186
|
-
kind: "turn",
|
|
1187
|
-
issueKey: issue.issueKey,
|
|
1188
|
-
projectId: run.projectId,
|
|
1189
|
-
stage: run.runType,
|
|
1190
|
-
status: "publish_incomplete",
|
|
1191
|
-
summary: publishedOutcomeError,
|
|
1192
|
-
});
|
|
1193
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
1194
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
1195
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1196
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1197
|
-
return;
|
|
1198
|
-
}
|
|
1199
|
-
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
1200
|
-
const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
|
|
1201
|
-
const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
|
|
1202
|
-
const reconciled = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
|
|
1203
|
-
this.db.finishRun(run.id, {
|
|
1204
|
-
status: "completed",
|
|
1205
|
-
...(run.threadId ? { threadId: run.threadId } : {}),
|
|
1206
|
-
...(latestTurn.id ? { turnId: latestTurn.id } : {}),
|
|
1207
|
-
summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
|
|
1208
|
-
reportJson: JSON.stringify(report),
|
|
1209
|
-
});
|
|
1210
|
-
this.db.upsertIssue({
|
|
1211
|
-
projectId: run.projectId,
|
|
1212
|
-
linearIssueId: run.linearIssueId,
|
|
1213
|
-
activeRunId: null,
|
|
1214
|
-
...(postRunState ? { factoryState: postRunState } : {}),
|
|
1215
|
-
pendingRunType: null,
|
|
1216
|
-
pendingRunContextJson: null,
|
|
1217
|
-
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
1218
|
-
? {
|
|
1219
|
-
lastGitHubFailureSource: null,
|
|
1220
|
-
lastGitHubFailureHeadSha: null,
|
|
1221
|
-
lastGitHubFailureSignature: null,
|
|
1222
|
-
lastGitHubFailureCheckName: null,
|
|
1223
|
-
lastGitHubFailureCheckUrl: null,
|
|
1224
|
-
lastGitHubFailureContextJson: null,
|
|
1225
|
-
lastGitHubFailureAt: null,
|
|
1226
|
-
lastQueueIncidentJson: null,
|
|
1227
|
-
lastAttemptedFailureHeadSha: null,
|
|
1228
|
-
lastAttemptedFailureSignature: null,
|
|
1229
|
-
}
|
|
1230
|
-
: {})),
|
|
1231
|
-
});
|
|
1232
|
-
if (postRunFollowUp) {
|
|
1233
|
-
return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
1234
|
-
}
|
|
1235
|
-
return true;
|
|
486
|
+
await this.runFinalizer.finalizeCompletedRun({
|
|
487
|
+
source: "reconciliation",
|
|
488
|
+
run,
|
|
489
|
+
issue,
|
|
490
|
+
thread,
|
|
491
|
+
threadId: run.threadId,
|
|
492
|
+
...(latestTurn.id ? { completedTurnId: latestTurn.id } : {}),
|
|
493
|
+
resolveRecoverableRunState: resolveRecoverablePostRunState,
|
|
1236
494
|
});
|
|
1237
|
-
if (!reconciled) {
|
|
1238
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping reconciled completion writes after losing issue-session lease");
|
|
1239
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1240
|
-
return;
|
|
1241
|
-
}
|
|
1242
|
-
if (postRunFollowUp) {
|
|
1243
|
-
this.feed?.publish({
|
|
1244
|
-
level: "info",
|
|
1245
|
-
kind: "stage",
|
|
1246
|
-
issueKey: issue.issueKey,
|
|
1247
|
-
projectId: run.projectId,
|
|
1248
|
-
stage: postRunFollowUp.factoryState,
|
|
1249
|
-
status: "follow_up_queued",
|
|
1250
|
-
summary: postRunFollowUp.summary,
|
|
1251
|
-
});
|
|
1252
|
-
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
1253
|
-
}
|
|
1254
|
-
if (postRunState) {
|
|
1255
|
-
this.feed?.publish({
|
|
1256
|
-
level: "info",
|
|
1257
|
-
kind: "turn",
|
|
1258
|
-
issueKey: issue.issueKey,
|
|
1259
|
-
projectId: run.projectId,
|
|
1260
|
-
stage: run.runType,
|
|
1261
|
-
status: "completed",
|
|
1262
|
-
summary: `Reconciliation: ${run.runType} completed \u2192 ${postRunState}`,
|
|
1263
|
-
});
|
|
1264
|
-
}
|
|
1265
|
-
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
1266
|
-
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
1267
|
-
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
1268
|
-
runType: run.runType,
|
|
1269
|
-
completionSummary,
|
|
1270
|
-
postRunState: updatedIssue.factoryState,
|
|
1271
|
-
...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
|
|
1272
|
-
}));
|
|
1273
|
-
void this.linearSync.syncSession(updatedIssue);
|
|
1274
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1275
495
|
return;
|
|
1276
496
|
}
|
|
1277
497
|
if (acquiredRecoveryLease)
|
|
@@ -1279,402 +499,24 @@ export class RunOrchestrator {
|
|
|
1279
499
|
}
|
|
1280
500
|
// ─── Internal helpers ─────────────────────────────────────────────
|
|
1281
501
|
escalate(issue, runType, reason) {
|
|
1282
|
-
this.
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
}
|
|
1287
|
-
this.db.clearPendingIssueSessionEventsWithLease(lease);
|
|
1288
|
-
this.db.upsertIssueWithLease(lease, {
|
|
1289
|
-
projectId: issue.projectId,
|
|
1290
|
-
linearIssueId: issue.linearIssueId,
|
|
1291
|
-
pendingRunType: null,
|
|
1292
|
-
pendingRunContextJson: null,
|
|
1293
|
-
activeRunId: null,
|
|
1294
|
-
factoryState: "escalated",
|
|
1295
|
-
});
|
|
1296
|
-
return true;
|
|
1297
|
-
});
|
|
1298
|
-
if (!escalated) {
|
|
1299
|
-
this.logger.warn({ issueKey: issue.issueKey, runType }, "Skipping escalation write after losing issue-session lease");
|
|
1300
|
-
this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
1301
|
-
return;
|
|
1302
|
-
}
|
|
1303
|
-
this.feed?.publish({
|
|
1304
|
-
level: "error",
|
|
1305
|
-
kind: "workflow",
|
|
1306
|
-
issueKey: issue.issueKey,
|
|
1307
|
-
projectId: issue.projectId,
|
|
1308
|
-
stage: runType,
|
|
1309
|
-
status: "escalated",
|
|
1310
|
-
summary: `Escalated: ${reason}`,
|
|
1311
|
-
});
|
|
1312
|
-
const escalatedIssue = this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
|
|
1313
|
-
void this.linearSync.emitActivity(escalatedIssue, {
|
|
1314
|
-
type: "error",
|
|
1315
|
-
body: `PatchRelay needs human help to continue.\n\n${reason}`,
|
|
502
|
+
this.runRecovery.escalate({
|
|
503
|
+
issue,
|
|
504
|
+
runType,
|
|
505
|
+
reason,
|
|
1316
506
|
});
|
|
1317
|
-
void this.linearSync.syncSession(escalatedIssue);
|
|
1318
|
-
this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
1319
507
|
}
|
|
1320
508
|
failRunAndClear(run, message, nextState = "failed") {
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
}
|
|
1326
|
-
this.db.upsertIssue({
|
|
1327
|
-
projectId: run.projectId,
|
|
1328
|
-
linearIssueId: run.linearIssueId,
|
|
1329
|
-
activeRunId: null,
|
|
1330
|
-
factoryState: nextState,
|
|
1331
|
-
});
|
|
1332
|
-
const branchOwner = this.resolveBranchOwnerForStateTransition(nextState);
|
|
1333
|
-
if (branchOwner) {
|
|
1334
|
-
const lease = this.getHeldIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1335
|
-
if (lease) {
|
|
1336
|
-
this.db.setBranchOwnerWithLease(lease, branchOwner);
|
|
1337
|
-
}
|
|
1338
|
-
}
|
|
1339
|
-
return true;
|
|
509
|
+
this.runRecovery.failRunAndClear({
|
|
510
|
+
run,
|
|
511
|
+
message,
|
|
512
|
+
nextState,
|
|
1340
513
|
});
|
|
1341
|
-
if (!updated) {
|
|
1342
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping failure cleanup after losing issue-session lease");
|
|
1343
|
-
}
|
|
1344
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1345
514
|
}
|
|
1346
515
|
resolveBranchOwnerForStateTransition(newState, pendingRunType) {
|
|
1347
516
|
return resolveBranchOwnerForStateTransition(newState, pendingRunType);
|
|
1348
517
|
}
|
|
1349
|
-
async
|
|
1350
|
-
|
|
1351
|
-
return undefined;
|
|
1352
|
-
}
|
|
1353
|
-
if (!issue.prNumber || issue.prState !== "open" || !issue.lastGitHubFailureHeadSha) {
|
|
1354
|
-
return undefined;
|
|
1355
|
-
}
|
|
1356
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1357
|
-
if (!project?.github?.repoFullName) {
|
|
1358
|
-
return undefined;
|
|
1359
|
-
}
|
|
1360
|
-
try {
|
|
1361
|
-
const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
|
|
1362
|
-
if (!pr || pr.state?.toUpperCase() !== "OPEN")
|
|
1363
|
-
return undefined;
|
|
1364
|
-
if (!pr.headRefOid || pr.headRefOid !== issue.lastGitHubFailureHeadSha)
|
|
1365
|
-
return undefined;
|
|
1366
|
-
return `Repair finished but PR #${issue.prNumber} is still on failing head ${issue.lastGitHubFailureHeadSha.slice(0, 8)}`;
|
|
1367
|
-
}
|
|
1368
|
-
catch (error) {
|
|
1369
|
-
this.logger.debug({
|
|
1370
|
-
issueKey: issue.issueKey,
|
|
1371
|
-
prNumber: issue.prNumber,
|
|
1372
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1373
|
-
}, "Failed to verify PR head advancement after repair");
|
|
1374
|
-
return undefined;
|
|
1375
|
-
}
|
|
1376
|
-
}
|
|
1377
|
-
async verifyReviewFixAdvancedHead(run, issue) {
|
|
1378
|
-
if (!isRequestedChangesRunType(run.runType)) {
|
|
1379
|
-
return undefined;
|
|
1380
|
-
}
|
|
1381
|
-
if (!issue.prNumber || issue.prState !== "open") {
|
|
1382
|
-
return undefined;
|
|
1383
|
-
}
|
|
1384
|
-
if (!run.sourceHeadSha) {
|
|
1385
|
-
return `Requested-changes run finished for PR #${issue.prNumber} without a recorded starting head SHA. PatchRelay cannot verify that a new head was published.`;
|
|
1386
|
-
}
|
|
1387
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1388
|
-
if (!project?.github?.repoFullName) {
|
|
1389
|
-
return undefined;
|
|
1390
|
-
}
|
|
1391
|
-
try {
|
|
1392
|
-
const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
|
|
1393
|
-
if (!pr || pr.state?.toUpperCase() !== "OPEN")
|
|
1394
|
-
return undefined;
|
|
1395
|
-
if (!pr.headRefOid) {
|
|
1396
|
-
return `Requested-changes run finished for PR #${issue.prNumber} but GitHub did not report a current head SHA.`;
|
|
1397
|
-
}
|
|
1398
|
-
if (pr.headRefOid === run.sourceHeadSha) {
|
|
1399
|
-
return `Requested-changes run finished for PR #${issue.prNumber} without pushing a new head; PatchRelay must not hand the same SHA back to review.`;
|
|
1400
|
-
}
|
|
1401
|
-
return undefined;
|
|
1402
|
-
}
|
|
1403
|
-
catch (error) {
|
|
1404
|
-
this.logger.debug({
|
|
1405
|
-
issueKey: issue.issueKey,
|
|
1406
|
-
prNumber: issue.prNumber,
|
|
1407
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1408
|
-
}, "Failed to verify PR head advancement after requested-changes work");
|
|
1409
|
-
return undefined;
|
|
1410
|
-
}
|
|
1411
|
-
}
|
|
1412
|
-
async refreshIssueAfterReactivePublish(run, issue) {
|
|
1413
|
-
if (run.runType !== "ci_repair" && run.runType !== "queue_repair" && !isRequestedChangesRunType(run.runType)) {
|
|
1414
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1415
|
-
}
|
|
1416
|
-
if (!issue.prNumber) {
|
|
1417
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1418
|
-
}
|
|
1419
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1420
|
-
const repoFullName = project?.github?.repoFullName;
|
|
1421
|
-
if (!repoFullName) {
|
|
1422
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1423
|
-
}
|
|
1424
|
-
try {
|
|
1425
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1426
|
-
if (!pr) {
|
|
1427
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1428
|
-
}
|
|
1429
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1430
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1431
|
-
const gateCheckName = project?.gateChecks?.find((entry) => entry.trim())?.trim() ?? "verify";
|
|
1432
|
-
const headAdvanced = Boolean(pr.headRefOid && pr.headRefOid !== issue.lastGitHubFailureHeadSha);
|
|
1433
|
-
const reviewFixHeadAdvanced = isRequestedChangesRunType(run.runType)
|
|
1434
|
-
&& Boolean(pr.headRefOid && run.sourceHeadSha && pr.headRefOid !== run.sourceHeadSha);
|
|
1435
|
-
this.upsertIssueIfLeaseHeld(run.projectId, run.linearIssueId, {
|
|
1436
|
-
projectId: run.projectId,
|
|
1437
|
-
linearIssueId: run.linearIssueId,
|
|
1438
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1439
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1440
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1441
|
-
...((headAdvanced || reviewFixHeadAdvanced)
|
|
1442
|
-
? {
|
|
1443
|
-
prCheckStatus: "pending",
|
|
1444
|
-
lastGitHubFailureSource: null,
|
|
1445
|
-
lastGitHubFailureHeadSha: null,
|
|
1446
|
-
lastGitHubFailureSignature: null,
|
|
1447
|
-
lastGitHubFailureCheckName: null,
|
|
1448
|
-
lastGitHubFailureCheckUrl: null,
|
|
1449
|
-
lastGitHubFailureContextJson: null,
|
|
1450
|
-
lastGitHubFailureAt: null,
|
|
1451
|
-
lastQueueIncidentJson: null,
|
|
1452
|
-
lastAttemptedFailureHeadSha: null,
|
|
1453
|
-
lastAttemptedFailureSignature: null,
|
|
1454
|
-
lastGitHubCiSnapshotHeadSha: pr.headRefOid ?? null,
|
|
1455
|
-
lastGitHubCiSnapshotGateCheckName: gateCheckName,
|
|
1456
|
-
lastGitHubCiSnapshotGateCheckStatus: "pending",
|
|
1457
|
-
lastGitHubCiSnapshotJson: null,
|
|
1458
|
-
lastGitHubCiSnapshotSettledAt: null,
|
|
1459
|
-
}
|
|
1460
|
-
: {}),
|
|
1461
|
-
}, "reactive publish refresh");
|
|
1462
|
-
}
|
|
1463
|
-
catch (error) {
|
|
1464
|
-
this.logger.debug({
|
|
1465
|
-
issueKey: issue.issueKey,
|
|
1466
|
-
prNumber: issue.prNumber,
|
|
1467
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1468
|
-
}, "Failed to refresh PR state after reactive publish");
|
|
1469
|
-
}
|
|
1470
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
1471
|
-
}
|
|
1472
|
-
async loadRemotePrState(repoFullName, prNumber) {
|
|
1473
|
-
const { stdout, exitCode } = await execCommand("gh", [
|
|
1474
|
-
"pr", "view", String(prNumber),
|
|
1475
|
-
"--repo", repoFullName,
|
|
1476
|
-
"--json", "headRefOid,state,reviewDecision,mergeStateStatus",
|
|
1477
|
-
], { timeoutMs: 10_000 });
|
|
1478
|
-
if (exitCode !== 0)
|
|
1479
|
-
return undefined;
|
|
1480
|
-
return JSON.parse(stdout);
|
|
1481
|
-
}
|
|
1482
|
-
async resolveRequestedChangesWakeContext(issue, runType, context, project) {
|
|
1483
|
-
if (runType === "branch_upkeep" || isBranchUpkeepRequired(context)) {
|
|
1484
|
-
return context;
|
|
1485
|
-
}
|
|
1486
|
-
if (!issue.prNumber || issue.prState !== "open" || issue.prReviewState !== "changes_requested") {
|
|
1487
|
-
return context;
|
|
1488
|
-
}
|
|
1489
|
-
const repoFullName = project.github?.repoFullName;
|
|
1490
|
-
if (!repoFullName) {
|
|
1491
|
-
return context;
|
|
1492
|
-
}
|
|
1493
|
-
try {
|
|
1494
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1495
|
-
if (!pr)
|
|
1496
|
-
return context;
|
|
1497
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1498
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1499
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1500
|
-
projectId: issue.projectId,
|
|
1501
|
-
linearIssueId: issue.linearIssueId,
|
|
1502
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1503
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1504
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1505
|
-
}, "review-fix wake refresh");
|
|
1506
|
-
if (nextPrState !== "open")
|
|
1507
|
-
return context;
|
|
1508
|
-
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
1509
|
-
return context;
|
|
1510
|
-
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
1511
|
-
return context;
|
|
1512
|
-
return buildReviewFixBranchUpkeepContext(issue.prNumber, project.github?.baseBranch ?? "main", pr, context);
|
|
1513
|
-
}
|
|
1514
|
-
catch (error) {
|
|
1515
|
-
this.logger.debug({
|
|
1516
|
-
issueKey: issue.issueKey,
|
|
1517
|
-
prNumber: issue.prNumber,
|
|
1518
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1519
|
-
}, "Failed to resolve requested-changes wake context");
|
|
1520
|
-
return context;
|
|
1521
|
-
}
|
|
1522
|
-
}
|
|
1523
|
-
async resolvePostRunFollowUp(run, issue, projectOverride) {
|
|
1524
|
-
if (run.runType !== "review_fix") {
|
|
1525
|
-
return undefined;
|
|
1526
|
-
}
|
|
1527
|
-
if (!issue.prNumber || issue.prState !== "open") {
|
|
1528
|
-
return undefined;
|
|
1529
|
-
}
|
|
1530
|
-
if (issue.prReviewState !== "changes_requested") {
|
|
1531
|
-
return undefined;
|
|
1532
|
-
}
|
|
1533
|
-
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1534
|
-
const repoFullName = project?.github?.repoFullName;
|
|
1535
|
-
if (!repoFullName) {
|
|
1536
|
-
return undefined;
|
|
1537
|
-
}
|
|
1538
|
-
try {
|
|
1539
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
1540
|
-
if (!pr)
|
|
1541
|
-
return undefined;
|
|
1542
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
1543
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
1544
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1545
|
-
projectId: issue.projectId,
|
|
1546
|
-
linearIssueId: issue.linearIssueId,
|
|
1547
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
1548
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1549
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
1550
|
-
}, "post-run follow-up refresh");
|
|
1551
|
-
if (nextPrState !== "open")
|
|
1552
|
-
return undefined;
|
|
1553
|
-
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
1554
|
-
return undefined;
|
|
1555
|
-
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
1556
|
-
return undefined;
|
|
1557
|
-
return {
|
|
1558
|
-
pendingRunType: "branch_upkeep",
|
|
1559
|
-
factoryState: "changes_requested",
|
|
1560
|
-
context: buildReviewFixBranchUpkeepContext(issue.prNumber, project?.github?.baseBranch ?? "main", pr),
|
|
1561
|
-
summary: `PR #${issue.prNumber} is still dirty after review fix; queued branch upkeep`,
|
|
1562
|
-
};
|
|
1563
|
-
}
|
|
1564
|
-
catch (error) {
|
|
1565
|
-
this.logger.debug({
|
|
1566
|
-
issueKey: issue.issueKey,
|
|
1567
|
-
prNumber: issue.prNumber,
|
|
1568
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1569
|
-
}, "Failed to resolve post-run PR upkeep");
|
|
1570
|
-
return undefined;
|
|
1571
|
-
}
|
|
1572
|
-
}
|
|
1573
|
-
async verifyPublishedRunOutcome(run, issue, projectOverride) {
|
|
1574
|
-
if (run.runType !== "implementation") {
|
|
1575
|
-
return undefined;
|
|
1576
|
-
}
|
|
1577
|
-
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
1578
|
-
const baseBranch = project?.github?.baseBranch ?? "main";
|
|
1579
|
-
const deliveryMode = resolveImplementationDeliveryMode(issue, undefined, run.promptText);
|
|
1580
|
-
if (deliveryMode === "linear_only") {
|
|
1581
|
-
if (issue.prNumber !== undefined) {
|
|
1582
|
-
return `Planning-only implementation should not open a PR, but PR #${issue.prNumber} was observed`;
|
|
1583
|
-
}
|
|
1584
|
-
return this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
1585
|
-
}
|
|
1586
|
-
if (issue.prNumber && issue.prState && issue.prState !== "closed") {
|
|
1587
|
-
return undefined;
|
|
1588
|
-
}
|
|
1589
|
-
if (project?.github?.repoFullName && issue.branchName) {
|
|
1590
|
-
try {
|
|
1591
|
-
const { stdout, exitCode } = await execCommand("gh", [
|
|
1592
|
-
"pr",
|
|
1593
|
-
"list",
|
|
1594
|
-
"--repo",
|
|
1595
|
-
project.github.repoFullName,
|
|
1596
|
-
"--head",
|
|
1597
|
-
issue.branchName,
|
|
1598
|
-
"--state",
|
|
1599
|
-
"all",
|
|
1600
|
-
"--json",
|
|
1601
|
-
"number,url,state,author,headRefOid",
|
|
1602
|
-
], { timeoutMs: 10_000 });
|
|
1603
|
-
if (exitCode === 0) {
|
|
1604
|
-
const matches = JSON.parse(stdout);
|
|
1605
|
-
const pr = matches[0];
|
|
1606
|
-
if (pr?.number) {
|
|
1607
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1608
|
-
projectId: issue.projectId,
|
|
1609
|
-
linearIssueId: issue.linearIssueId,
|
|
1610
|
-
prNumber: pr.number,
|
|
1611
|
-
...(pr.url ? { prUrl: pr.url } : {}),
|
|
1612
|
-
...(pr.state ? { prState: pr.state.toLowerCase() } : {}),
|
|
1613
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1614
|
-
...(pr.author?.login ? { prAuthorLogin: pr.author.login } : {}),
|
|
1615
|
-
}, "published PR verification refresh");
|
|
1616
|
-
return undefined;
|
|
1617
|
-
}
|
|
1618
|
-
}
|
|
1619
|
-
}
|
|
1620
|
-
catch (error) {
|
|
1621
|
-
this.logger.debug({
|
|
1622
|
-
issueKey: issue.issueKey,
|
|
1623
|
-
branchName: issue.branchName,
|
|
1624
|
-
repoFullName: project.github.repoFullName,
|
|
1625
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1626
|
-
}, "Failed to verify published PR state after implementation");
|
|
1627
|
-
}
|
|
1628
|
-
}
|
|
1629
|
-
const details = await this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
1630
|
-
return details ?? `Implementation completed without opening a PR for branch ${issue.branchName ?? issue.linearIssueId}`;
|
|
1631
|
-
}
|
|
1632
|
-
async describeLocalImplementationOutcome(issue, baseBranch, deliveryMode = "publish_pr") {
|
|
1633
|
-
if (!issue.worktreePath) {
|
|
1634
|
-
return undefined;
|
|
1635
|
-
}
|
|
1636
|
-
try {
|
|
1637
|
-
const status = await execCommand(this.config.runner.gitBin, [
|
|
1638
|
-
"-C",
|
|
1639
|
-
issue.worktreePath,
|
|
1640
|
-
"status",
|
|
1641
|
-
"--short",
|
|
1642
|
-
], { timeoutMs: 10_000 });
|
|
1643
|
-
const dirtyEntries = status.exitCode === 0
|
|
1644
|
-
? status.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
1645
|
-
: [];
|
|
1646
|
-
if (dirtyEntries.length > 0) {
|
|
1647
|
-
if (deliveryMode === "linear_only") {
|
|
1648
|
-
return `Planning-only implementation should not modify the repo; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1649
|
-
}
|
|
1650
|
-
return `Implementation completed without opening a PR; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1651
|
-
}
|
|
1652
|
-
}
|
|
1653
|
-
catch {
|
|
1654
|
-
// Best effort only.
|
|
1655
|
-
}
|
|
1656
|
-
try {
|
|
1657
|
-
const ahead = await execCommand(this.config.runner.gitBin, [
|
|
1658
|
-
"-C",
|
|
1659
|
-
issue.worktreePath,
|
|
1660
|
-
"rev-list",
|
|
1661
|
-
"--count",
|
|
1662
|
-
`origin/${baseBranch}..HEAD`,
|
|
1663
|
-
], { timeoutMs: 10_000 });
|
|
1664
|
-
if (ahead.exitCode === 0) {
|
|
1665
|
-
const count = Number(ahead.stdout.trim());
|
|
1666
|
-
if (Number.isFinite(count) && count > 0) {
|
|
1667
|
-
if (deliveryMode === "linear_only") {
|
|
1668
|
-
return `Planning-only implementation should not create repo commits; worktree is ${count} local commit(s) ahead of origin/${baseBranch}`;
|
|
1669
|
-
}
|
|
1670
|
-
return `Implementation completed with ${count} local commit(s) ahead of origin/${baseBranch} but no PR was observed`;
|
|
1671
|
-
}
|
|
1672
|
-
}
|
|
1673
|
-
}
|
|
1674
|
-
catch {
|
|
1675
|
-
// Best effort only.
|
|
1676
|
-
}
|
|
1677
|
-
return undefined;
|
|
518
|
+
async resolveRequestedChangesWakeContext(issue, runType, context) {
|
|
519
|
+
return await this.runCompletionPolicy.resolveRequestedChangesWakeContext(issue, runType, context);
|
|
1678
520
|
}
|
|
1679
521
|
async readThreadWithRetry(threadId, maxRetries = 3) {
|
|
1680
522
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
@@ -1689,32 +531,11 @@ export class RunOrchestrator {
|
|
|
1689
531
|
}
|
|
1690
532
|
throw new Error(`Failed to read thread ${threadId}`);
|
|
1691
533
|
}
|
|
1692
|
-
issueSessionLeaseKey(projectId, linearIssueId) {
|
|
1693
|
-
return `${projectId}:${linearIssueId}`;
|
|
1694
|
-
}
|
|
1695
534
|
getHeldIssueSessionLease(projectId, linearIssueId) {
|
|
1696
|
-
|
|
1697
|
-
if (!leaseId)
|
|
1698
|
-
return undefined;
|
|
1699
|
-
return { projectId, linearIssueId, leaseId };
|
|
535
|
+
return this.leaseService.getHeldLease(projectId, linearIssueId);
|
|
1700
536
|
}
|
|
1701
537
|
withHeldIssueSessionLease(projectId, linearIssueId, fn) {
|
|
1702
|
-
|
|
1703
|
-
if (!lease)
|
|
1704
|
-
return undefined;
|
|
1705
|
-
return this.db.withIssueSessionLease(projectId, linearIssueId, lease.leaseId, () => fn(lease));
|
|
1706
|
-
}
|
|
1707
|
-
upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
|
|
1708
|
-
const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
|
|
1709
|
-
if (!lease) {
|
|
1710
|
-
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write without a held issue-session lease");
|
|
1711
|
-
return undefined;
|
|
1712
|
-
}
|
|
1713
|
-
const updated = this.db.upsertIssueWithLease(lease, params);
|
|
1714
|
-
if (!updated) {
|
|
1715
|
-
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
|
|
1716
|
-
}
|
|
1717
|
-
return updated;
|
|
538
|
+
return this.leaseService.withHeldLease(projectId, linearIssueId, fn);
|
|
1718
539
|
}
|
|
1719
540
|
assertLaunchLease(run, phase) {
|
|
1720
541
|
if (this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
@@ -1725,197 +546,16 @@ export class RunOrchestrator {
|
|
|
1725
546
|
this.logger.warn({ runId: run.id, issueId: run.linearIssueId, phase }, "Aborting run launch after losing issue-session lease");
|
|
1726
547
|
throw error;
|
|
1727
548
|
}
|
|
1728
|
-
acquireIssueSessionLease(projectId, linearIssueId) {
|
|
1729
|
-
const leaseId = randomUUID();
|
|
1730
|
-
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
1731
|
-
const acquired = this.db.acquireIssueSessionLease({
|
|
1732
|
-
projectId,
|
|
1733
|
-
linearIssueId,
|
|
1734
|
-
leaseId,
|
|
1735
|
-
workerId: this.workerId,
|
|
1736
|
-
leasedUntil,
|
|
1737
|
-
});
|
|
1738
|
-
if (!acquired)
|
|
1739
|
-
return undefined;
|
|
1740
|
-
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
1741
|
-
return leaseId;
|
|
1742
|
-
}
|
|
1743
|
-
forceAcquireIssueSessionLease(projectId, linearIssueId) {
|
|
1744
|
-
const leaseId = randomUUID();
|
|
1745
|
-
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
1746
|
-
const acquired = this.db.forceAcquireIssueSessionLease({
|
|
1747
|
-
projectId,
|
|
1748
|
-
linearIssueId,
|
|
1749
|
-
leaseId,
|
|
1750
|
-
workerId: this.workerId,
|
|
1751
|
-
leasedUntil,
|
|
1752
|
-
});
|
|
1753
|
-
if (!acquired)
|
|
1754
|
-
return undefined;
|
|
1755
|
-
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
1756
|
-
return leaseId;
|
|
1757
|
-
}
|
|
1758
549
|
claimLeaseForReconciliation(projectId, linearIssueId) {
|
|
1759
|
-
|
|
1760
|
-
if (this.activeSessionLeases.has(key)) {
|
|
1761
|
-
return "owned";
|
|
1762
|
-
}
|
|
1763
|
-
const session = this.db.getIssueSession(projectId, linearIssueId);
|
|
1764
|
-
if (!session)
|
|
1765
|
-
return "skip";
|
|
1766
|
-
const leasedUntilMs = session.leasedUntil ? Date.parse(session.leasedUntil) : undefined;
|
|
1767
|
-
if (leasedUntilMs !== undefined && Number.isFinite(leasedUntilMs) && leasedUntilMs > Date.now()) {
|
|
1768
|
-
return "skip";
|
|
1769
|
-
}
|
|
1770
|
-
return this.acquireIssueSessionLease(projectId, linearIssueId) ? true : "skip";
|
|
550
|
+
return this.leaseService.claimForReconciliation(projectId, linearIssueId);
|
|
1771
551
|
}
|
|
1772
552
|
async reclaimForeignRecoveryLeaseIfSafe(run, issue) {
|
|
1773
|
-
|
|
1774
|
-
if (this.activeSessionLeases.has(key)) {
|
|
1775
|
-
return false;
|
|
1776
|
-
}
|
|
1777
|
-
const session = this.db.getIssueSession(run.projectId, run.linearIssueId);
|
|
1778
|
-
if (!session?.leaseId || !session.workerId || session.workerId === this.workerId) {
|
|
1779
|
-
return false;
|
|
1780
|
-
}
|
|
1781
|
-
if (issue.activeRunId !== run.id) {
|
|
1782
|
-
return false;
|
|
1783
|
-
}
|
|
1784
|
-
let safeToReclaim = !run.threadId;
|
|
1785
|
-
if (!safeToReclaim && run.threadId) {
|
|
1786
|
-
try {
|
|
1787
|
-
const thread = await this.readThreadWithRetry(run.threadId, 1);
|
|
1788
|
-
const latestTurn = getThreadTurns(thread).at(-1);
|
|
1789
|
-
safeToReclaim = thread.status === "notLoaded"
|
|
1790
|
-
|| latestTurn?.status === "interrupted"
|
|
1791
|
-
|| latestTurn?.status === "completed";
|
|
1792
|
-
}
|
|
1793
|
-
catch {
|
|
1794
|
-
safeToReclaim = true;
|
|
1795
|
-
}
|
|
1796
|
-
}
|
|
1797
|
-
if (!safeToReclaim) {
|
|
1798
|
-
return false;
|
|
1799
|
-
}
|
|
1800
|
-
const leaseId = this.forceAcquireIssueSessionLease(run.projectId, run.linearIssueId);
|
|
1801
|
-
if (!leaseId) {
|
|
1802
|
-
return false;
|
|
1803
|
-
}
|
|
1804
|
-
this.logger.info({
|
|
1805
|
-
issueKey: issue.issueKey,
|
|
1806
|
-
runId: run.id,
|
|
1807
|
-
previousWorkerId: session.workerId,
|
|
1808
|
-
previousLeaseId: session.leaseId,
|
|
1809
|
-
reclaimedLeaseId: leaseId,
|
|
1810
|
-
}, "Reclaimed foreign issue-session lease for active-run recovery");
|
|
1811
|
-
return true;
|
|
553
|
+
return await this.leaseService.reclaimForeignRecoveryLeaseIfSafe(run, issue);
|
|
1812
554
|
}
|
|
1813
555
|
heartbeatIssueSessionLease(projectId, linearIssueId) {
|
|
1814
|
-
|
|
1815
|
-
const leaseId = this.activeSessionLeases.get(key) ?? this.db.getIssueSession(projectId, linearIssueId)?.leaseId;
|
|
1816
|
-
if (!leaseId)
|
|
1817
|
-
return false;
|
|
1818
|
-
const renewed = this.db.renewIssueSessionLease({
|
|
1819
|
-
projectId,
|
|
1820
|
-
linearIssueId,
|
|
1821
|
-
leaseId,
|
|
1822
|
-
leasedUntil: new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString(),
|
|
1823
|
-
});
|
|
1824
|
-
if (renewed) {
|
|
1825
|
-
this.activeSessionLeases.set(key, leaseId);
|
|
1826
|
-
return true;
|
|
1827
|
-
}
|
|
1828
|
-
this.activeSessionLeases.delete(key);
|
|
1829
|
-
return false;
|
|
556
|
+
return this.leaseService.heartbeat(projectId, linearIssueId);
|
|
1830
557
|
}
|
|
1831
558
|
releaseIssueSessionLease(projectId, linearIssueId) {
|
|
1832
|
-
|
|
1833
|
-
const leaseId = this.activeSessionLeases.get(key);
|
|
1834
|
-
this.db.releaseIssueSessionLease(projectId, linearIssueId, leaseId);
|
|
1835
|
-
this.activeSessionLeases.delete(key);
|
|
559
|
+
this.leaseService.release(projectId, linearIssueId);
|
|
1836
560
|
}
|
|
1837
561
|
}
|
|
1838
|
-
/**
|
|
1839
|
-
* Determine post-run factory state from current PR metadata.
|
|
1840
|
-
* Used by both the normal completion path and reconciliation.
|
|
1841
|
-
*/
|
|
1842
|
-
function resolvePostRunState(issue) {
|
|
1843
|
-
if (ACTIVE_RUN_STATES.has(issue.factoryState) && issue.prNumber) {
|
|
1844
|
-
// Check merged first — a merged PR is both approved and merged,
|
|
1845
|
-
// and "done" must take priority over "awaiting_queue".
|
|
1846
|
-
if (issue.prState === "merged")
|
|
1847
|
-
return "done";
|
|
1848
|
-
if (issue.prReviewState === "approved")
|
|
1849
|
-
return "awaiting_queue";
|
|
1850
|
-
return "pr_open";
|
|
1851
|
-
}
|
|
1852
|
-
return undefined;
|
|
1853
|
-
}
|
|
1854
|
-
function resolveCompletedRunState(issue, run) {
|
|
1855
|
-
if (run.runType === "implementation" && resolveImplementationDeliveryMode(issue, undefined, run.promptText) === "linear_only") {
|
|
1856
|
-
return "done";
|
|
1857
|
-
}
|
|
1858
|
-
return resolvePostRunState(issue);
|
|
1859
|
-
}
|
|
1860
|
-
function resolveRecoverablePostRunState(issue) {
|
|
1861
|
-
if (!issue.prNumber) {
|
|
1862
|
-
return resolvePostRunState(issue);
|
|
1863
|
-
}
|
|
1864
|
-
if (issue.prState === "merged")
|
|
1865
|
-
return "done";
|
|
1866
|
-
if (issue.prState === "open") {
|
|
1867
|
-
const reactiveIntent = deriveIssueSessionReactiveIntent({
|
|
1868
|
-
prNumber: issue.prNumber,
|
|
1869
|
-
prState: issue.prState,
|
|
1870
|
-
prReviewState: issue.prReviewState,
|
|
1871
|
-
prCheckStatus: issue.prCheckStatus,
|
|
1872
|
-
latestFailureSource: issue.lastGitHubFailureSource,
|
|
1873
|
-
});
|
|
1874
|
-
if (reactiveIntent)
|
|
1875
|
-
return reactiveIntent.compatibilityFactoryState;
|
|
1876
|
-
if (issue.prReviewState === "approved")
|
|
1877
|
-
return "awaiting_queue";
|
|
1878
|
-
return "pr_open";
|
|
1879
|
-
}
|
|
1880
|
-
return resolvePostRunState(issue);
|
|
1881
|
-
}
|
|
1882
|
-
function normalizeRemotePrState(value) {
|
|
1883
|
-
const normalized = value?.trim().toUpperCase();
|
|
1884
|
-
if (normalized === "OPEN")
|
|
1885
|
-
return "open";
|
|
1886
|
-
if (normalized === "CLOSED")
|
|
1887
|
-
return "closed";
|
|
1888
|
-
if (normalized === "MERGED")
|
|
1889
|
-
return "merged";
|
|
1890
|
-
return undefined;
|
|
1891
|
-
}
|
|
1892
|
-
function normalizeRemoteReviewDecision(value) {
|
|
1893
|
-
const normalized = value?.trim().toUpperCase();
|
|
1894
|
-
if (normalized === "APPROVED")
|
|
1895
|
-
return "approved";
|
|
1896
|
-
if (normalized === "CHANGES_REQUESTED")
|
|
1897
|
-
return "changes_requested";
|
|
1898
|
-
if (normalized === "REVIEW_REQUIRED")
|
|
1899
|
-
return "commented";
|
|
1900
|
-
return undefined;
|
|
1901
|
-
}
|
|
1902
|
-
function isDirtyMergeStateStatus(value) {
|
|
1903
|
-
return value?.trim().toUpperCase() === "DIRTY";
|
|
1904
|
-
}
|
|
1905
|
-
function buildReviewFixBranchUpkeepContext(prNumber, baseBranch, pr, context) {
|
|
1906
|
-
const promptContext = [
|
|
1907
|
-
`The requested code change may already be present, but GitHub still reports PR #${prNumber} as ${String(pr.mergeStateStatus)} against latest ${baseBranch}.`,
|
|
1908
|
-
`This turn is branch upkeep on the existing PR branch: update onto latest ${baseBranch}, resolve any conflicts, rerun the narrowest relevant verification, and push a newer head.`,
|
|
1909
|
-
"Do not stop just because the requested code change is already present. Review can only move forward after a new pushed head.",
|
|
1910
|
-
].join(" ");
|
|
1911
|
-
return {
|
|
1912
|
-
...(context ?? {}),
|
|
1913
|
-
branchUpkeepRequired: true,
|
|
1914
|
-
reviewFixMode: "branch_upkeep",
|
|
1915
|
-
wakeReason: "branch_upkeep",
|
|
1916
|
-
promptContext,
|
|
1917
|
-
...(pr.mergeStateStatus ? { mergeStateStatus: pr.mergeStateStatus } : {}),
|
|
1918
|
-
...(pr.headRefOid ? { failingHeadSha: pr.headRefOid } : {}),
|
|
1919
|
-
baseBranch,
|
|
1920
|
-
};
|
|
1921
|
-
}
|