patchrelay 0.36.7 → 0.36.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,63 +1,25 @@
1
- import { randomUUID } from "node:crypto";
2
- import { ACTIVE_RUN_STATES, TERMINAL_STATES } from "./factory-state.js";
3
- import { buildHookEnv, runProjectHook } from "./hook-runner.js";
4
- import { buildStageReport, countEventMethods, extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread, } from "./run-reporting.js";
5
- import { buildRunCompletedActivity, buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
1
+ import { TERMINAL_STATES } from "./factory-state.js";
2
+ import { extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread } from "./run-reporting.js";
3
+ import { buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
6
4
  import { WorktreeManager } from "./worktree-manager.js";
7
5
  import { resolveAuthoritativeLinearStopState, resolvePreferredCompletedLinearState } from "./linear-workflow.js";
8
- import { execCommand } from "./utils.js";
9
6
  import { getThreadTurns } from "./codex-thread-utils.js";
10
- import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
11
- import { loadPatchRelayRepoPrompting } from "./patchrelay-customization.js";
12
- import { buildRunPrompt as buildPatchRelayRunPrompt, findDisallowedPatchRelayPromptSectionIds, findUnknownPatchRelayPromptSectionIds, mergePromptCustomizationLayers, resolveImplementationDeliveryMode, resolvePromptLayers, } from "./prompting/patchrelay.js";
13
- const DEFAULT_CI_REPAIR_BUDGET = 3;
14
- const DEFAULT_QUEUE_REPAIR_BUDGET = 3;
15
- // Requested-changes loops can legitimately take more iterations than CI/queue
16
- // repair when the reviewer is catching nuanced product or timing bugs across
17
- // successive heads. Keep a hard ceiling to prevent infinite ping-pong, but make
18
- // it wide enough that real review cycles can continue after multiple successful
19
- // head advances.
20
- const DEFAULT_REVIEW_FIX_BUDGET = 12;
21
- const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
22
- const ZOMBIE_RECOVERY_BASE_DELAY_MS = 15_000; // 15s, 30s, 60s, 120s, 240s
23
- const ISSUE_SESSION_LEASE_MS = 10 * 60_000;
24
- const MAX_THREAD_GENERATION_BEFORE_COMPACTION = 4;
25
- const MAX_FOLLOW_UPS_BEFORE_COMPACTION = 4;
26
7
  import { QueueHealthMonitor } from "./queue-health-monitor.js";
27
8
  import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
28
9
  import { LinearSessionSync } from "./linear-session-sync.js";
29
- function slugify(value) {
30
- return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
31
- }
32
- function sanitizePathSegment(value) {
33
- return value.replace(/[^a-zA-Z0-9._-]+/g, "-");
34
- }
10
+ import { IssueSessionLeaseService } from "./issue-session-lease-service.js";
11
+ import { InterruptedRunRecovery, resolveRecoverablePostRunState } from "./interrupted-run-recovery.js";
12
+ import { RunCompletionPolicy } from "./run-completion-policy.js";
13
+ import { RunFinalizer } from "./run-finalizer.js";
14
+ import { RunLauncher } from "./run-launcher.js";
15
+ import { RunRecoveryService } from "./run-recovery-service.js";
16
+ import { RunWakePlanner } from "./run-wake-planner.js";
35
17
  function lowerCaseFirst(value) {
36
18
  return value ? `${value.slice(0, 1).toLowerCase()}${value.slice(1)}` : value;
37
19
  }
38
20
  function isRequestedChangesRunType(runType) {
39
21
  return runType === "review_fix" || runType === "branch_upkeep";
40
22
  }
41
- function resolveRequestedChangesMode(runType, context) {
42
- if (runType === "branch_upkeep") {
43
- return "branch_upkeep";
44
- }
45
- return context?.reviewFixMode === "branch_upkeep" || context?.branchUpkeepRequired === true
46
- ? "branch_upkeep"
47
- : "address_review_feedback";
48
- }
49
- function shouldCompactThread(issue, threadGeneration, context) {
50
- const followUpCount = typeof context?.followUpCount === "number" ? context.followUpCount : 0;
51
- return issue.threadId !== undefined
52
- && (threadGeneration ?? 0) >= MAX_THREAD_GENERATION_BEFORE_COMPACTION
53
- && followUpCount >= MAX_FOLLOW_UPS_BEFORE_COMPACTION;
54
- }
55
- export function shouldReuseIssueThread(params) {
56
- return Boolean(params.existingThreadId) && !params.compactThread && params.resumeThread;
57
- }
58
- function isBranchUpkeepRequired(context) {
59
- return context?.branchUpkeepRequired === true;
60
- }
61
23
  export class RunOrchestrator {
62
24
  config;
63
25
  db;
@@ -73,7 +35,14 @@ export class RunOrchestrator {
73
35
  linearSync;
74
36
  activeThreadId;
75
37
  workerId = `patchrelay:${process.pid}`;
76
- activeSessionLeases = new Map();
38
+ leaseService;
39
+ runFinalizer;
40
+ runLauncher;
41
+ runRecovery;
42
+ runWakePlanner;
43
+ interruptedRunRecovery;
44
+ runCompletionPolicy;
45
+ activeSessionLeases;
77
46
  botIdentity;
78
47
  constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
79
48
  this.config = config;
@@ -85,6 +54,14 @@ export class RunOrchestrator {
85
54
  this.feed = feed;
86
55
  this.worktreeManager = new WorktreeManager(config);
87
56
  this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
57
+ this.leaseService = new IssueSessionLeaseService(db, logger, this.workerId, (threadId, maxRetries) => this.readThreadWithRetry(threadId, maxRetries));
58
+ this.activeSessionLeases = this.leaseService.activeSessionLeases;
59
+ this.runCompletionPolicy = new RunCompletionPolicy(config, db, logger, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn));
60
+ this.runFinalizer = new RunFinalizer(db, logger, this.linearSync, this.enqueueIssue, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (lease, issue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope), (run, message, nextState) => this.failRunAndClear(run, message, nextState), this.runCompletionPolicy, feed);
61
+ this.runLauncher = new RunLauncher(config, db, codex, logger, this.worktreeManager);
62
+ this.runRecovery = new RunRecoveryService(db, logger, this.linearSync, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.getHeldIssueSessionLease(projectId, linearIssueId), (lease, issue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (projectId, issueId) => this.enqueueIssue(projectId, issueId), (newState, pendingRunType) => this.resolveBranchOwnerForStateTransition(newState, pendingRunType), feed);
63
+ this.interruptedRunRecovery = new InterruptedRunRecovery(db, logger, this.linearSync, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (run, message, nextState) => this.failRunAndClear(run, message, nextState), (issue) => this.restoreIdleWorktree(issue), this.runCompletionPolicy, (projectId, issueId) => this.enqueueIssue(projectId, issueId), feed);
64
+ this.runWakePlanner = new RunWakePlanner(db);
88
65
  this.idleReconciler = new IdleIssueReconciler(db, config, {
89
66
  enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
90
67
  }, logger, feed);
@@ -94,205 +71,78 @@ export class RunOrchestrator {
94
71
  }, logger, feed);
95
72
  }
96
73
  resolveRunWake(issue) {
97
- const sessionWake = this.db.peekIssueSessionWake(issue.projectId, issue.linearIssueId);
98
- if (sessionWake) {
99
- return {
100
- runType: sessionWake.runType,
101
- context: sessionWake.context,
102
- wakeReason: sessionWake.wakeReason,
103
- resumeThread: sessionWake.resumeThread,
104
- eventIds: sessionWake.eventIds,
105
- };
106
- }
107
- return undefined;
74
+ return this.runWakePlanner.resolveRunWake(issue);
108
75
  }
109
76
  appendWakeEventWithLease(lease, issue, runType, context, dedupeScope) {
110
- let eventType;
111
- let dedupeKey;
112
- if (runType === "queue_repair") {
113
- eventType = "merge_steward_incident";
114
- dedupeKey = `${dedupeScope ?? "wake"}:queue_repair:${issue.linearIssueId}:${issue.prHeadSha ?? issue.lastGitHubFailureHeadSha ?? "unknown-sha"}`;
115
- }
116
- else if (runType === "ci_repair") {
117
- eventType = "settled_red_ci";
118
- dedupeKey = `${dedupeScope ?? "wake"}:ci_repair:${issue.linearIssueId}:${issue.lastGitHubFailureSignature ?? issue.prHeadSha ?? "unknown-sha"}`;
119
- }
120
- else if (runType === "review_fix" || runType === "branch_upkeep") {
121
- eventType = "review_changes_requested";
122
- dedupeKey = `${dedupeScope ?? "wake"}:${runType}:${issue.linearIssueId}:${issue.prHeadSha ?? "unknown-sha"}`;
123
- }
124
- else {
125
- eventType = "delegated";
126
- dedupeKey = `${dedupeScope ?? "wake"}:implementation:${issue.linearIssueId}`;
127
- }
128
- return Boolean(this.db.appendIssueSessionEventWithLease(lease, {
129
- projectId: issue.projectId,
130
- linearIssueId: issue.linearIssueId,
131
- eventType,
132
- ...(context ? { eventJson: JSON.stringify(context) } : {}),
133
- dedupeKey,
134
- }));
77
+ return this.runWakePlanner.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope);
135
78
  }
136
79
  materializeLegacyPendingWake(issue, lease) {
137
- if (!issue.pendingRunType)
138
- return issue;
139
- const context = issue.pendingRunContextJson
140
- ? JSON.parse(issue.pendingRunContextJson)
141
- : undefined;
142
- this.appendWakeEventWithLease(lease, issue, issue.pendingRunType, context, "legacy_pending");
143
- const updated = this.db.upsertIssueWithLease(lease, {
144
- projectId: issue.projectId,
145
- linearIssueId: issue.linearIssueId,
146
- pendingRunType: null,
147
- pendingRunContextJson: null,
148
- });
149
- if (!updated)
150
- return issue;
151
- return this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
80
+ return this.runWakePlanner.materializeLegacyPendingWake(issue, lease);
152
81
  }
153
82
  // ─── Run ────────────────────────────────────────────────────────
154
83
  async run(item) {
155
84
  const project = this.config.projects.find((p) => p.id === item.projectId);
156
85
  if (!project)
157
86
  return;
158
- if (this.activeSessionLeases.has(this.issueSessionLeaseKey(item.projectId, item.issueId))) {
87
+ if (this.leaseService.hasLocalLease(item.projectId, item.issueId)) {
159
88
  return;
160
89
  }
161
- const issue = this.db.getIssue(item.projectId, item.issueId);
90
+ const issue = this.db.issues.getIssue(item.projectId, item.issueId);
162
91
  if (!issue || issue.activeRunId !== undefined)
163
92
  return;
164
- const issueSession = this.db.getIssueSession(item.projectId, item.issueId);
165
- const leaseId = this.acquireIssueSessionLease(item.projectId, item.issueId);
93
+ const issueSession = this.db.issueSessions.getIssueSession(item.projectId, item.issueId);
94
+ const leaseId = this.leaseService.acquire(item.projectId, item.issueId);
166
95
  if (!leaseId) {
167
96
  this.logger.info({ issueKey: issue.issueKey, projectId: item.projectId }, "Skipped run because another worker holds the session lease");
168
97
  return;
169
98
  }
170
99
  if (issue.prState === "merged") {
171
- this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
172
- this.releaseIssueSessionLease(item.projectId, item.issueId);
100
+ this.db.issueSessions.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
101
+ this.leaseService.release(item.projectId, item.issueId);
173
102
  return;
174
103
  }
175
104
  const wakeIssue = this.materializeLegacyPendingWake(issue, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
176
105
  const wake = this.resolveRunWake(wakeIssue);
177
106
  if (!wake) {
178
- this.releaseIssueSessionLease(item.projectId, item.issueId);
107
+ this.leaseService.release(item.projectId, item.issueId);
179
108
  return;
180
109
  }
181
110
  const { runType, context, resumeThread } = wake;
182
111
  const effectiveContext = isRequestedChangesRunType(runType)
183
- ? await this.resolveRequestedChangesWakeContext(issue, runType, context, project)
112
+ ? await this.runCompletionPolicy.resolveRequestedChangesWakeContext(issue, runType, context)
184
113
  : context;
185
114
  const sourceHeadSha = typeof effectiveContext?.failureHeadSha === "string"
186
115
  ? effectiveContext.failureHeadSha
187
116
  : typeof effectiveContext?.headSha === "string"
188
117
  ? effectiveContext.headSha
189
118
  : issue.prHeadSha;
190
- // Check repair budgets
191
- if (runType === "ci_repair" && issue.ciRepairAttempts >= DEFAULT_CI_REPAIR_BUDGET) {
192
- this.escalate(issue, runType, `CI repair budget exhausted (${DEFAULT_CI_REPAIR_BUDGET} attempts)`);
193
- return;
194
- }
195
- if (runType === "queue_repair" && issue.queueRepairAttempts >= DEFAULT_QUEUE_REPAIR_BUDGET) {
196
- this.escalate(issue, runType, `Queue repair budget exhausted (${DEFAULT_QUEUE_REPAIR_BUDGET} attempts)`);
119
+ const budgetExceeded = this.runWakePlanner.budgetExceeded(issue, runType, isRequestedChangesRunType);
120
+ if (budgetExceeded) {
121
+ this.escalate(issue, runType, budgetExceeded);
197
122
  return;
198
123
  }
199
- if (isRequestedChangesRunType(runType) && issue.reviewFixAttempts >= DEFAULT_REVIEW_FIX_BUDGET) {
200
- this.escalate(issue, runType, `Requested-changes budget exhausted (${DEFAULT_REVIEW_FIX_BUDGET} attempts)`);
124
+ if (!this.runWakePlanner.incrementAttemptCounters(issue, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, runType, isRequestedChangesRunType)) {
125
+ this.releaseIssueSessionLease(item.projectId, item.issueId);
201
126
  return;
202
127
  }
203
- // Increment repair counters
204
- if (runType === "ci_repair") {
205
- const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, ciRepairAttempts: issue.ciRepairAttempts + 1 });
206
- if (!updated) {
207
- this.releaseIssueSessionLease(item.projectId, item.issueId);
208
- return;
209
- }
210
- }
211
- if (runType === "queue_repair") {
212
- const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueRepairAttempts: issue.queueRepairAttempts + 1 });
213
- if (!updated) {
214
- this.releaseIssueSessionLease(item.projectId, item.issueId);
215
- return;
216
- }
217
- }
218
- if (isRequestedChangesRunType(runType)) {
219
- const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, reviewFixAttempts: issue.reviewFixAttempts + 1 });
220
- if (!updated) {
221
- this.releaseIssueSessionLease(item.projectId, item.issueId);
222
- return;
223
- }
224
- }
225
- const repoPrompting = loadPatchRelayRepoPrompting({
226
- repoRoot: project.repoPath,
227
- logger: this.logger,
228
- });
229
- const promptLayer = mergePromptCustomizationLayers(resolvePromptLayers(this.config.prompting, runType), resolvePromptLayers(repoPrompting, runType));
230
- const unknownPromptSections = findUnknownPatchRelayPromptSectionIds(promptLayer);
231
- if (unknownPromptSections.length > 0) {
232
- this.logger.warn({ issueKey: issue.issueKey, runType, unknownPromptSections }, "PatchRelay prompt customization references unknown section ids");
233
- }
234
- const disallowedPromptSections = findDisallowedPatchRelayPromptSectionIds(promptLayer);
235
- if (disallowedPromptSections.length > 0) {
236
- this.logger.warn({ issueKey: issue.issueKey, runType, disallowedPromptSections }, "PatchRelay prompt customization attempted to replace non-overridable sections");
237
- }
238
- const prompt = buildPatchRelayRunPrompt({
128
+ const { prompt, branchName, worktreePath } = this.runLauncher.prepareLaunchPlan({
129
+ project,
239
130
  issue,
240
131
  runType,
241
- repoPath: project.repoPath,
242
- ...(effectiveContext ? { context: effectiveContext } : {}),
243
- ...(promptLayer ? { promptLayer } : {}),
132
+ ...(effectiveContext ? { effectiveContext } : {}),
244
133
  });
245
- // Resolve workspace
246
- const issueRef = sanitizePathSegment(issue.issueKey ?? issue.linearIssueId);
247
- const slug = issue.title ? slugify(issue.title) : "";
248
- const branchSuffix = slug ? `${issueRef}-${slug}` : issueRef;
249
- const branchName = issue.branchName ?? `${project.branchPrefix}/${branchSuffix}`;
250
- const worktreePath = issue.worktreePath ?? `${project.worktreeRoot}/${issueRef}`;
251
- // Claim the run atomically
252
- const run = this.db.withIssueSessionLease(item.projectId, item.issueId, leaseId, () => {
253
- const fresh = this.db.getIssue(item.projectId, item.issueId);
254
- if (!fresh || fresh.activeRunId !== undefined)
255
- return undefined;
256
- const wakeIssue = this.materializeLegacyPendingWake(fresh, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
257
- const freshWake = this.resolveRunWake(wakeIssue);
258
- if (!freshWake || freshWake.runType !== runType)
259
- return undefined;
260
- const created = this.db.createRun({
261
- issueId: fresh.id,
262
- projectId: item.projectId,
263
- linearIssueId: item.issueId,
264
- runType,
265
- ...(sourceHeadSha ? { sourceHeadSha } : {}),
266
- promptText: prompt,
267
- });
268
- const failureHeadSha = typeof effectiveContext?.failureHeadSha === "string"
269
- ? effectiveContext.failureHeadSha
270
- : typeof effectiveContext?.headSha === "string" ? effectiveContext.headSha : undefined;
271
- const failureSignature = typeof effectiveContext?.failureSignature === "string" ? effectiveContext.failureSignature : undefined;
272
- this.db.upsertIssue({
273
- projectId: item.projectId,
274
- linearIssueId: item.issueId,
275
- pendingRunType: null,
276
- pendingRunContextJson: null,
277
- activeRunId: created.id,
278
- branchName,
279
- worktreePath,
280
- factoryState: runType === "implementation" ? "implementing"
281
- : runType === "ci_repair" ? "repairing_ci"
282
- : runType === "review_fix" || runType === "branch_upkeep" ? "changes_requested"
283
- : runType === "queue_repair" ? "repairing_queue"
284
- : "implementing",
285
- ...((runType === "ci_repair" || runType === "queue_repair") && failureSignature
286
- ? {
287
- lastAttemptedFailureSignature: failureSignature,
288
- lastAttemptedFailureHeadSha: failureHeadSha ?? null,
289
- }
290
- : {}),
291
- });
292
- this.db.consumeIssueSessionEvents(item.projectId, item.issueId, freshWake.eventIds, created.id);
293
- this.db.setIssueSessionLastWakeReason(item.projectId, item.issueId, freshWake.wakeReason ?? null);
294
- this.db.setBranchOwnerWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, "patchrelay");
295
- return created;
134
+ const run = this.runLauncher.claimRun({
135
+ item,
136
+ issue,
137
+ leaseId,
138
+ runType,
139
+ prompt,
140
+ ...(sourceHeadSha ? { sourceHeadSha } : {}),
141
+ ...(effectiveContext ? { effectiveContext } : {}),
142
+ materializeLegacyPendingWake: (targetIssue, lease) => this.materializeLegacyPendingWake(targetIssue, lease),
143
+ resolveRunWake: (targetIssue) => this.resolveRunWake(targetIssue),
144
+ branchName,
145
+ worktreePath,
296
146
  });
297
147
  if (!run) {
298
148
  this.releaseIssueSessionLease(item.projectId, item.issueId);
@@ -307,110 +157,34 @@ export class RunOrchestrator {
307
157
  status: "starting",
308
158
  summary: `Starting ${runType} run`,
309
159
  });
310
- let threadId;
311
- let turnId;
312
- let parentThreadId;
313
- try {
314
- // Ensure worktree
315
- await this.worktreeManager.ensureIssueWorktree(project.repoPath, project.worktreeRoot, worktreePath, branchName, { allowExistingOutsideRoot: issue.branchName !== undefined });
316
- // Set bot git identity and push credentials when GitHub App is configured.
317
- // This ensures commits are authored by and pushes are authenticated as
318
- // patchrelay[bot], not the system user.
319
- if (this.botIdentity) {
320
- const gitBin = this.config.runner.gitBin;
321
- await execCommand(gitBin, ["-C", worktreePath, "config", "user.name", this.botIdentity.name], { timeoutMs: 5_000 });
322
- await execCommand(gitBin, ["-C", worktreePath, "config", "user.email", this.botIdentity.email], { timeoutMs: 5_000 });
323
- // Override credential helper to use the App installation token for git push.
324
- // The helper script reads the token file and returns it as the password.
325
- const credentialHelper = `!f() { echo "username=x-access-token"; echo "password=$(cat ${this.botIdentity.tokenFile})"; }; f`;
326
- await execCommand(gitBin, ["-C", worktreePath, "config", "credential.helper", credentialHelper], { timeoutMs: 5_000 });
327
- }
328
- await this.resetWorktreeToTrackedBranch(worktreePath, branchName, issue);
329
- // Freshen the worktree: fetch + rebase onto latest base branch.
330
- // This prevents branch contamination when local main has drifted
331
- // and avoids scope-bundling review rejections from stale commits.
332
- // Skip for queue_repair — its entire purpose is to resolve rebase conflicts.
333
- if (runType !== "queue_repair") {
334
- await this.freshenWorktree(worktreePath, project, issue);
335
- }
336
- // Run prepare-worktree hook
337
- const hookEnv = buildHookEnv(issue.issueKey ?? issue.linearIssueId, branchName, runType, worktreePath);
338
- const prepareResult = await runProjectHook(project.repoPath, "prepare-worktree", { cwd: worktreePath, env: hookEnv });
339
- if (prepareResult.ran && prepareResult.exitCode !== 0) {
340
- throw new Error(`prepare-worktree hook failed (exit ${prepareResult.exitCode}): ${prepareResult.stderr?.slice(0, 500) ?? ""}`);
341
- }
342
- this.assertLaunchLease(run, "before starting the Codex turn");
343
- // Reuse the existing thread only for additive follow-ups that explicitly
344
- // request continuity. Fresh review-fix runs now start a new thread so the
345
- // model is not anchored to the implementation conversation that produced
346
- // the rejected patch. If the thread has accumulated many resumptions and
347
- // batched follow-ups, compact by starting a fresh main thread while
348
- // keeping a parent link.
349
- const compactThread = shouldCompactThread(issue, issueSession?.threadGeneration, effectiveContext);
350
- if (compactThread && issue.threadId) {
351
- parentThreadId = issue.threadId;
352
- }
353
- if (shouldReuseIssueThread({ existingThreadId: issue.threadId, compactThread, resumeThread })) {
354
- threadId = issue.threadId;
355
- }
356
- else {
357
- const thread = await this.codex.startThread({ cwd: worktreePath });
358
- threadId = thread.id;
359
- this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
360
- }
361
- try {
362
- const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
363
- turnId = turn.turnId;
364
- }
365
- catch (turnError) {
366
- // If the thread is stale (e.g. after app-server restart), start fresh and retry once.
367
- const msg = turnError instanceof Error ? turnError.message : String(turnError);
368
- if (msg.includes("thread not found") || msg.includes("not materialized")) {
369
- this.logger.info({ issueKey: issue.issueKey, staleThreadId: threadId }, "Thread is stale, retrying with fresh thread");
370
- const thread = await this.codex.startThread({ cwd: worktreePath });
371
- threadId = thread.id;
372
- this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
373
- const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
374
- turnId = turn.turnId;
375
- }
376
- else {
377
- throw turnError;
378
- }
379
- }
380
- this.assertLaunchLease(run, "after starting the Codex turn");
381
- }
382
- catch (error) {
383
- const message = error instanceof Error ? error.message : String(error);
384
- const lostLease = error instanceof Error && error.name === "IssueSessionLeaseLostError";
385
- if (!lostLease) {
386
- const nextState = isRequestedChangesRunType(runType) ? "escalated" : "failed";
387
- this.db.finishRunWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, run.id, {
388
- status: "failed",
389
- failureReason: message,
390
- });
391
- this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
392
- projectId: item.projectId,
393
- linearIssueId: item.issueId,
394
- activeRunId: null,
395
- factoryState: nextState,
396
- });
397
- }
398
- this.logger.error({ issueKey: issue.issueKey, runType, error: message }, `Failed to launch ${runType} run`);
399
- const failedIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
400
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(runType, `Failed to start ${lowerCaseFirst(message)}`));
401
- void this.linearSync.syncSession(failedIssue, { activeRunType: runType });
402
- this.releaseIssueSessionLease(item.projectId, item.issueId);
403
- throw error;
404
- }
160
+ const { threadId, turnId, parentThreadId, } = await this.runLauncher.launchTurn({
161
+ project,
162
+ issue,
163
+ ...(issueSession ? { issueSession } : {}),
164
+ run,
165
+ runType,
166
+ prompt,
167
+ branchName,
168
+ worktreePath,
169
+ resumeThread,
170
+ ...(effectiveContext ? { effectiveContext } : {}),
171
+ leaseId,
172
+ ...(this.botIdentity ? { botIdentity: this.botIdentity } : {}),
173
+ assertLaunchLease: (targetRun, phase) => this.assertLaunchLease(targetRun, phase),
174
+ linearSync: this.linearSync,
175
+ releaseLease: (projectId, issueId) => this.releaseIssueSessionLease(projectId, issueId),
176
+ isRequestedChangesRunType,
177
+ lowerCaseFirst,
178
+ });
405
179
  this.assertLaunchLease(run, "before recording the active thread");
406
- if (!this.db.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
180
+ if (!this.db.issueSessions.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
407
181
  this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping run thread update after losing issue-session lease");
408
182
  this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
409
183
  return;
410
184
  }
411
185
  // Reset zombie recovery counter — this run started successfully
412
186
  if (issue.zombieRecoveryAttempts > 0) {
413
- this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
187
+ this.db.issueSessions.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
414
188
  projectId: item.projectId,
415
189
  linearIssueId: item.issueId,
416
190
  zombieRecoveryAttempts: 0,
@@ -419,98 +193,15 @@ export class RunOrchestrator {
419
193
  }
420
194
  this.logger.info({ issueKey: issue.issueKey, runType, threadId, turnId }, `Started ${runType} run`);
421
195
  // Emit Linear activity + plan
422
- const freshIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
196
+ const freshIssue = this.db.issues.getIssue(item.projectId, item.issueId) ?? issue;
423
197
  void this.linearSync.emitActivity(freshIssue, buildRunStartedActivity(runType));
424
198
  void this.linearSync.syncSession(freshIssue, { activeRunType: runType });
425
199
  }
426
- // ─── Pre-run branch freshening ────────────────────────────────────
427
- /**
428
- * Fetch origin and rebase the worktree onto the latest base branch.
429
- *
430
- * Risks mitigated:
431
- * - Dirty worktree from interrupted run → stash before, pop after
432
- * - Conflicts → abort rebase, throw so the run fails with a clear reason
433
- * - Already up-to-date → no-op
434
- * - Keep publishing explicit: the orchestrator updates the local worktree
435
- * only; the agent/run owns any later branch push.
436
- */
437
- async freshenWorktree(worktreePath, project, issue) {
438
- const gitBin = this.config.runner.gitBin;
439
- const baseBranch = project.github?.baseBranch ?? "main";
440
- // Stash any uncommitted changes from a previous interrupted run
441
- const stashResult = await execCommand(gitBin, ["-C", worktreePath, "stash"], { timeoutMs: 30_000 });
442
- const didStash = stashResult.exitCode === 0 && !stashResult.stdout?.includes("No local changes");
443
- // Fetch latest base
444
- const fetchResult = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", baseBranch], { timeoutMs: 60_000 });
445
- if (fetchResult.exitCode !== 0) {
446
- this.logger.warn({ issueKey: issue.issueKey, stderr: fetchResult.stderr?.slice(0, 300) }, "Pre-run fetch failed, proceeding with current base");
447
- if (didStash)
448
- await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
449
- return;
450
- }
451
- // Check if rebase is needed: is HEAD already on top of origin/baseBranch?
452
- const mergeBaseResult = await execCommand(gitBin, ["-C", worktreePath, "merge-base", "--is-ancestor", `origin/${baseBranch}`, "HEAD"], { timeoutMs: 10_000 });
453
- if (mergeBaseResult.exitCode === 0) {
454
- // Already up-to-date — no rebase needed
455
- this.logger.debug({ issueKey: issue.issueKey }, "Pre-run freshen: branch already up to date");
456
- if (didStash)
457
- await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
458
- return;
459
- }
460
- // Rebase onto latest base
461
- const rebaseResult = await execCommand(gitBin, ["-C", worktreePath, "rebase", `origin/${baseBranch}`], { timeoutMs: 120_000 });
462
- if (rebaseResult.exitCode !== 0) {
463
- // Abort the failed rebase and restore state — then let the agent run
464
- // proceed. The agent can resolve the conflict itself (the workflow
465
- // prompt tells it to rebase and handle conflicts).
466
- await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
467
- if (didStash)
468
- await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
469
- this.logger.warn({ issueKey: issue.issueKey, baseBranch }, "Pre-run freshen: rebase conflict, agent will resolve");
470
- return;
471
- }
472
- this.logger.info({ issueKey: issue.issueKey, baseBranch }, "Pre-run freshen: rebased locally onto latest base");
473
- // Restore stashed changes
474
- if (didStash)
475
- await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
476
- }
477
200
  async resetWorktreeToTrackedBranch(worktreePath, branchName, issue) {
478
- const gitBin = this.config.runner.gitBin;
479
- const branchFetch = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", branchName], { timeoutMs: 60_000 });
480
- const hasRemoteBranch = branchFetch.exitCode === 0;
481
- await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
482
- await execCommand(gitBin, ["-C", worktreePath, "merge", "--abort"], { timeoutMs: 10_000 });
483
- await execCommand(gitBin, ["-C", worktreePath, "cherry-pick", "--abort"], { timeoutMs: 10_000 });
484
- await execCommand(gitBin, ["-C", worktreePath, "am", "--abort"], { timeoutMs: 10_000 });
485
- await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", "HEAD"], { timeoutMs: 30_000 });
486
- await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
487
- const checkoutTarget = hasRemoteBranch ? `origin/${branchName}` : branchName;
488
- const checkoutResult = await execCommand(gitBin, ["-C", worktreePath, "checkout", "-B", branchName, checkoutTarget], { timeoutMs: 30_000 });
489
- if (checkoutResult.exitCode !== 0) {
490
- throw new Error(`Failed to restore ${branchName} worktree state: ${checkoutResult.stderr?.slice(0, 300) ?? "git checkout failed"}`);
491
- }
492
- const resetTarget = hasRemoteBranch ? `origin/${branchName}` : "HEAD";
493
- const resetResult = await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", resetTarget], { timeoutMs: 30_000 });
494
- if (resetResult.exitCode !== 0) {
495
- throw new Error(`Failed to reset ${branchName} worktree state: ${resetResult.stderr?.slice(0, 300) ?? "git reset failed"}`);
496
- }
497
- await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
498
- this.logger.debug({ issueKey: issue.issueKey, branchName, hasRemoteBranch }, "Reset issue worktree to tracked branch state");
201
+ await this.worktreeManager.resetWorktreeToTrackedBranch(worktreePath, branchName, issue, this.logger);
499
202
  }
500
203
  async restoreIdleWorktree(issue) {
501
- if (!issue.worktreePath || !issue.branchName)
502
- return;
503
- try {
504
- await this.resetWorktreeToTrackedBranch(issue.worktreePath, issue.branchName, issue);
505
- }
506
- catch (error) {
507
- this.logger.warn({
508
- issueKey: issue.issueKey,
509
- branchName: issue.branchName,
510
- worktreePath: issue.worktreePath,
511
- error: error instanceof Error ? error.message : String(error),
512
- }, "Failed to restore idle worktree after interrupted run");
513
- }
204
+ await this.worktreeManager.restoreIdleWorktree(issue, this.logger);
514
205
  }
515
206
  // ─── Notification handler ─────────────────────────────────────────
516
207
  async handleCodexNotification(notification) {
@@ -526,7 +217,7 @@ export class RunOrchestrator {
526
217
  if (notification.method === "turn/started" && threadId) {
527
218
  this.activeThreadId = threadId;
528
219
  }
529
- const run = this.db.getRunByThreadId(threadId);
220
+ const run = this.db.runs.getRunByThreadId(threadId);
530
221
  if (!run)
531
222
  return;
532
223
  if (!this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
@@ -535,7 +226,7 @@ export class RunOrchestrator {
535
226
  }
536
227
  const turnId = typeof notification.params.turnId === "string" ? notification.params.turnId : undefined;
537
228
  if (this.config.runner.codex.persistExtendedHistory) {
538
- this.db.saveThreadEvent({
229
+ this.db.runs.saveThreadEvent({
539
230
  runId: run.id,
540
231
  threadId,
541
232
  ...(turnId ? { turnId } : {}),
@@ -547,7 +238,7 @@ export class RunOrchestrator {
547
238
  this.linearSync.maybeEmitProgress(notification, run);
548
239
  // Sync codex plan to Linear session when it updates
549
240
  if (notification.method === "turn/plan/updated") {
550
- const issue = this.db.getIssue(run.projectId, run.linearIssueId);
241
+ const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
551
242
  if (issue) {
552
243
  void this.linearSync.syncCodexPlan(issue, notification.params);
553
244
  }
@@ -555,7 +246,7 @@ export class RunOrchestrator {
555
246
  if (notification.method !== "turn/completed")
556
247
  return;
557
248
  const thread = await this.readThreadWithRetry(threadId);
558
- const issue = this.db.getIssue(run.projectId, run.linearIssueId);
249
+ const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
559
250
  if (!issue)
560
251
  return;
561
252
  const completedTurnId = extractTurnId(notification.params);
@@ -563,13 +254,13 @@ export class RunOrchestrator {
563
254
  if (status === "failed") {
564
255
  const nextState = isRequestedChangesRunType(run.runType) ? "escalated" : "failed";
565
256
  const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
566
- this.db.finishRunWithLease(lease, run.id, {
257
+ this.db.issueSessions.finishRunWithLease(lease, run.id, {
567
258
  status: "failed",
568
259
  threadId,
569
260
  ...(completedTurnId ? { turnId: completedTurnId } : {}),
570
261
  failureReason: "Codex reported the turn completed in a failed state",
571
262
  });
572
- this.db.upsertIssueWithLease(lease, {
263
+ this.db.issueSessions.upsertIssueWithLease(lease, {
573
264
  projectId: run.projectId,
574
265
  linearIssueId: run.linearIssueId,
575
266
  activeRunId: null,
@@ -591,7 +282,7 @@ export class RunOrchestrator {
591
282
  status: "failed",
592
283
  summary: `Turn failed for ${run.runType}`,
593
284
  });
594
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
285
+ const failedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
595
286
  void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType));
596
287
  void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
597
288
  this.linearSync.clearProgress(run.id);
@@ -599,158 +290,23 @@ export class RunOrchestrator {
599
290
  this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
600
291
  return;
601
292
  }
602
- // Complete the run
603
- const trackedIssue = this.db.issueToTrackedIssue(issue);
604
- const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
605
- // Determine post-run state based on current PR metadata.
606
- const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
607
- const verifiedRepairError = await this.verifyReactiveRunAdvancedBranch(run, freshIssue);
608
- if (verifiedRepairError) {
609
- const holdState = resolveRecoverablePostRunState(freshIssue) ?? "failed";
610
- this.failRunAndClear(run, verifiedRepairError, holdState);
611
- const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
612
- this.feed?.publish({
613
- level: "warn",
614
- kind: "turn",
615
- issueKey: freshIssue.issueKey,
616
- projectId: run.projectId,
617
- stage: run.runType,
618
- status: "branch_not_advanced",
619
- summary: verifiedRepairError,
620
- });
621
- void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
622
- void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
623
- this.linearSync.clearProgress(run.id);
624
- this.activeThreadId = undefined;
625
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
626
- return;
627
- }
628
- const missingReviewFixHeadError = await this.verifyReviewFixAdvancedHead(run, freshIssue);
629
- if (missingReviewFixHeadError) {
630
- this.failRunAndClear(run, missingReviewFixHeadError, "escalated");
631
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
632
- this.feed?.publish({
633
- level: "error",
634
- kind: "turn",
635
- issueKey: freshIssue.issueKey,
636
- projectId: run.projectId,
637
- stage: run.runType,
638
- status: "same_head_review_handoff_blocked",
639
- summary: missingReviewFixHeadError,
640
- });
641
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
642
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
643
- this.linearSync.clearProgress(run.id);
644
- this.activeThreadId = undefined;
645
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
646
- return;
647
- }
648
- const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
649
- if (publishedOutcomeError) {
650
- this.failRunAndClear(run, publishedOutcomeError, "failed");
651
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
652
- this.feed?.publish({
653
- level: "warn",
654
- kind: "turn",
655
- issueKey: freshIssue.issueKey,
656
- projectId: run.projectId,
657
- stage: run.runType,
658
- status: "publish_incomplete",
659
- summary: publishedOutcomeError,
660
- });
661
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
662
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
663
- this.linearSync.clearProgress(run.id);
664
- this.activeThreadId = undefined;
665
- return;
666
- }
667
- const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
668
- const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
669
- const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
670
- const completed = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
671
- this.db.finishRun(run.id, {
672
- status: "completed",
673
- threadId,
674
- ...(completedTurnId ? { turnId: completedTurnId } : {}),
675
- summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
676
- reportJson: JSON.stringify(report),
677
- });
678
- this.db.upsertIssue({
679
- projectId: run.projectId,
680
- linearIssueId: run.linearIssueId,
681
- activeRunId: null,
682
- ...(postRunState ? { factoryState: postRunState } : {}),
683
- pendingRunType: null,
684
- pendingRunContextJson: null,
685
- ...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
686
- ? {
687
- lastGitHubFailureSource: null,
688
- lastGitHubFailureHeadSha: null,
689
- lastGitHubFailureSignature: null,
690
- lastGitHubFailureCheckName: null,
691
- lastGitHubFailureCheckUrl: null,
692
- lastGitHubFailureContextJson: null,
693
- lastGitHubFailureAt: null,
694
- lastQueueIncidentJson: null,
695
- lastAttemptedFailureHeadSha: null,
696
- lastAttemptedFailureSignature: null,
697
- }
698
- : {})),
699
- });
700
- if (postRunFollowUp) {
701
- return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
702
- }
703
- return true;
704
- });
705
- if (!completed) {
706
- this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping completion writes after losing issue-session lease");
707
- this.linearSync.clearProgress(run.id);
708
- this.activeThreadId = undefined;
709
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
710
- return;
711
- }
712
- if (postRunFollowUp) {
713
- this.feed?.publish({
714
- level: "info",
715
- kind: "stage",
716
- issueKey: issue.issueKey,
717
- projectId: run.projectId,
718
- stage: postRunFollowUp.factoryState,
719
- status: "follow_up_queued",
720
- summary: postRunFollowUp.summary,
721
- });
722
- this.enqueueIssue(run.projectId, run.linearIssueId);
723
- }
724
- this.feed?.publish({
725
- level: "info",
726
- kind: "turn",
727
- issueKey: issue.issueKey,
728
- projectId: run.projectId,
729
- stage: run.runType,
730
- status: "completed",
731
- summary: `Turn completed for ${run.runType}`,
732
- detail: summarizeCurrentThread(thread).latestAgentMessage,
293
+ await this.runFinalizer.finalizeCompletedRun({
294
+ source: "notification",
295
+ run,
296
+ issue,
297
+ thread,
298
+ threadId,
299
+ ...(completedTurnId ? { completedTurnId } : {}),
300
+ resolveRecoverableRunState: resolveRecoverablePostRunState,
733
301
  });
734
- // Emit Linear completion activity + plan
735
- const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
736
- const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
737
- void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
738
- runType: run.runType,
739
- completionSummary,
740
- postRunState: updatedIssue.factoryState,
741
- ...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
742
- }));
743
- void this.linearSync.syncSession(updatedIssue);
744
- this.linearSync.clearProgress(run.id);
745
302
  this.activeThreadId = undefined;
746
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
747
303
  }
748
304
  // ─── Active status for query ──────────────────────────────────────
749
305
  async getActiveRunStatus(issueKey) {
750
- const issue = this.db.getIssueByKey(issueKey);
306
+ const issue = this.db.issues.getIssueByKey(issueKey);
751
307
  if (!issue?.activeRunId)
752
308
  return undefined;
753
- const run = this.db.getRun(issue.activeRunId);
309
+ const run = this.db.runs.getRunById(issue.activeRunId);
754
310
  if (!run?.threadId)
755
311
  return undefined;
756
312
  const trackedIssue = this.db.issueToTrackedIssue(issue);
@@ -763,7 +319,7 @@ export class RunOrchestrator {
763
319
  }
764
320
  // ─── Reconciliation ───────────────────────────────────────────────
765
321
  async reconcileActiveRuns() {
766
- for (const run of this.db.listRunningRuns()) {
322
+ for (const run of this.db.runs.listRunningRuns()) {
767
323
  await this.reconcileRun(run);
768
324
  }
769
325
  // Preemptively detect stuck merge-queue PRs (conflicts visible on
@@ -775,7 +331,7 @@ export class RunOrchestrator {
775
331
  await this.reconcileMergedLinearCompletion();
776
332
  }
777
333
  async reconcileMergedLinearCompletion() {
778
- for (const issue of this.db.listIssues()) {
334
+ for (const issue of this.db.issues.listIssues()) {
779
335
  if (issue.prState !== "merged")
780
336
  continue;
781
337
  if (issue.currentLinearStateType?.trim().toLowerCase() === "completed")
@@ -790,7 +346,7 @@ export class RunOrchestrator {
790
346
  continue;
791
347
  const normalizedCurrent = liveIssue.stateName?.trim().toLowerCase();
792
348
  if (normalizedCurrent === targetState.trim().toLowerCase()) {
793
- this.db.upsertIssue({
349
+ this.db.issues.upsertIssue({
794
350
  projectId: issue.projectId,
795
351
  linearIssueId: issue.linearIssueId,
796
352
  ...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
@@ -799,7 +355,7 @@ export class RunOrchestrator {
799
355
  continue;
800
356
  }
801
357
  const updated = await linear.setIssueState(issue.linearIssueId, targetState);
802
- this.db.upsertIssue({
358
+ this.db.issues.upsertIssue({
803
359
  projectId: issue.projectId,
804
360
  linearIssueId: issue.linearIssueId,
805
361
  ...(updated.stateName ? { currentLinearState: updated.stateName } : {}),
@@ -821,121 +377,15 @@ export class RunOrchestrator {
821
377
  * escalate; backoff delay not elapsed → skip.
822
378
  */
823
379
  recoverOrEscalate(issue, runType, reason) {
824
- // Re-read issue after the run was cleared (activeRunId is now null)
825
- const fresh = this.db.getIssue(issue.projectId, issue.linearIssueId);
826
- if (!fresh)
827
- return;
828
- if (isRequestedChangesRunType(runType)) {
829
- const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
830
- this.db.clearPendingIssueSessionEventsWithLease(lease);
831
- this.db.upsertIssueWithLease(lease, {
832
- projectId: fresh.projectId,
833
- linearIssueId: fresh.linearIssueId,
834
- pendingRunType: null,
835
- pendingRunContextJson: null,
836
- factoryState: "escalated",
837
- });
838
- return true;
839
- });
840
- if (!updated) {
841
- this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping review-fix recovery escalation after losing issue-session lease");
842
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
843
- return;
844
- }
845
- this.logger.warn({ issueKey: fresh.issueKey, reason }, "Requested-changes run failed before a new head was published — escalating");
846
- this.feed?.publish({
847
- level: "error",
848
- kind: "workflow",
849
- issueKey: fresh.issueKey,
850
- projectId: fresh.projectId,
851
- stage: runType,
852
- status: "escalated",
853
- summary: `Requested-changes run failed before publishing a new head (${reason})`,
854
- });
855
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
856
- return;
857
- }
858
- // If PR already merged, transition to done — no retry needed
859
- if (fresh.prState === "merged") {
860
- const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
861
- this.db.upsertIssueWithLease(lease, {
862
- projectId: fresh.projectId,
863
- linearIssueId: fresh.linearIssueId,
864
- factoryState: "done",
865
- zombieRecoveryAttempts: 0,
866
- lastZombieRecoveryAt: null,
867
- });
868
- return true;
869
- });
870
- if (!updated) {
871
- this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping merged recovery completion after losing issue-session lease");
872
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
873
- return;
874
- }
875
- this.logger.info({ issueKey: fresh.issueKey, reason }, "Recovery: PR already merged — transitioning to done");
876
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
877
- return;
878
- }
879
- // Budget check
880
- const attempts = fresh.zombieRecoveryAttempts + 1;
881
- if (attempts > DEFAULT_ZOMBIE_RECOVERY_BUDGET) {
882
- const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
883
- this.db.upsertIssueWithLease(lease, {
884
- projectId: fresh.projectId,
885
- linearIssueId: fresh.linearIssueId,
886
- factoryState: "escalated",
887
- });
888
- return true;
889
- });
890
- if (!updated) {
891
- this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery escalation after losing issue-session lease");
892
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
893
- return;
894
- }
895
- this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: budget exhausted — escalating");
896
- this.feed?.publish({
897
- level: "error",
898
- kind: "workflow",
899
- issueKey: fresh.issueKey,
900
- projectId: fresh.projectId,
901
- stage: "escalated",
902
- status: "budget_exhausted",
903
- summary: `${reason} recovery failed after ${DEFAULT_ZOMBIE_RECOVERY_BUDGET} attempts`,
904
- });
905
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
906
- return;
907
- }
908
- // Exponential backoff — skip if delay hasn't elapsed
909
- if (fresh.lastZombieRecoveryAt) {
910
- const elapsed = Date.now() - new Date(fresh.lastZombieRecoveryAt).getTime();
911
- const delay = ZOMBIE_RECOVERY_BASE_DELAY_MS * Math.pow(2, fresh.zombieRecoveryAttempts);
912
- if (elapsed < delay) {
913
- this.logger.debug({ issueKey: fresh.issueKey, attempts: fresh.zombieRecoveryAttempts, delay, elapsed }, "Recovery: backoff not elapsed, skipping");
914
- return;
915
- }
916
- }
917
- // Re-enqueue with backoff tracking
918
- const requeued = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
919
- this.db.upsertIssueWithLease(lease, {
920
- projectId: fresh.projectId,
921
- linearIssueId: fresh.linearIssueId,
922
- pendingRunType: null,
923
- pendingRunContextJson: null,
924
- zombieRecoveryAttempts: attempts,
925
- lastZombieRecoveryAt: new Date().toISOString(),
926
- });
927
- return this.appendWakeEventWithLease(lease, fresh, runType, undefined, `recovery:${attempts}`);
380
+ this.runRecovery.recoverOrEscalate({
381
+ issue,
382
+ runType,
383
+ reason,
384
+ isRequestedChangesRunType,
928
385
  });
929
- if (!requeued) {
930
- this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery re-enqueue after losing issue-session lease");
931
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
932
- return;
933
- }
934
- this.enqueueIssue(fresh.projectId, fresh.linearIssueId);
935
- this.logger.info({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: re-enqueued with backoff");
936
386
  }
937
387
  async reconcileRun(run) {
938
- const issue = this.db.getIssue(run.projectId, run.linearIssueId);
388
+ const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
939
389
  if (!issue)
940
390
  return;
941
391
  let recoveryLease = this.claimLeaseForReconciliation(run.projectId, run.linearIssueId);
@@ -949,11 +399,11 @@ export class RunOrchestrator {
949
399
  // (e.g. pr_merged processed, DB manually edited), just release the run.
950
400
  if (TERMINAL_STATES.has(issue.factoryState)) {
951
401
  this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
952
- this.db.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
953
- this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
402
+ this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
403
+ this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
954
404
  });
955
405
  this.logger.info({ issueKey: issue.issueKey, runId: run.id, factoryState: issue.factoryState }, "Reconciliation: released run on terminal issue");
956
- const releasedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
406
+ const releasedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
957
407
  void this.linearSync.syncSession(releasedIssue, { activeRunType: run.runType });
958
408
  this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
959
409
  return;
@@ -962,11 +412,11 @@ export class RunOrchestrator {
962
412
  if (!run.threadId) {
963
413
  this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
964
414
  this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
965
- this.db.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
966
- this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
415
+ this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
416
+ this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
967
417
  });
968
418
  this.recoverOrEscalate(issue, run.runType, "zombie");
969
- const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
419
+ const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
970
420
  void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "The Codex turn never started before PatchRelay restarted."));
971
421
  void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
972
422
  this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
@@ -980,11 +430,11 @@ export class RunOrchestrator {
980
430
  catch {
981
431
  this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
982
432
  this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
983
- this.db.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
984
- this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
433
+ this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
434
+ this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
985
435
  });
986
436
  this.recoverOrEscalate(issue, run.runType, "stale_thread");
987
- const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
437
+ const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
988
438
  void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "PatchRelay lost the active Codex thread after restart and needs to recover."));
989
439
  void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
990
440
  this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
@@ -998,8 +448,8 @@ export class RunOrchestrator {
998
448
  const stopState = resolveAuthoritativeLinearStopState(linearIssue);
999
449
  if (stopState?.isFinal) {
1000
450
  this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
1001
- this.db.finishRun(run.id, { status: "released" });
1002
- this.db.upsertIssue({
451
+ this.db.runs.finishRun(run.id, { status: "released" });
452
+ this.db.issues.upsertIssue({
1003
453
  projectId: run.projectId,
1004
454
  linearIssueId: run.linearIssueId,
1005
455
  activeRunId: null,
@@ -1016,7 +466,7 @@ export class RunOrchestrator {
1016
466
  status: "reconciled",
1017
467
  summary: `Linear state ${stopState.stateName} \u2192 done`,
1018
468
  });
1019
- const doneIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
469
+ const doneIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
1020
470
  void this.linearSync.syncSession(doneIssue, { activeRunType: run.runType });
1021
471
  this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1022
472
  return;
@@ -1028,250 +478,20 @@ export class RunOrchestrator {
1028
478
  // The agent may have partially completed work (commits, PR) before interruption.
1029
479
  // Reactive loops (CI repair, review fix) will handle follow-up if needed.
1030
480
  if (latestTurn?.status === "interrupted") {
1031
- this.logger.warn({ issueKey: issue.issueKey, runType: run.runType, threadId: run.threadId }, "Run has interrupted turn — marking as failed");
1032
- // Interrupted runs are not real failures — undo the budget increment.
1033
- const repairedCounters = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
1034
- if (run.runType === "ci_repair" && issue.ciRepairAttempts > 0) {
1035
- this.db.upsertIssueWithLease(lease, {
1036
- projectId: issue.projectId,
1037
- linearIssueId: issue.linearIssueId,
1038
- ciRepairAttempts: issue.ciRepairAttempts - 1,
1039
- });
1040
- }
1041
- else if (run.runType === "queue_repair" && issue.queueRepairAttempts > 0) {
1042
- this.db.upsertIssueWithLease(lease, {
1043
- projectId: issue.projectId,
1044
- linearIssueId: issue.linearIssueId,
1045
- queueRepairAttempts: issue.queueRepairAttempts - 1,
1046
- });
1047
- }
1048
- if (run.runType === "ci_repair" || run.runType === "queue_repair") {
1049
- this.db.upsertIssueWithLease(lease, {
1050
- projectId: issue.projectId,
1051
- linearIssueId: issue.linearIssueId,
1052
- lastAttemptedFailureHeadSha: null,
1053
- lastAttemptedFailureSignature: null,
1054
- });
1055
- }
1056
- return true;
1057
- });
1058
- if (!repairedCounters) {
1059
- this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping interrupted-run recovery after losing issue-session lease");
1060
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1061
- return;
1062
- }
1063
- if (isRequestedChangesRunType(run.runType)) {
1064
- const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
1065
- const project = this.config.projects.find((entry) => entry.id === run.projectId);
1066
- const retryContext = project
1067
- ? await this.resolveRequestedChangesWakeContext(refreshedIssue, run.runType, run.runType === "branch_upkeep"
1068
- ? {
1069
- branchUpkeepRequired: true,
1070
- reviewFixMode: "branch_upkeep",
1071
- wakeReason: "branch_upkeep",
1072
- }
1073
- : undefined, project)
1074
- : undefined;
1075
- const retryRunType = resolveRequestedChangesMode(run.runType, retryContext) === "branch_upkeep"
1076
- ? "branch_upkeep"
1077
- : "review_fix";
1078
- const recoveredState = resolveRecoverablePostRunState(refreshedIssue) ?? "failed";
1079
- const interruptedMessage = "Requested-changes run was interrupted before PatchRelay could verify that a new PR head was published";
1080
- this.failRunAndClear(run, interruptedMessage, recoveredState);
1081
- await this.restoreIdleWorktree(issue);
1082
- const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
1083
- if (recoveredState === "changes_requested") {
1084
- this.db.upsertIssue({
1085
- projectId: run.projectId,
1086
- linearIssueId: run.linearIssueId,
1087
- pendingRunType: retryRunType,
1088
- pendingRunContextJson: retryContext ? JSON.stringify(retryContext) : null,
1089
- });
1090
- this.feed?.publish({
1091
- level: "warn",
1092
- kind: "workflow",
1093
- issueKey: issue.issueKey,
1094
- projectId: run.projectId,
1095
- stage: run.runType,
1096
- status: "retry_queued",
1097
- summary: "Requested-changes run was interrupted; PatchRelay will retry from fresh GitHub truth",
1098
- });
1099
- this.enqueueIssue(run.projectId, run.linearIssueId);
1100
- }
1101
- else {
1102
- this.feed?.publish({
1103
- level: "error",
1104
- kind: "workflow",
1105
- issueKey: issue.issueKey,
1106
- projectId: run.projectId,
1107
- stage: run.runType,
1108
- status: "escalated",
1109
- summary: interruptedMessage,
1110
- });
1111
- }
1112
- void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, interruptedMessage));
1113
- void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
1114
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1115
- return;
1116
- }
1117
- const recoveredState = resolveRecoverablePostRunState(this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
1118
- this.failRunAndClear(run, "Codex turn was interrupted", recoveredState);
1119
- await this.restoreIdleWorktree(issue);
1120
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
1121
- if (recoveredState) {
1122
- this.feed?.publish({
1123
- level: "info",
1124
- kind: "stage",
1125
- issueKey: issue.issueKey,
1126
- projectId: run.projectId,
1127
- stage: recoveredState,
1128
- status: "reconciled",
1129
- summary: `Interrupted ${run.runType} recovered \u2192 ${recoveredState}`,
1130
- });
1131
- }
1132
- else {
1133
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, "The Codex turn was interrupted."));
1134
- }
1135
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
1136
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
481
+ await this.interruptedRunRecovery.handle(run, issue);
1137
482
  return;
1138
483
  }
1139
484
  // Handle completed turn discovered during reconciliation
1140
485
  if (latestTurn?.status === "completed") {
1141
- const trackedIssue = this.db.issueToTrackedIssue(issue);
1142
- const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
1143
- const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
1144
- const verifiedRepairError = await this.verifyReactiveRunAdvancedBranch(run, freshIssue);
1145
- if (verifiedRepairError) {
1146
- const holdState = resolveRecoverablePostRunState(freshIssue) ?? "failed";
1147
- this.failRunAndClear(run, verifiedRepairError, holdState);
1148
- this.feed?.publish({
1149
- level: "warn",
1150
- kind: "turn",
1151
- issueKey: issue.issueKey,
1152
- projectId: run.projectId,
1153
- stage: run.runType,
1154
- status: "branch_not_advanced",
1155
- summary: verifiedRepairError,
1156
- });
1157
- const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
1158
- void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
1159
- void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
1160
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1161
- return;
1162
- }
1163
- const missingReviewFixHeadError = await this.verifyReviewFixAdvancedHead(run, freshIssue);
1164
- if (missingReviewFixHeadError) {
1165
- this.failRunAndClear(run, missingReviewFixHeadError, "escalated");
1166
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
1167
- this.feed?.publish({
1168
- level: "error",
1169
- kind: "turn",
1170
- issueKey: freshIssue.issueKey,
1171
- projectId: run.projectId,
1172
- stage: run.runType,
1173
- status: "same_head_review_handoff_blocked",
1174
- summary: missingReviewFixHeadError,
1175
- });
1176
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
1177
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
1178
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1179
- return;
1180
- }
1181
- const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
1182
- if (publishedOutcomeError) {
1183
- this.failRunAndClear(run, publishedOutcomeError, "failed");
1184
- this.feed?.publish({
1185
- level: "warn",
1186
- kind: "turn",
1187
- issueKey: issue.issueKey,
1188
- projectId: run.projectId,
1189
- stage: run.runType,
1190
- status: "publish_incomplete",
1191
- summary: publishedOutcomeError,
1192
- });
1193
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
1194
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
1195
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
1196
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1197
- return;
1198
- }
1199
- const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
1200
- const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
1201
- const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
1202
- const reconciled = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
1203
- this.db.finishRun(run.id, {
1204
- status: "completed",
1205
- ...(run.threadId ? { threadId: run.threadId } : {}),
1206
- ...(latestTurn.id ? { turnId: latestTurn.id } : {}),
1207
- summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
1208
- reportJson: JSON.stringify(report),
1209
- });
1210
- this.db.upsertIssue({
1211
- projectId: run.projectId,
1212
- linearIssueId: run.linearIssueId,
1213
- activeRunId: null,
1214
- ...(postRunState ? { factoryState: postRunState } : {}),
1215
- pendingRunType: null,
1216
- pendingRunContextJson: null,
1217
- ...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
1218
- ? {
1219
- lastGitHubFailureSource: null,
1220
- lastGitHubFailureHeadSha: null,
1221
- lastGitHubFailureSignature: null,
1222
- lastGitHubFailureCheckName: null,
1223
- lastGitHubFailureCheckUrl: null,
1224
- lastGitHubFailureContextJson: null,
1225
- lastGitHubFailureAt: null,
1226
- lastQueueIncidentJson: null,
1227
- lastAttemptedFailureHeadSha: null,
1228
- lastAttemptedFailureSignature: null,
1229
- }
1230
- : {})),
1231
- });
1232
- if (postRunFollowUp) {
1233
- return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
1234
- }
1235
- return true;
486
+ await this.runFinalizer.finalizeCompletedRun({
487
+ source: "reconciliation",
488
+ run,
489
+ issue,
490
+ thread,
491
+ threadId: run.threadId,
492
+ ...(latestTurn.id ? { completedTurnId: latestTurn.id } : {}),
493
+ resolveRecoverableRunState: resolveRecoverablePostRunState,
1236
494
  });
1237
- if (!reconciled) {
1238
- this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping reconciled completion writes after losing issue-session lease");
1239
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1240
- return;
1241
- }
1242
- if (postRunFollowUp) {
1243
- this.feed?.publish({
1244
- level: "info",
1245
- kind: "stage",
1246
- issueKey: issue.issueKey,
1247
- projectId: run.projectId,
1248
- stage: postRunFollowUp.factoryState,
1249
- status: "follow_up_queued",
1250
- summary: postRunFollowUp.summary,
1251
- });
1252
- this.enqueueIssue(run.projectId, run.linearIssueId);
1253
- }
1254
- if (postRunState) {
1255
- this.feed?.publish({
1256
- level: "info",
1257
- kind: "turn",
1258
- issueKey: issue.issueKey,
1259
- projectId: run.projectId,
1260
- stage: run.runType,
1261
- status: "completed",
1262
- summary: `Reconciliation: ${run.runType} completed \u2192 ${postRunState}`,
1263
- });
1264
- }
1265
- const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
1266
- const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
1267
- void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
1268
- runType: run.runType,
1269
- completionSummary,
1270
- postRunState: updatedIssue.factoryState,
1271
- ...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
1272
- }));
1273
- void this.linearSync.syncSession(updatedIssue);
1274
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1275
495
  return;
1276
496
  }
1277
497
  if (acquiredRecoveryLease)
@@ -1279,402 +499,24 @@ export class RunOrchestrator {
1279
499
  }
1280
500
  // ─── Internal helpers ─────────────────────────────────────────────
1281
501
  escalate(issue, runType, reason) {
1282
- this.logger.warn({ issueKey: issue.issueKey, runType, reason }, "Escalating to human");
1283
- const escalated = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
1284
- if (issue.activeRunId) {
1285
- this.db.finishRunWithLease(lease, issue.activeRunId, { status: "released" });
1286
- }
1287
- this.db.clearPendingIssueSessionEventsWithLease(lease);
1288
- this.db.upsertIssueWithLease(lease, {
1289
- projectId: issue.projectId,
1290
- linearIssueId: issue.linearIssueId,
1291
- pendingRunType: null,
1292
- pendingRunContextJson: null,
1293
- activeRunId: null,
1294
- factoryState: "escalated",
1295
- });
1296
- return true;
1297
- });
1298
- if (!escalated) {
1299
- this.logger.warn({ issueKey: issue.issueKey, runType }, "Skipping escalation write after losing issue-session lease");
1300
- this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
1301
- return;
1302
- }
1303
- this.feed?.publish({
1304
- level: "error",
1305
- kind: "workflow",
1306
- issueKey: issue.issueKey,
1307
- projectId: issue.projectId,
1308
- stage: runType,
1309
- status: "escalated",
1310
- summary: `Escalated: ${reason}`,
1311
- });
1312
- const escalatedIssue = this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
1313
- void this.linearSync.emitActivity(escalatedIssue, {
1314
- type: "error",
1315
- body: `PatchRelay needs human help to continue.\n\n${reason}`,
502
+ this.runRecovery.escalate({
503
+ issue,
504
+ runType,
505
+ reason,
1316
506
  });
1317
- void this.linearSync.syncSession(escalatedIssue);
1318
- this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
1319
507
  }
1320
508
  failRunAndClear(run, message, nextState = "failed") {
1321
- const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
1322
- this.db.finishRun(run.id, { status: "failed", failureReason: message });
1323
- if (nextState === "failed" || nextState === "escalated" || nextState === "awaiting_input" || nextState === "done") {
1324
- this.db.clearPendingIssueSessionEventsWithLease(lease);
1325
- }
1326
- this.db.upsertIssue({
1327
- projectId: run.projectId,
1328
- linearIssueId: run.linearIssueId,
1329
- activeRunId: null,
1330
- factoryState: nextState,
1331
- });
1332
- const branchOwner = this.resolveBranchOwnerForStateTransition(nextState);
1333
- if (branchOwner) {
1334
- const lease = this.getHeldIssueSessionLease(run.projectId, run.linearIssueId);
1335
- if (lease) {
1336
- this.db.setBranchOwnerWithLease(lease, branchOwner);
1337
- }
1338
- }
1339
- return true;
509
+ this.runRecovery.failRunAndClear({
510
+ run,
511
+ message,
512
+ nextState,
1340
513
  });
1341
- if (!updated) {
1342
- this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping failure cleanup after losing issue-session lease");
1343
- }
1344
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1345
514
  }
1346
515
  resolveBranchOwnerForStateTransition(newState, pendingRunType) {
1347
516
  return resolveBranchOwnerForStateTransition(newState, pendingRunType);
1348
517
  }
1349
- async verifyReactiveRunAdvancedBranch(run, issue) {
1350
- if (run.runType !== "ci_repair" && run.runType !== "queue_repair") {
1351
- return undefined;
1352
- }
1353
- if (!issue.prNumber || issue.prState !== "open" || !issue.lastGitHubFailureHeadSha) {
1354
- return undefined;
1355
- }
1356
- const project = this.config.projects.find((entry) => entry.id === run.projectId);
1357
- if (!project?.github?.repoFullName) {
1358
- return undefined;
1359
- }
1360
- try {
1361
- const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
1362
- if (!pr || pr.state?.toUpperCase() !== "OPEN")
1363
- return undefined;
1364
- if (!pr.headRefOid || pr.headRefOid !== issue.lastGitHubFailureHeadSha)
1365
- return undefined;
1366
- return `Repair finished but PR #${issue.prNumber} is still on failing head ${issue.lastGitHubFailureHeadSha.slice(0, 8)}`;
1367
- }
1368
- catch (error) {
1369
- this.logger.debug({
1370
- issueKey: issue.issueKey,
1371
- prNumber: issue.prNumber,
1372
- error: error instanceof Error ? error.message : String(error),
1373
- }, "Failed to verify PR head advancement after repair");
1374
- return undefined;
1375
- }
1376
- }
1377
- async verifyReviewFixAdvancedHead(run, issue) {
1378
- if (!isRequestedChangesRunType(run.runType)) {
1379
- return undefined;
1380
- }
1381
- if (!issue.prNumber || issue.prState !== "open") {
1382
- return undefined;
1383
- }
1384
- if (!run.sourceHeadSha) {
1385
- return `Requested-changes run finished for PR #${issue.prNumber} without a recorded starting head SHA. PatchRelay cannot verify that a new head was published.`;
1386
- }
1387
- const project = this.config.projects.find((entry) => entry.id === run.projectId);
1388
- if (!project?.github?.repoFullName) {
1389
- return undefined;
1390
- }
1391
- try {
1392
- const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
1393
- if (!pr || pr.state?.toUpperCase() !== "OPEN")
1394
- return undefined;
1395
- if (!pr.headRefOid) {
1396
- return `Requested-changes run finished for PR #${issue.prNumber} but GitHub did not report a current head SHA.`;
1397
- }
1398
- if (pr.headRefOid === run.sourceHeadSha) {
1399
- return `Requested-changes run finished for PR #${issue.prNumber} without pushing a new head; PatchRelay must not hand the same SHA back to review.`;
1400
- }
1401
- return undefined;
1402
- }
1403
- catch (error) {
1404
- this.logger.debug({
1405
- issueKey: issue.issueKey,
1406
- prNumber: issue.prNumber,
1407
- error: error instanceof Error ? error.message : String(error),
1408
- }, "Failed to verify PR head advancement after requested-changes work");
1409
- return undefined;
1410
- }
1411
- }
1412
- async refreshIssueAfterReactivePublish(run, issue) {
1413
- if (run.runType !== "ci_repair" && run.runType !== "queue_repair" && !isRequestedChangesRunType(run.runType)) {
1414
- return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
1415
- }
1416
- if (!issue.prNumber) {
1417
- return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
1418
- }
1419
- const project = this.config.projects.find((entry) => entry.id === run.projectId);
1420
- const repoFullName = project?.github?.repoFullName;
1421
- if (!repoFullName) {
1422
- return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
1423
- }
1424
- try {
1425
- const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
1426
- if (!pr) {
1427
- return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
1428
- }
1429
- const nextPrState = normalizeRemotePrState(pr.state);
1430
- const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
1431
- const gateCheckName = project?.gateChecks?.find((entry) => entry.trim())?.trim() ?? "verify";
1432
- const headAdvanced = Boolean(pr.headRefOid && pr.headRefOid !== issue.lastGitHubFailureHeadSha);
1433
- const reviewFixHeadAdvanced = isRequestedChangesRunType(run.runType)
1434
- && Boolean(pr.headRefOid && run.sourceHeadSha && pr.headRefOid !== run.sourceHeadSha);
1435
- this.upsertIssueIfLeaseHeld(run.projectId, run.linearIssueId, {
1436
- projectId: run.projectId,
1437
- linearIssueId: run.linearIssueId,
1438
- ...(nextPrState ? { prState: nextPrState } : {}),
1439
- ...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
1440
- ...(nextReviewState ? { prReviewState: nextReviewState } : {}),
1441
- ...((headAdvanced || reviewFixHeadAdvanced)
1442
- ? {
1443
- prCheckStatus: "pending",
1444
- lastGitHubFailureSource: null,
1445
- lastGitHubFailureHeadSha: null,
1446
- lastGitHubFailureSignature: null,
1447
- lastGitHubFailureCheckName: null,
1448
- lastGitHubFailureCheckUrl: null,
1449
- lastGitHubFailureContextJson: null,
1450
- lastGitHubFailureAt: null,
1451
- lastQueueIncidentJson: null,
1452
- lastAttemptedFailureHeadSha: null,
1453
- lastAttemptedFailureSignature: null,
1454
- lastGitHubCiSnapshotHeadSha: pr.headRefOid ?? null,
1455
- lastGitHubCiSnapshotGateCheckName: gateCheckName,
1456
- lastGitHubCiSnapshotGateCheckStatus: "pending",
1457
- lastGitHubCiSnapshotJson: null,
1458
- lastGitHubCiSnapshotSettledAt: null,
1459
- }
1460
- : {}),
1461
- }, "reactive publish refresh");
1462
- }
1463
- catch (error) {
1464
- this.logger.debug({
1465
- issueKey: issue.issueKey,
1466
- prNumber: issue.prNumber,
1467
- error: error instanceof Error ? error.message : String(error),
1468
- }, "Failed to refresh PR state after reactive publish");
1469
- }
1470
- return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
1471
- }
1472
- async loadRemotePrState(repoFullName, prNumber) {
1473
- const { stdout, exitCode } = await execCommand("gh", [
1474
- "pr", "view", String(prNumber),
1475
- "--repo", repoFullName,
1476
- "--json", "headRefOid,state,reviewDecision,mergeStateStatus",
1477
- ], { timeoutMs: 10_000 });
1478
- if (exitCode !== 0)
1479
- return undefined;
1480
- return JSON.parse(stdout);
1481
- }
1482
- async resolveRequestedChangesWakeContext(issue, runType, context, project) {
1483
- if (runType === "branch_upkeep" || isBranchUpkeepRequired(context)) {
1484
- return context;
1485
- }
1486
- if (!issue.prNumber || issue.prState !== "open" || issue.prReviewState !== "changes_requested") {
1487
- return context;
1488
- }
1489
- const repoFullName = project.github?.repoFullName;
1490
- if (!repoFullName) {
1491
- return context;
1492
- }
1493
- try {
1494
- const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
1495
- if (!pr)
1496
- return context;
1497
- const nextPrState = normalizeRemotePrState(pr.state);
1498
- const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
1499
- this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
1500
- projectId: issue.projectId,
1501
- linearIssueId: issue.linearIssueId,
1502
- ...(nextPrState ? { prState: nextPrState } : {}),
1503
- ...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
1504
- ...(nextReviewState ? { prReviewState: nextReviewState } : {}),
1505
- }, "review-fix wake refresh");
1506
- if (nextPrState !== "open")
1507
- return context;
1508
- if (nextReviewState && nextReviewState !== "changes_requested")
1509
- return context;
1510
- if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
1511
- return context;
1512
- return buildReviewFixBranchUpkeepContext(issue.prNumber, project.github?.baseBranch ?? "main", pr, context);
1513
- }
1514
- catch (error) {
1515
- this.logger.debug({
1516
- issueKey: issue.issueKey,
1517
- prNumber: issue.prNumber,
1518
- error: error instanceof Error ? error.message : String(error),
1519
- }, "Failed to resolve requested-changes wake context");
1520
- return context;
1521
- }
1522
- }
1523
- async resolvePostRunFollowUp(run, issue, projectOverride) {
1524
- if (run.runType !== "review_fix") {
1525
- return undefined;
1526
- }
1527
- if (!issue.prNumber || issue.prState !== "open") {
1528
- return undefined;
1529
- }
1530
- if (issue.prReviewState !== "changes_requested") {
1531
- return undefined;
1532
- }
1533
- const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
1534
- const repoFullName = project?.github?.repoFullName;
1535
- if (!repoFullName) {
1536
- return undefined;
1537
- }
1538
- try {
1539
- const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
1540
- if (!pr)
1541
- return undefined;
1542
- const nextPrState = normalizeRemotePrState(pr.state);
1543
- const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
1544
- this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
1545
- projectId: issue.projectId,
1546
- linearIssueId: issue.linearIssueId,
1547
- ...(nextPrState ? { prState: nextPrState } : {}),
1548
- ...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
1549
- ...(nextReviewState ? { prReviewState: nextReviewState } : {}),
1550
- }, "post-run follow-up refresh");
1551
- if (nextPrState !== "open")
1552
- return undefined;
1553
- if (nextReviewState && nextReviewState !== "changes_requested")
1554
- return undefined;
1555
- if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
1556
- return undefined;
1557
- return {
1558
- pendingRunType: "branch_upkeep",
1559
- factoryState: "changes_requested",
1560
- context: buildReviewFixBranchUpkeepContext(issue.prNumber, project?.github?.baseBranch ?? "main", pr),
1561
- summary: `PR #${issue.prNumber} is still dirty after review fix; queued branch upkeep`,
1562
- };
1563
- }
1564
- catch (error) {
1565
- this.logger.debug({
1566
- issueKey: issue.issueKey,
1567
- prNumber: issue.prNumber,
1568
- error: error instanceof Error ? error.message : String(error),
1569
- }, "Failed to resolve post-run PR upkeep");
1570
- return undefined;
1571
- }
1572
- }
1573
- async verifyPublishedRunOutcome(run, issue, projectOverride) {
1574
- if (run.runType !== "implementation") {
1575
- return undefined;
1576
- }
1577
- const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
1578
- const baseBranch = project?.github?.baseBranch ?? "main";
1579
- const deliveryMode = resolveImplementationDeliveryMode(issue, undefined, run.promptText);
1580
- if (deliveryMode === "linear_only") {
1581
- if (issue.prNumber !== undefined) {
1582
- return `Planning-only implementation should not open a PR, but PR #${issue.prNumber} was observed`;
1583
- }
1584
- return this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
1585
- }
1586
- if (issue.prNumber && issue.prState && issue.prState !== "closed") {
1587
- return undefined;
1588
- }
1589
- if (project?.github?.repoFullName && issue.branchName) {
1590
- try {
1591
- const { stdout, exitCode } = await execCommand("gh", [
1592
- "pr",
1593
- "list",
1594
- "--repo",
1595
- project.github.repoFullName,
1596
- "--head",
1597
- issue.branchName,
1598
- "--state",
1599
- "all",
1600
- "--json",
1601
- "number,url,state,author,headRefOid",
1602
- ], { timeoutMs: 10_000 });
1603
- if (exitCode === 0) {
1604
- const matches = JSON.parse(stdout);
1605
- const pr = matches[0];
1606
- if (pr?.number) {
1607
- this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
1608
- projectId: issue.projectId,
1609
- linearIssueId: issue.linearIssueId,
1610
- prNumber: pr.number,
1611
- ...(pr.url ? { prUrl: pr.url } : {}),
1612
- ...(pr.state ? { prState: pr.state.toLowerCase() } : {}),
1613
- ...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
1614
- ...(pr.author?.login ? { prAuthorLogin: pr.author.login } : {}),
1615
- }, "published PR verification refresh");
1616
- return undefined;
1617
- }
1618
- }
1619
- }
1620
- catch (error) {
1621
- this.logger.debug({
1622
- issueKey: issue.issueKey,
1623
- branchName: issue.branchName,
1624
- repoFullName: project.github.repoFullName,
1625
- error: error instanceof Error ? error.message : String(error),
1626
- }, "Failed to verify published PR state after implementation");
1627
- }
1628
- }
1629
- const details = await this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
1630
- return details ?? `Implementation completed without opening a PR for branch ${issue.branchName ?? issue.linearIssueId}`;
1631
- }
1632
- async describeLocalImplementationOutcome(issue, baseBranch, deliveryMode = "publish_pr") {
1633
- if (!issue.worktreePath) {
1634
- return undefined;
1635
- }
1636
- try {
1637
- const status = await execCommand(this.config.runner.gitBin, [
1638
- "-C",
1639
- issue.worktreePath,
1640
- "status",
1641
- "--short",
1642
- ], { timeoutMs: 10_000 });
1643
- const dirtyEntries = status.exitCode === 0
1644
- ? status.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
1645
- : [];
1646
- if (dirtyEntries.length > 0) {
1647
- if (deliveryMode === "linear_only") {
1648
- return `Planning-only implementation should not modify the repo; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
1649
- }
1650
- return `Implementation completed without opening a PR; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
1651
- }
1652
- }
1653
- catch {
1654
- // Best effort only.
1655
- }
1656
- try {
1657
- const ahead = await execCommand(this.config.runner.gitBin, [
1658
- "-C",
1659
- issue.worktreePath,
1660
- "rev-list",
1661
- "--count",
1662
- `origin/${baseBranch}..HEAD`,
1663
- ], { timeoutMs: 10_000 });
1664
- if (ahead.exitCode === 0) {
1665
- const count = Number(ahead.stdout.trim());
1666
- if (Number.isFinite(count) && count > 0) {
1667
- if (deliveryMode === "linear_only") {
1668
- return `Planning-only implementation should not create repo commits; worktree is ${count} local commit(s) ahead of origin/${baseBranch}`;
1669
- }
1670
- return `Implementation completed with ${count} local commit(s) ahead of origin/${baseBranch} but no PR was observed`;
1671
- }
1672
- }
1673
- }
1674
- catch {
1675
- // Best effort only.
1676
- }
1677
- return undefined;
518
+ async resolveRequestedChangesWakeContext(issue, runType, context) {
519
+ return await this.runCompletionPolicy.resolveRequestedChangesWakeContext(issue, runType, context);
1678
520
  }
1679
521
  async readThreadWithRetry(threadId, maxRetries = 3) {
1680
522
  for (let attempt = 0; attempt < maxRetries; attempt++) {
@@ -1689,32 +531,11 @@ export class RunOrchestrator {
1689
531
  }
1690
532
  throw new Error(`Failed to read thread ${threadId}`);
1691
533
  }
1692
- issueSessionLeaseKey(projectId, linearIssueId) {
1693
- return `${projectId}:${linearIssueId}`;
1694
- }
1695
534
  getHeldIssueSessionLease(projectId, linearIssueId) {
1696
- const leaseId = this.activeSessionLeases.get(this.issueSessionLeaseKey(projectId, linearIssueId));
1697
- if (!leaseId)
1698
- return undefined;
1699
- return { projectId, linearIssueId, leaseId };
535
+ return this.leaseService.getHeldLease(projectId, linearIssueId);
1700
536
  }
1701
537
  withHeldIssueSessionLease(projectId, linearIssueId, fn) {
1702
- const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
1703
- if (!lease)
1704
- return undefined;
1705
- return this.db.withIssueSessionLease(projectId, linearIssueId, lease.leaseId, () => fn(lease));
1706
- }
1707
- upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
1708
- const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
1709
- if (!lease) {
1710
- this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write without a held issue-session lease");
1711
- return undefined;
1712
- }
1713
- const updated = this.db.upsertIssueWithLease(lease, params);
1714
- if (!updated) {
1715
- this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
1716
- }
1717
- return updated;
538
+ return this.leaseService.withHeldLease(projectId, linearIssueId, fn);
1718
539
  }
1719
540
  assertLaunchLease(run, phase) {
1720
541
  if (this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
@@ -1725,197 +546,16 @@ export class RunOrchestrator {
1725
546
  this.logger.warn({ runId: run.id, issueId: run.linearIssueId, phase }, "Aborting run launch after losing issue-session lease");
1726
547
  throw error;
1727
548
  }
1728
- acquireIssueSessionLease(projectId, linearIssueId) {
1729
- const leaseId = randomUUID();
1730
- const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
1731
- const acquired = this.db.acquireIssueSessionLease({
1732
- projectId,
1733
- linearIssueId,
1734
- leaseId,
1735
- workerId: this.workerId,
1736
- leasedUntil,
1737
- });
1738
- if (!acquired)
1739
- return undefined;
1740
- this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
1741
- return leaseId;
1742
- }
1743
- forceAcquireIssueSessionLease(projectId, linearIssueId) {
1744
- const leaseId = randomUUID();
1745
- const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
1746
- const acquired = this.db.forceAcquireIssueSessionLease({
1747
- projectId,
1748
- linearIssueId,
1749
- leaseId,
1750
- workerId: this.workerId,
1751
- leasedUntil,
1752
- });
1753
- if (!acquired)
1754
- return undefined;
1755
- this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
1756
- return leaseId;
1757
- }
1758
549
  claimLeaseForReconciliation(projectId, linearIssueId) {
1759
- const key = this.issueSessionLeaseKey(projectId, linearIssueId);
1760
- if (this.activeSessionLeases.has(key)) {
1761
- return "owned";
1762
- }
1763
- const session = this.db.getIssueSession(projectId, linearIssueId);
1764
- if (!session)
1765
- return "skip";
1766
- const leasedUntilMs = session.leasedUntil ? Date.parse(session.leasedUntil) : undefined;
1767
- if (leasedUntilMs !== undefined && Number.isFinite(leasedUntilMs) && leasedUntilMs > Date.now()) {
1768
- return "skip";
1769
- }
1770
- return this.acquireIssueSessionLease(projectId, linearIssueId) ? true : "skip";
550
+ return this.leaseService.claimForReconciliation(projectId, linearIssueId);
1771
551
  }
1772
552
  async reclaimForeignRecoveryLeaseIfSafe(run, issue) {
1773
- const key = this.issueSessionLeaseKey(run.projectId, run.linearIssueId);
1774
- if (this.activeSessionLeases.has(key)) {
1775
- return false;
1776
- }
1777
- const session = this.db.getIssueSession(run.projectId, run.linearIssueId);
1778
- if (!session?.leaseId || !session.workerId || session.workerId === this.workerId) {
1779
- return false;
1780
- }
1781
- if (issue.activeRunId !== run.id) {
1782
- return false;
1783
- }
1784
- let safeToReclaim = !run.threadId;
1785
- if (!safeToReclaim && run.threadId) {
1786
- try {
1787
- const thread = await this.readThreadWithRetry(run.threadId, 1);
1788
- const latestTurn = getThreadTurns(thread).at(-1);
1789
- safeToReclaim = thread.status === "notLoaded"
1790
- || latestTurn?.status === "interrupted"
1791
- || latestTurn?.status === "completed";
1792
- }
1793
- catch {
1794
- safeToReclaim = true;
1795
- }
1796
- }
1797
- if (!safeToReclaim) {
1798
- return false;
1799
- }
1800
- const leaseId = this.forceAcquireIssueSessionLease(run.projectId, run.linearIssueId);
1801
- if (!leaseId) {
1802
- return false;
1803
- }
1804
- this.logger.info({
1805
- issueKey: issue.issueKey,
1806
- runId: run.id,
1807
- previousWorkerId: session.workerId,
1808
- previousLeaseId: session.leaseId,
1809
- reclaimedLeaseId: leaseId,
1810
- }, "Reclaimed foreign issue-session lease for active-run recovery");
1811
- return true;
553
+ return await this.leaseService.reclaimForeignRecoveryLeaseIfSafe(run, issue);
1812
554
  }
1813
555
  heartbeatIssueSessionLease(projectId, linearIssueId) {
1814
- const key = this.issueSessionLeaseKey(projectId, linearIssueId);
1815
- const leaseId = this.activeSessionLeases.get(key) ?? this.db.getIssueSession(projectId, linearIssueId)?.leaseId;
1816
- if (!leaseId)
1817
- return false;
1818
- const renewed = this.db.renewIssueSessionLease({
1819
- projectId,
1820
- linearIssueId,
1821
- leaseId,
1822
- leasedUntil: new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString(),
1823
- });
1824
- if (renewed) {
1825
- this.activeSessionLeases.set(key, leaseId);
1826
- return true;
1827
- }
1828
- this.activeSessionLeases.delete(key);
1829
- return false;
556
+ return this.leaseService.heartbeat(projectId, linearIssueId);
1830
557
  }
1831
558
  releaseIssueSessionLease(projectId, linearIssueId) {
1832
- const key = this.issueSessionLeaseKey(projectId, linearIssueId);
1833
- const leaseId = this.activeSessionLeases.get(key);
1834
- this.db.releaseIssueSessionLease(projectId, linearIssueId, leaseId);
1835
- this.activeSessionLeases.delete(key);
559
+ this.leaseService.release(projectId, linearIssueId);
1836
560
  }
1837
561
  }
1838
- /**
1839
- * Determine post-run factory state from current PR metadata.
1840
- * Used by both the normal completion path and reconciliation.
1841
- */
1842
- function resolvePostRunState(issue) {
1843
- if (ACTIVE_RUN_STATES.has(issue.factoryState) && issue.prNumber) {
1844
- // Check merged first — a merged PR is both approved and merged,
1845
- // and "done" must take priority over "awaiting_queue".
1846
- if (issue.prState === "merged")
1847
- return "done";
1848
- if (issue.prReviewState === "approved")
1849
- return "awaiting_queue";
1850
- return "pr_open";
1851
- }
1852
- return undefined;
1853
- }
1854
- function resolveCompletedRunState(issue, run) {
1855
- if (run.runType === "implementation" && resolveImplementationDeliveryMode(issue, undefined, run.promptText) === "linear_only") {
1856
- return "done";
1857
- }
1858
- return resolvePostRunState(issue);
1859
- }
1860
- function resolveRecoverablePostRunState(issue) {
1861
- if (!issue.prNumber) {
1862
- return resolvePostRunState(issue);
1863
- }
1864
- if (issue.prState === "merged")
1865
- return "done";
1866
- if (issue.prState === "open") {
1867
- const reactiveIntent = deriveIssueSessionReactiveIntent({
1868
- prNumber: issue.prNumber,
1869
- prState: issue.prState,
1870
- prReviewState: issue.prReviewState,
1871
- prCheckStatus: issue.prCheckStatus,
1872
- latestFailureSource: issue.lastGitHubFailureSource,
1873
- });
1874
- if (reactiveIntent)
1875
- return reactiveIntent.compatibilityFactoryState;
1876
- if (issue.prReviewState === "approved")
1877
- return "awaiting_queue";
1878
- return "pr_open";
1879
- }
1880
- return resolvePostRunState(issue);
1881
- }
1882
- function normalizeRemotePrState(value) {
1883
- const normalized = value?.trim().toUpperCase();
1884
- if (normalized === "OPEN")
1885
- return "open";
1886
- if (normalized === "CLOSED")
1887
- return "closed";
1888
- if (normalized === "MERGED")
1889
- return "merged";
1890
- return undefined;
1891
- }
1892
- function normalizeRemoteReviewDecision(value) {
1893
- const normalized = value?.trim().toUpperCase();
1894
- if (normalized === "APPROVED")
1895
- return "approved";
1896
- if (normalized === "CHANGES_REQUESTED")
1897
- return "changes_requested";
1898
- if (normalized === "REVIEW_REQUIRED")
1899
- return "commented";
1900
- return undefined;
1901
- }
1902
- function isDirtyMergeStateStatus(value) {
1903
- return value?.trim().toUpperCase() === "DIRTY";
1904
- }
1905
- function buildReviewFixBranchUpkeepContext(prNumber, baseBranch, pr, context) {
1906
- const promptContext = [
1907
- `The requested code change may already be present, but GitHub still reports PR #${prNumber} as ${String(pr.mergeStateStatus)} against latest ${baseBranch}.`,
1908
- `This turn is branch upkeep on the existing PR branch: update onto latest ${baseBranch}, resolve any conflicts, rerun the narrowest relevant verification, and push a newer head.`,
1909
- "Do not stop just because the requested code change is already present. Review can only move forward after a new pushed head.",
1910
- ].join(" ");
1911
- return {
1912
- ...(context ?? {}),
1913
- branchUpkeepRequired: true,
1914
- reviewFixMode: "branch_upkeep",
1915
- wakeReason: "branch_upkeep",
1916
- promptContext,
1917
- ...(pr.mergeStateStatus ? { mergeStateStatus: pr.mergeStateStatus } : {}),
1918
- ...(pr.headRefOid ? { failingHeadSha: pr.headRefOid } : {}),
1919
- baseBranch,
1920
- };
1921
- }