patchrelay 0.36.7 → 0.36.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,37 +1,20 @@
1
- import { randomUUID } from "node:crypto";
2
1
  import { ACTIVE_RUN_STATES, TERMINAL_STATES } from "./factory-state.js";
3
- import { buildHookEnv, runProjectHook } from "./hook-runner.js";
4
- import { buildStageReport, countEventMethods, extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread, } from "./run-reporting.js";
5
- import { buildRunCompletedActivity, buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
2
+ import { extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread } from "./run-reporting.js";
3
+ import { buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
6
4
  import { WorktreeManager } from "./worktree-manager.js";
7
5
  import { resolveAuthoritativeLinearStopState, resolvePreferredCompletedLinearState } from "./linear-workflow.js";
8
6
  import { execCommand } from "./utils.js";
9
7
  import { getThreadTurns } from "./codex-thread-utils.js";
10
8
  import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
11
- import { loadPatchRelayRepoPrompting } from "./patchrelay-customization.js";
12
- import { buildRunPrompt as buildPatchRelayRunPrompt, findDisallowedPatchRelayPromptSectionIds, findUnknownPatchRelayPromptSectionIds, mergePromptCustomizationLayers, resolveImplementationDeliveryMode, resolvePromptLayers, } from "./prompting/patchrelay.js";
13
- const DEFAULT_CI_REPAIR_BUDGET = 3;
14
- const DEFAULT_QUEUE_REPAIR_BUDGET = 3;
15
- // Requested-changes loops can legitimately take more iterations than CI/queue
16
- // repair when the reviewer is catching nuanced product or timing bugs across
17
- // successive heads. Keep a hard ceiling to prevent infinite ping-pong, but make
18
- // it wide enough that real review cycles can continue after multiple successful
19
- // head advances.
20
- const DEFAULT_REVIEW_FIX_BUDGET = 12;
21
- const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
22
- const ZOMBIE_RECOVERY_BASE_DELAY_MS = 15_000; // 15s, 30s, 60s, 120s, 240s
23
- const ISSUE_SESSION_LEASE_MS = 10 * 60_000;
24
- const MAX_THREAD_GENERATION_BEFORE_COMPACTION = 4;
25
- const MAX_FOLLOW_UPS_BEFORE_COMPACTION = 4;
26
9
  import { QueueHealthMonitor } from "./queue-health-monitor.js";
10
+ import { resolveImplementationDeliveryMode, } from "./prompting/patchrelay.js";
27
11
  import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
28
12
  import { LinearSessionSync } from "./linear-session-sync.js";
29
- function slugify(value) {
30
- return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
31
- }
32
- function sanitizePathSegment(value) {
33
- return value.replace(/[^a-zA-Z0-9._-]+/g, "-");
34
- }
13
+ import { IssueSessionLeaseService } from "./issue-session-lease-service.js";
14
+ import { RunFinalizer } from "./run-finalizer.js";
15
+ import { RunLauncher } from "./run-launcher.js";
16
+ import { RunRecoveryService } from "./run-recovery-service.js";
17
+ import { RunWakePlanner } from "./run-wake-planner.js";
35
18
  function lowerCaseFirst(value) {
36
19
  return value ? `${value.slice(0, 1).toLowerCase()}${value.slice(1)}` : value;
37
20
  }
@@ -46,15 +29,6 @@ function resolveRequestedChangesMode(runType, context) {
46
29
  ? "branch_upkeep"
47
30
  : "address_review_feedback";
48
31
  }
49
- function shouldCompactThread(issue, threadGeneration, context) {
50
- const followUpCount = typeof context?.followUpCount === "number" ? context.followUpCount : 0;
51
- return issue.threadId !== undefined
52
- && (threadGeneration ?? 0) >= MAX_THREAD_GENERATION_BEFORE_COMPACTION
53
- && followUpCount >= MAX_FOLLOW_UPS_BEFORE_COMPACTION;
54
- }
55
- export function shouldReuseIssueThread(params) {
56
- return Boolean(params.existingThreadId) && !params.compactThread && params.resumeThread;
57
- }
58
32
  function isBranchUpkeepRequired(context) {
59
33
  return context?.branchUpkeepRequired === true;
60
34
  }
@@ -73,7 +47,12 @@ export class RunOrchestrator {
73
47
  linearSync;
74
48
  activeThreadId;
75
49
  workerId = `patchrelay:${process.pid}`;
76
- activeSessionLeases = new Map();
50
+ leaseService;
51
+ runFinalizer;
52
+ runLauncher;
53
+ runRecovery;
54
+ runWakePlanner;
55
+ activeSessionLeases;
77
56
  botIdentity;
78
57
  constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
79
58
  this.config = config;
@@ -85,6 +64,12 @@ export class RunOrchestrator {
85
64
  this.feed = feed;
86
65
  this.worktreeManager = new WorktreeManager(config);
87
66
  this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
67
+ this.leaseService = new IssueSessionLeaseService(db, logger, this.workerId, (threadId, maxRetries) => this.readThreadWithRetry(threadId, maxRetries));
68
+ this.activeSessionLeases = this.leaseService.activeSessionLeases;
69
+ this.runFinalizer = new RunFinalizer(db, logger, this.linearSync, this.enqueueIssue, feed);
70
+ this.runLauncher = new RunLauncher(config, db, codex, logger, this.worktreeManager);
71
+ this.runRecovery = new RunRecoveryService(db, logger, this.linearSync, (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (projectId, issueId) => this.enqueueIssue(projectId, issueId), (newState, pendingRunType) => this.resolveBranchOwnerForStateTransition(newState, pendingRunType), feed);
72
+ this.runWakePlanner = new RunWakePlanner(db);
88
73
  this.idleReconciler = new IdleIssueReconciler(db, config, {
89
74
  enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
90
75
  }, logger, feed);
@@ -94,88 +79,40 @@ export class RunOrchestrator {
94
79
  }, logger, feed);
95
80
  }
96
81
  resolveRunWake(issue) {
97
- const sessionWake = this.db.peekIssueSessionWake(issue.projectId, issue.linearIssueId);
98
- if (sessionWake) {
99
- return {
100
- runType: sessionWake.runType,
101
- context: sessionWake.context,
102
- wakeReason: sessionWake.wakeReason,
103
- resumeThread: sessionWake.resumeThread,
104
- eventIds: sessionWake.eventIds,
105
- };
106
- }
107
- return undefined;
82
+ return this.runWakePlanner.resolveRunWake(issue);
108
83
  }
109
84
  appendWakeEventWithLease(lease, issue, runType, context, dedupeScope) {
110
- let eventType;
111
- let dedupeKey;
112
- if (runType === "queue_repair") {
113
- eventType = "merge_steward_incident";
114
- dedupeKey = `${dedupeScope ?? "wake"}:queue_repair:${issue.linearIssueId}:${issue.prHeadSha ?? issue.lastGitHubFailureHeadSha ?? "unknown-sha"}`;
115
- }
116
- else if (runType === "ci_repair") {
117
- eventType = "settled_red_ci";
118
- dedupeKey = `${dedupeScope ?? "wake"}:ci_repair:${issue.linearIssueId}:${issue.lastGitHubFailureSignature ?? issue.prHeadSha ?? "unknown-sha"}`;
119
- }
120
- else if (runType === "review_fix" || runType === "branch_upkeep") {
121
- eventType = "review_changes_requested";
122
- dedupeKey = `${dedupeScope ?? "wake"}:${runType}:${issue.linearIssueId}:${issue.prHeadSha ?? "unknown-sha"}`;
123
- }
124
- else {
125
- eventType = "delegated";
126
- dedupeKey = `${dedupeScope ?? "wake"}:implementation:${issue.linearIssueId}`;
127
- }
128
- return Boolean(this.db.appendIssueSessionEventWithLease(lease, {
129
- projectId: issue.projectId,
130
- linearIssueId: issue.linearIssueId,
131
- eventType,
132
- ...(context ? { eventJson: JSON.stringify(context) } : {}),
133
- dedupeKey,
134
- }));
85
+ return this.runWakePlanner.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope);
135
86
  }
136
87
  materializeLegacyPendingWake(issue, lease) {
137
- if (!issue.pendingRunType)
138
- return issue;
139
- const context = issue.pendingRunContextJson
140
- ? JSON.parse(issue.pendingRunContextJson)
141
- : undefined;
142
- this.appendWakeEventWithLease(lease, issue, issue.pendingRunType, context, "legacy_pending");
143
- const updated = this.db.upsertIssueWithLease(lease, {
144
- projectId: issue.projectId,
145
- linearIssueId: issue.linearIssueId,
146
- pendingRunType: null,
147
- pendingRunContextJson: null,
148
- });
149
- if (!updated)
150
- return issue;
151
- return this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
88
+ return this.runWakePlanner.materializeLegacyPendingWake(issue, lease);
152
89
  }
153
90
  // ─── Run ────────────────────────────────────────────────────────
154
91
  async run(item) {
155
92
  const project = this.config.projects.find((p) => p.id === item.projectId);
156
93
  if (!project)
157
94
  return;
158
- if (this.activeSessionLeases.has(this.issueSessionLeaseKey(item.projectId, item.issueId))) {
95
+ if (this.leaseService.hasLocalLease(item.projectId, item.issueId)) {
159
96
  return;
160
97
  }
161
98
  const issue = this.db.getIssue(item.projectId, item.issueId);
162
99
  if (!issue || issue.activeRunId !== undefined)
163
100
  return;
164
- const issueSession = this.db.getIssueSession(item.projectId, item.issueId);
165
- const leaseId = this.acquireIssueSessionLease(item.projectId, item.issueId);
101
+ const issueSession = this.db.issueSessions.getIssueSession(item.projectId, item.issueId);
102
+ const leaseId = this.leaseService.acquire(item.projectId, item.issueId);
166
103
  if (!leaseId) {
167
104
  this.logger.info({ issueKey: issue.issueKey, projectId: item.projectId }, "Skipped run because another worker holds the session lease");
168
105
  return;
169
106
  }
170
107
  if (issue.prState === "merged") {
171
- this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
172
- this.releaseIssueSessionLease(item.projectId, item.issueId);
108
+ this.db.issueSessions.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" });
109
+ this.leaseService.release(item.projectId, item.issueId);
173
110
  return;
174
111
  }
175
112
  const wakeIssue = this.materializeLegacyPendingWake(issue, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
176
113
  const wake = this.resolveRunWake(wakeIssue);
177
114
  if (!wake) {
178
- this.releaseIssueSessionLease(item.projectId, item.issueId);
115
+ this.leaseService.release(item.projectId, item.issueId);
179
116
  return;
180
117
  }
181
118
  const { runType, context, resumeThread } = wake;
@@ -187,112 +124,33 @@ export class RunOrchestrator {
187
124
  : typeof effectiveContext?.headSha === "string"
188
125
  ? effectiveContext.headSha
189
126
  : issue.prHeadSha;
190
- // Check repair budgets
191
- if (runType === "ci_repair" && issue.ciRepairAttempts >= DEFAULT_CI_REPAIR_BUDGET) {
192
- this.escalate(issue, runType, `CI repair budget exhausted (${DEFAULT_CI_REPAIR_BUDGET} attempts)`);
193
- return;
194
- }
195
- if (runType === "queue_repair" && issue.queueRepairAttempts >= DEFAULT_QUEUE_REPAIR_BUDGET) {
196
- this.escalate(issue, runType, `Queue repair budget exhausted (${DEFAULT_QUEUE_REPAIR_BUDGET} attempts)`);
127
+ const budgetExceeded = this.runWakePlanner.budgetExceeded(issue, runType, isRequestedChangesRunType);
128
+ if (budgetExceeded) {
129
+ this.escalate(issue, runType, budgetExceeded);
197
130
  return;
198
131
  }
199
- if (isRequestedChangesRunType(runType) && issue.reviewFixAttempts >= DEFAULT_REVIEW_FIX_BUDGET) {
200
- this.escalate(issue, runType, `Requested-changes budget exhausted (${DEFAULT_REVIEW_FIX_BUDGET} attempts)`);
132
+ if (!this.runWakePlanner.incrementAttemptCounters(issue, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, runType, isRequestedChangesRunType)) {
133
+ this.releaseIssueSessionLease(item.projectId, item.issueId);
201
134
  return;
202
135
  }
203
- // Increment repair counters
204
- if (runType === "ci_repair") {
205
- const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, ciRepairAttempts: issue.ciRepairAttempts + 1 });
206
- if (!updated) {
207
- this.releaseIssueSessionLease(item.projectId, item.issueId);
208
- return;
209
- }
210
- }
211
- if (runType === "queue_repair") {
212
- const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueRepairAttempts: issue.queueRepairAttempts + 1 });
213
- if (!updated) {
214
- this.releaseIssueSessionLease(item.projectId, item.issueId);
215
- return;
216
- }
217
- }
218
- if (isRequestedChangesRunType(runType)) {
219
- const updated = this.db.upsertIssueWithLease({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId }, { projectId: issue.projectId, linearIssueId: issue.linearIssueId, reviewFixAttempts: issue.reviewFixAttempts + 1 });
220
- if (!updated) {
221
- this.releaseIssueSessionLease(item.projectId, item.issueId);
222
- return;
223
- }
224
- }
225
- const repoPrompting = loadPatchRelayRepoPrompting({
226
- repoRoot: project.repoPath,
227
- logger: this.logger,
228
- });
229
- const promptLayer = mergePromptCustomizationLayers(resolvePromptLayers(this.config.prompting, runType), resolvePromptLayers(repoPrompting, runType));
230
- const unknownPromptSections = findUnknownPatchRelayPromptSectionIds(promptLayer);
231
- if (unknownPromptSections.length > 0) {
232
- this.logger.warn({ issueKey: issue.issueKey, runType, unknownPromptSections }, "PatchRelay prompt customization references unknown section ids");
233
- }
234
- const disallowedPromptSections = findDisallowedPatchRelayPromptSectionIds(promptLayer);
235
- if (disallowedPromptSections.length > 0) {
236
- this.logger.warn({ issueKey: issue.issueKey, runType, disallowedPromptSections }, "PatchRelay prompt customization attempted to replace non-overridable sections");
237
- }
238
- const prompt = buildPatchRelayRunPrompt({
136
+ const { prompt, branchName, worktreePath } = this.runLauncher.prepareLaunchPlan({
137
+ project,
239
138
  issue,
240
139
  runType,
241
- repoPath: project.repoPath,
242
- ...(effectiveContext ? { context: effectiveContext } : {}),
243
- ...(promptLayer ? { promptLayer } : {}),
140
+ ...(effectiveContext ? { effectiveContext } : {}),
244
141
  });
245
- // Resolve workspace
246
- const issueRef = sanitizePathSegment(issue.issueKey ?? issue.linearIssueId);
247
- const slug = issue.title ? slugify(issue.title) : "";
248
- const branchSuffix = slug ? `${issueRef}-${slug}` : issueRef;
249
- const branchName = issue.branchName ?? `${project.branchPrefix}/${branchSuffix}`;
250
- const worktreePath = issue.worktreePath ?? `${project.worktreeRoot}/${issueRef}`;
251
- // Claim the run atomically
252
- const run = this.db.withIssueSessionLease(item.projectId, item.issueId, leaseId, () => {
253
- const fresh = this.db.getIssue(item.projectId, item.issueId);
254
- if (!fresh || fresh.activeRunId !== undefined)
255
- return undefined;
256
- const wakeIssue = this.materializeLegacyPendingWake(fresh, { projectId: item.projectId, linearIssueId: item.issueId, leaseId });
257
- const freshWake = this.resolveRunWake(wakeIssue);
258
- if (!freshWake || freshWake.runType !== runType)
259
- return undefined;
260
- const created = this.db.createRun({
261
- issueId: fresh.id,
262
- projectId: item.projectId,
263
- linearIssueId: item.issueId,
264
- runType,
265
- ...(sourceHeadSha ? { sourceHeadSha } : {}),
266
- promptText: prompt,
267
- });
268
- const failureHeadSha = typeof effectiveContext?.failureHeadSha === "string"
269
- ? effectiveContext.failureHeadSha
270
- : typeof effectiveContext?.headSha === "string" ? effectiveContext.headSha : undefined;
271
- const failureSignature = typeof effectiveContext?.failureSignature === "string" ? effectiveContext.failureSignature : undefined;
272
- this.db.upsertIssue({
273
- projectId: item.projectId,
274
- linearIssueId: item.issueId,
275
- pendingRunType: null,
276
- pendingRunContextJson: null,
277
- activeRunId: created.id,
278
- branchName,
279
- worktreePath,
280
- factoryState: runType === "implementation" ? "implementing"
281
- : runType === "ci_repair" ? "repairing_ci"
282
- : runType === "review_fix" || runType === "branch_upkeep" ? "changes_requested"
283
- : runType === "queue_repair" ? "repairing_queue"
284
- : "implementing",
285
- ...((runType === "ci_repair" || runType === "queue_repair") && failureSignature
286
- ? {
287
- lastAttemptedFailureSignature: failureSignature,
288
- lastAttemptedFailureHeadSha: failureHeadSha ?? null,
289
- }
290
- : {}),
291
- });
292
- this.db.consumeIssueSessionEvents(item.projectId, item.issueId, freshWake.eventIds, created.id);
293
- this.db.setIssueSessionLastWakeReason(item.projectId, item.issueId, freshWake.wakeReason ?? null);
294
- this.db.setBranchOwnerWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, "patchrelay");
295
- return created;
142
+ const run = this.runLauncher.claimRun({
143
+ item,
144
+ issue,
145
+ leaseId,
146
+ runType,
147
+ prompt,
148
+ ...(sourceHeadSha ? { sourceHeadSha } : {}),
149
+ ...(effectiveContext ? { effectiveContext } : {}),
150
+ materializeLegacyPendingWake: (targetIssue, lease) => this.materializeLegacyPendingWake(targetIssue, lease),
151
+ resolveRunWake: (targetIssue) => this.resolveRunWake(targetIssue),
152
+ branchName,
153
+ worktreePath,
296
154
  });
297
155
  if (!run) {
298
156
  this.releaseIssueSessionLease(item.projectId, item.issueId);
@@ -307,110 +165,36 @@ export class RunOrchestrator {
307
165
  status: "starting",
308
166
  summary: `Starting ${runType} run`,
309
167
  });
310
- let threadId;
311
- let turnId;
312
- let parentThreadId;
313
- try {
314
- // Ensure worktree
315
- await this.worktreeManager.ensureIssueWorktree(project.repoPath, project.worktreeRoot, worktreePath, branchName, { allowExistingOutsideRoot: issue.branchName !== undefined });
316
- // Set bot git identity and push credentials when GitHub App is configured.
317
- // This ensures commits are authored by and pushes are authenticated as
318
- // patchrelay[bot], not the system user.
319
- if (this.botIdentity) {
320
- const gitBin = this.config.runner.gitBin;
321
- await execCommand(gitBin, ["-C", worktreePath, "config", "user.name", this.botIdentity.name], { timeoutMs: 5_000 });
322
- await execCommand(gitBin, ["-C", worktreePath, "config", "user.email", this.botIdentity.email], { timeoutMs: 5_000 });
323
- // Override credential helper to use the App installation token for git push.
324
- // The helper script reads the token file and returns it as the password.
325
- const credentialHelper = `!f() { echo "username=x-access-token"; echo "password=$(cat ${this.botIdentity.tokenFile})"; }; f`;
326
- await execCommand(gitBin, ["-C", worktreePath, "config", "credential.helper", credentialHelper], { timeoutMs: 5_000 });
327
- }
328
- await this.resetWorktreeToTrackedBranch(worktreePath, branchName, issue);
329
- // Freshen the worktree: fetch + rebase onto latest base branch.
330
- // This prevents branch contamination when local main has drifted
331
- // and avoids scope-bundling review rejections from stale commits.
332
- // Skip for queue_repair — its entire purpose is to resolve rebase conflicts.
333
- if (runType !== "queue_repair") {
334
- await this.freshenWorktree(worktreePath, project, issue);
335
- }
336
- // Run prepare-worktree hook
337
- const hookEnv = buildHookEnv(issue.issueKey ?? issue.linearIssueId, branchName, runType, worktreePath);
338
- const prepareResult = await runProjectHook(project.repoPath, "prepare-worktree", { cwd: worktreePath, env: hookEnv });
339
- if (prepareResult.ran && prepareResult.exitCode !== 0) {
340
- throw new Error(`prepare-worktree hook failed (exit ${prepareResult.exitCode}): ${prepareResult.stderr?.slice(0, 500) ?? ""}`);
341
- }
342
- this.assertLaunchLease(run, "before starting the Codex turn");
343
- // Reuse the existing thread only for additive follow-ups that explicitly
344
- // request continuity. Fresh review-fix runs now start a new thread so the
345
- // model is not anchored to the implementation conversation that produced
346
- // the rejected patch. If the thread has accumulated many resumptions and
347
- // batched follow-ups, compact by starting a fresh main thread while
348
- // keeping a parent link.
349
- const compactThread = shouldCompactThread(issue, issueSession?.threadGeneration, effectiveContext);
350
- if (compactThread && issue.threadId) {
351
- parentThreadId = issue.threadId;
352
- }
353
- if (shouldReuseIssueThread({ existingThreadId: issue.threadId, compactThread, resumeThread })) {
354
- threadId = issue.threadId;
355
- }
356
- else {
357
- const thread = await this.codex.startThread({ cwd: worktreePath });
358
- threadId = thread.id;
359
- this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
360
- }
361
- try {
362
- const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
363
- turnId = turn.turnId;
364
- }
365
- catch (turnError) {
366
- // If the thread is stale (e.g. after app-server restart), start fresh and retry once.
367
- const msg = turnError instanceof Error ? turnError.message : String(turnError);
368
- if (msg.includes("thread not found") || msg.includes("not materialized")) {
369
- this.logger.info({ issueKey: issue.issueKey, staleThreadId: threadId }, "Thread is stale, retrying with fresh thread");
370
- const thread = await this.codex.startThread({ cwd: worktreePath });
371
- threadId = thread.id;
372
- this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, { projectId: item.projectId, linearIssueId: item.issueId, threadId });
373
- const turn = await this.codex.startTurn({ threadId, cwd: worktreePath, input: prompt });
374
- turnId = turn.turnId;
375
- }
376
- else {
377
- throw turnError;
378
- }
379
- }
380
- this.assertLaunchLease(run, "after starting the Codex turn");
381
- }
382
- catch (error) {
383
- const message = error instanceof Error ? error.message : String(error);
384
- const lostLease = error instanceof Error && error.name === "IssueSessionLeaseLostError";
385
- if (!lostLease) {
386
- const nextState = isRequestedChangesRunType(runType) ? "escalated" : "failed";
387
- this.db.finishRunWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, run.id, {
388
- status: "failed",
389
- failureReason: message,
390
- });
391
- this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
392
- projectId: item.projectId,
393
- linearIssueId: item.issueId,
394
- activeRunId: null,
395
- factoryState: nextState,
396
- });
397
- }
398
- this.logger.error({ issueKey: issue.issueKey, runType, error: message }, `Failed to launch ${runType} run`);
399
- const failedIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
400
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(runType, `Failed to start ${lowerCaseFirst(message)}`));
401
- void this.linearSync.syncSession(failedIssue, { activeRunType: runType });
402
- this.releaseIssueSessionLease(item.projectId, item.issueId);
403
- throw error;
404
- }
168
+ const { threadId, turnId, parentThreadId, } = await this.runLauncher.launchTurn({
169
+ project,
170
+ issue,
171
+ ...(issueSession ? { issueSession } : {}),
172
+ run,
173
+ runType,
174
+ prompt,
175
+ branchName,
176
+ worktreePath,
177
+ resumeThread,
178
+ ...(effectiveContext ? { effectiveContext } : {}),
179
+ leaseId,
180
+ ...(this.botIdentity ? { botIdentity: this.botIdentity } : {}),
181
+ assertLaunchLease: (targetRun, phase) => this.assertLaunchLease(targetRun, phase),
182
+ resetWorktreeToTrackedBranch: (targetWorktreePath, targetBranchName, targetIssue) => this.resetWorktreeToTrackedBranch(targetWorktreePath, targetBranchName, targetIssue),
183
+ freshenWorktree: (targetWorktreePath, targetProject, targetIssue) => this.freshenWorktree(targetWorktreePath, targetProject, targetIssue),
184
+ linearSync: this.linearSync,
185
+ releaseLease: (projectId, issueId) => this.releaseIssueSessionLease(projectId, issueId),
186
+ isRequestedChangesRunType,
187
+ lowerCaseFirst,
188
+ });
405
189
  this.assertLaunchLease(run, "before recording the active thread");
406
- if (!this.db.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
190
+ if (!this.db.issueSessions.updateRunThreadWithLease({ projectId: run.projectId, linearIssueId: run.linearIssueId, leaseId }, run.id, { threadId, turnId, ...(parentThreadId ? { parentThreadId } : {}) })) {
407
191
  this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping run thread update after losing issue-session lease");
408
192
  this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
409
193
  return;
410
194
  }
411
195
  // Reset zombie recovery counter — this run started successfully
412
196
  if (issue.zombieRecoveryAttempts > 0) {
413
- this.db.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
197
+ this.db.issueSessions.upsertIssueWithLease({ projectId: item.projectId, linearIssueId: item.issueId, leaseId }, {
414
198
  projectId: item.projectId,
415
199
  linearIssueId: item.issueId,
416
200
  zombieRecoveryAttempts: 0,
@@ -526,7 +310,7 @@ export class RunOrchestrator {
526
310
  if (notification.method === "turn/started" && threadId) {
527
311
  this.activeThreadId = threadId;
528
312
  }
529
- const run = this.db.getRunByThreadId(threadId);
313
+ const run = this.db.runs.getRunByThreadId(threadId);
530
314
  if (!run)
531
315
  return;
532
316
  if (!this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
@@ -535,7 +319,7 @@ export class RunOrchestrator {
535
319
  }
536
320
  const turnId = typeof notification.params.turnId === "string" ? notification.params.turnId : undefined;
537
321
  if (this.config.runner.codex.persistExtendedHistory) {
538
- this.db.saveThreadEvent({
322
+ this.db.runs.saveThreadEvent({
539
323
  runId: run.id,
540
324
  threadId,
541
325
  ...(turnId ? { turnId } : {}),
@@ -563,13 +347,13 @@ export class RunOrchestrator {
563
347
  if (status === "failed") {
564
348
  const nextState = isRequestedChangesRunType(run.runType) ? "escalated" : "failed";
565
349
  const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
566
- this.db.finishRunWithLease(lease, run.id, {
350
+ this.db.issueSessions.finishRunWithLease(lease, run.id, {
567
351
  status: "failed",
568
352
  threadId,
569
353
  ...(completedTurnId ? { turnId: completedTurnId } : {}),
570
354
  failureReason: "Codex reported the turn completed in a failed state",
571
355
  });
572
- this.db.upsertIssueWithLease(lease, {
356
+ this.db.issueSessions.upsertIssueWithLease(lease, {
573
357
  projectId: run.projectId,
574
358
  linearIssueId: run.linearIssueId,
575
359
  activeRunId: null,
@@ -599,158 +383,33 @@ export class RunOrchestrator {
599
383
  this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
600
384
  return;
601
385
  }
602
- // Complete the run
603
- const trackedIssue = this.db.issueToTrackedIssue(issue);
604
- const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
605
- // Determine post-run state based on current PR metadata.
606
- const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
607
- const verifiedRepairError = await this.verifyReactiveRunAdvancedBranch(run, freshIssue);
608
- if (verifiedRepairError) {
609
- const holdState = resolveRecoverablePostRunState(freshIssue) ?? "failed";
610
- this.failRunAndClear(run, verifiedRepairError, holdState);
611
- const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
612
- this.feed?.publish({
613
- level: "warn",
614
- kind: "turn",
615
- issueKey: freshIssue.issueKey,
616
- projectId: run.projectId,
617
- stage: run.runType,
618
- status: "branch_not_advanced",
619
- summary: verifiedRepairError,
620
- });
621
- void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
622
- void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
623
- this.linearSync.clearProgress(run.id);
624
- this.activeThreadId = undefined;
625
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
626
- return;
627
- }
628
- const missingReviewFixHeadError = await this.verifyReviewFixAdvancedHead(run, freshIssue);
629
- if (missingReviewFixHeadError) {
630
- this.failRunAndClear(run, missingReviewFixHeadError, "escalated");
631
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
632
- this.feed?.publish({
633
- level: "error",
634
- kind: "turn",
635
- issueKey: freshIssue.issueKey,
636
- projectId: run.projectId,
637
- stage: run.runType,
638
- status: "same_head_review_handoff_blocked",
639
- summary: missingReviewFixHeadError,
640
- });
641
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
642
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
643
- this.linearSync.clearProgress(run.id);
644
- this.activeThreadId = undefined;
645
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
646
- return;
647
- }
648
- const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
649
- if (publishedOutcomeError) {
650
- this.failRunAndClear(run, publishedOutcomeError, "failed");
651
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
652
- this.feed?.publish({
653
- level: "warn",
654
- kind: "turn",
655
- issueKey: freshIssue.issueKey,
656
- projectId: run.projectId,
657
- stage: run.runType,
658
- status: "publish_incomplete",
659
- summary: publishedOutcomeError,
660
- });
661
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
662
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
663
- this.linearSync.clearProgress(run.id);
664
- this.activeThreadId = undefined;
665
- return;
666
- }
667
- const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
668
- const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
669
- const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
670
- const completed = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
671
- this.db.finishRun(run.id, {
672
- status: "completed",
673
- threadId,
674
- ...(completedTurnId ? { turnId: completedTurnId } : {}),
675
- summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
676
- reportJson: JSON.stringify(report),
677
- });
678
- this.db.upsertIssue({
679
- projectId: run.projectId,
680
- linearIssueId: run.linearIssueId,
681
- activeRunId: null,
682
- ...(postRunState ? { factoryState: postRunState } : {}),
683
- pendingRunType: null,
684
- pendingRunContextJson: null,
685
- ...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
686
- ? {
687
- lastGitHubFailureSource: null,
688
- lastGitHubFailureHeadSha: null,
689
- lastGitHubFailureSignature: null,
690
- lastGitHubFailureCheckName: null,
691
- lastGitHubFailureCheckUrl: null,
692
- lastGitHubFailureContextJson: null,
693
- lastGitHubFailureAt: null,
694
- lastQueueIncidentJson: null,
695
- lastAttemptedFailureHeadSha: null,
696
- lastAttemptedFailureSignature: null,
697
- }
698
- : {})),
699
- });
700
- if (postRunFollowUp) {
701
- return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
702
- }
703
- return true;
704
- });
705
- if (!completed) {
706
- this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping completion writes after losing issue-session lease");
707
- this.linearSync.clearProgress(run.id);
708
- this.activeThreadId = undefined;
709
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
710
- return;
711
- }
712
- if (postRunFollowUp) {
713
- this.feed?.publish({
714
- level: "info",
715
- kind: "stage",
716
- issueKey: issue.issueKey,
717
- projectId: run.projectId,
718
- stage: postRunFollowUp.factoryState,
719
- status: "follow_up_queued",
720
- summary: postRunFollowUp.summary,
721
- });
722
- this.enqueueIssue(run.projectId, run.linearIssueId);
723
- }
724
- this.feed?.publish({
725
- level: "info",
726
- kind: "turn",
727
- issueKey: issue.issueKey,
728
- projectId: run.projectId,
729
- stage: run.runType,
730
- status: "completed",
731
- summary: `Turn completed for ${run.runType}`,
732
- detail: summarizeCurrentThread(thread).latestAgentMessage,
386
+ await this.runFinalizer.finalizeCompletedRun({
387
+ source: "notification",
388
+ run,
389
+ issue,
390
+ thread,
391
+ threadId,
392
+ ...(completedTurnId ? { completedTurnId } : {}),
393
+ withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
394
+ releaseLease: (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId),
395
+ failRunAndClear: (targetRun, message, nextState) => this.failRunAndClear(targetRun, message, nextState),
396
+ verifyReactiveRunAdvancedBranch: (targetRun, targetIssue) => this.verifyReactiveRunAdvancedBranch(targetRun, targetIssue),
397
+ verifyReviewFixAdvancedHead: (targetRun, targetIssue) => this.verifyReviewFixAdvancedHead(targetRun, targetIssue),
398
+ verifyPublishedRunOutcome: (targetRun, targetIssue) => this.verifyPublishedRunOutcome(targetRun, targetIssue),
399
+ refreshIssueAfterReactivePublish: (targetRun, targetIssue) => this.refreshIssueAfterReactivePublish(targetRun, targetIssue),
400
+ resolvePostRunFollowUp: (targetRun, targetIssue) => this.resolvePostRunFollowUp(targetRun, targetIssue),
401
+ resolveCompletedRunState,
402
+ resolveRecoverableRunState: resolveRecoverablePostRunState,
403
+ appendWakeEventWithLease: (lease, targetIssue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, runType, context, dedupeScope),
733
404
  });
734
- // Emit Linear completion activity + plan
735
- const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
736
- const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
737
- void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
738
- runType: run.runType,
739
- completionSummary,
740
- postRunState: updatedIssue.factoryState,
741
- ...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
742
- }));
743
- void this.linearSync.syncSession(updatedIssue);
744
- this.linearSync.clearProgress(run.id);
745
405
  this.activeThreadId = undefined;
746
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
747
406
  }
748
407
  // ─── Active status for query ──────────────────────────────────────
749
408
  async getActiveRunStatus(issueKey) {
750
409
  const issue = this.db.getIssueByKey(issueKey);
751
410
  if (!issue?.activeRunId)
752
411
  return undefined;
753
- const run = this.db.getRun(issue.activeRunId);
412
+ const run = this.db.runs.getRunById(issue.activeRunId);
754
413
  if (!run?.threadId)
755
414
  return undefined;
756
415
  const trackedIssue = this.db.issueToTrackedIssue(issue);
@@ -763,7 +422,7 @@ export class RunOrchestrator {
763
422
  }
764
423
  // ─── Reconciliation ───────────────────────────────────────────────
765
424
  async reconcileActiveRuns() {
766
- for (const run of this.db.listRunningRuns()) {
425
+ for (const run of this.db.runs.listRunningRuns()) {
767
426
  await this.reconcileRun(run);
768
427
  }
769
428
  // Preemptively detect stuck merge-queue PRs (conflicts visible on
@@ -821,118 +480,14 @@ export class RunOrchestrator {
821
480
  * escalate; backoff delay not elapsed → skip.
822
481
  */
823
482
  recoverOrEscalate(issue, runType, reason) {
824
- // Re-read issue after the run was cleared (activeRunId is now null)
825
- const fresh = this.db.getIssue(issue.projectId, issue.linearIssueId);
826
- if (!fresh)
827
- return;
828
- if (isRequestedChangesRunType(runType)) {
829
- const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
830
- this.db.clearPendingIssueSessionEventsWithLease(lease);
831
- this.db.upsertIssueWithLease(lease, {
832
- projectId: fresh.projectId,
833
- linearIssueId: fresh.linearIssueId,
834
- pendingRunType: null,
835
- pendingRunContextJson: null,
836
- factoryState: "escalated",
837
- });
838
- return true;
839
- });
840
- if (!updated) {
841
- this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping review-fix recovery escalation after losing issue-session lease");
842
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
843
- return;
844
- }
845
- this.logger.warn({ issueKey: fresh.issueKey, reason }, "Requested-changes run failed before a new head was published — escalating");
846
- this.feed?.publish({
847
- level: "error",
848
- kind: "workflow",
849
- issueKey: fresh.issueKey,
850
- projectId: fresh.projectId,
851
- stage: runType,
852
- status: "escalated",
853
- summary: `Requested-changes run failed before publishing a new head (${reason})`,
854
- });
855
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
856
- return;
857
- }
858
- // If PR already merged, transition to done — no retry needed
859
- if (fresh.prState === "merged") {
860
- const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
861
- this.db.upsertIssueWithLease(lease, {
862
- projectId: fresh.projectId,
863
- linearIssueId: fresh.linearIssueId,
864
- factoryState: "done",
865
- zombieRecoveryAttempts: 0,
866
- lastZombieRecoveryAt: null,
867
- });
868
- return true;
869
- });
870
- if (!updated) {
871
- this.logger.warn({ issueKey: fresh.issueKey, reason }, "Skipping merged recovery completion after losing issue-session lease");
872
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
873
- return;
874
- }
875
- this.logger.info({ issueKey: fresh.issueKey, reason }, "Recovery: PR already merged — transitioning to done");
876
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
877
- return;
878
- }
879
- // Budget check
880
- const attempts = fresh.zombieRecoveryAttempts + 1;
881
- if (attempts > DEFAULT_ZOMBIE_RECOVERY_BUDGET) {
882
- const updated = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
883
- this.db.upsertIssueWithLease(lease, {
884
- projectId: fresh.projectId,
885
- linearIssueId: fresh.linearIssueId,
886
- factoryState: "escalated",
887
- });
888
- return true;
889
- });
890
- if (!updated) {
891
- this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery escalation after losing issue-session lease");
892
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
893
- return;
894
- }
895
- this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: budget exhausted — escalating");
896
- this.feed?.publish({
897
- level: "error",
898
- kind: "workflow",
899
- issueKey: fresh.issueKey,
900
- projectId: fresh.projectId,
901
- stage: "escalated",
902
- status: "budget_exhausted",
903
- summary: `${reason} recovery failed after ${DEFAULT_ZOMBIE_RECOVERY_BUDGET} attempts`,
904
- });
905
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
906
- return;
907
- }
908
- // Exponential backoff — skip if delay hasn't elapsed
909
- if (fresh.lastZombieRecoveryAt) {
910
- const elapsed = Date.now() - new Date(fresh.lastZombieRecoveryAt).getTime();
911
- const delay = ZOMBIE_RECOVERY_BASE_DELAY_MS * Math.pow(2, fresh.zombieRecoveryAttempts);
912
- if (elapsed < delay) {
913
- this.logger.debug({ issueKey: fresh.issueKey, attempts: fresh.zombieRecoveryAttempts, delay, elapsed }, "Recovery: backoff not elapsed, skipping");
914
- return;
915
- }
916
- }
917
- // Re-enqueue with backoff tracking
918
- const requeued = this.withHeldIssueSessionLease(fresh.projectId, fresh.linearIssueId, (lease) => {
919
- this.db.upsertIssueWithLease(lease, {
920
- projectId: fresh.projectId,
921
- linearIssueId: fresh.linearIssueId,
922
- pendingRunType: null,
923
- pendingRunContextJson: null,
924
- zombieRecoveryAttempts: attempts,
925
- lastZombieRecoveryAt: new Date().toISOString(),
926
- });
927
- return this.appendWakeEventWithLease(lease, fresh, runType, undefined, `recovery:${attempts}`);
483
+ this.runRecovery.recoverOrEscalate({
484
+ issue,
485
+ runType,
486
+ reason,
487
+ isRequestedChangesRunType,
488
+ withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
489
+ appendWakeEventWithLease: (lease, targetIssue, pendingRunType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, pendingRunType, context, dedupeScope),
928
490
  });
929
- if (!requeued) {
930
- this.logger.warn({ issueKey: fresh.issueKey, attempts, reason }, "Skipping recovery re-enqueue after losing issue-session lease");
931
- this.releaseIssueSessionLease(fresh.projectId, fresh.linearIssueId);
932
- return;
933
- }
934
- this.enqueueIssue(fresh.projectId, fresh.linearIssueId);
935
- this.logger.info({ issueKey: fresh.issueKey, attempts, reason }, "Recovery: re-enqueued with backoff");
936
491
  }
937
492
  async reconcileRun(run) {
938
493
  const issue = this.db.getIssue(run.projectId, run.linearIssueId);
@@ -949,7 +504,7 @@ export class RunOrchestrator {
949
504
  // (e.g. pr_merged processed, DB manually edited), just release the run.
950
505
  if (TERMINAL_STATES.has(issue.factoryState)) {
951
506
  this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
952
- this.db.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
507
+ this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
953
508
  this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
954
509
  });
955
510
  this.logger.info({ issueKey: issue.issueKey, runId: run.id, factoryState: issue.factoryState }, "Reconciliation: released run on terminal issue");
@@ -962,7 +517,7 @@ export class RunOrchestrator {
962
517
  if (!run.threadId) {
963
518
  this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
964
519
  this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
965
- this.db.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
520
+ this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
966
521
  this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
967
522
  });
968
523
  this.recoverOrEscalate(issue, run.runType, "zombie");
@@ -980,7 +535,7 @@ export class RunOrchestrator {
980
535
  catch {
981
536
  this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
982
537
  this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
983
- this.db.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
538
+ this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
984
539
  this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
985
540
  });
986
541
  this.recoverOrEscalate(issue, run.runType, "stale_thread");
@@ -998,7 +553,7 @@ export class RunOrchestrator {
998
553
  const stopState = resolveAuthoritativeLinearStopState(linearIssue);
999
554
  if (stopState?.isFinal) {
1000
555
  this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
1001
- this.db.finishRun(run.id, { status: "released" });
556
+ this.db.runs.finishRun(run.id, { status: "released" });
1002
557
  this.db.upsertIssue({
1003
558
  projectId: run.projectId,
1004
559
  linearIssueId: run.linearIssueId,
@@ -1032,21 +587,21 @@ export class RunOrchestrator {
1032
587
  // Interrupted runs are not real failures — undo the budget increment.
1033
588
  const repairedCounters = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
1034
589
  if (run.runType === "ci_repair" && issue.ciRepairAttempts > 0) {
1035
- this.db.upsertIssueWithLease(lease, {
590
+ this.db.issueSessions.upsertIssueWithLease(lease, {
1036
591
  projectId: issue.projectId,
1037
592
  linearIssueId: issue.linearIssueId,
1038
593
  ciRepairAttempts: issue.ciRepairAttempts - 1,
1039
594
  });
1040
595
  }
1041
596
  else if (run.runType === "queue_repair" && issue.queueRepairAttempts > 0) {
1042
- this.db.upsertIssueWithLease(lease, {
597
+ this.db.issueSessions.upsertIssueWithLease(lease, {
1043
598
  projectId: issue.projectId,
1044
599
  linearIssueId: issue.linearIssueId,
1045
600
  queueRepairAttempts: issue.queueRepairAttempts - 1,
1046
601
  });
1047
602
  }
1048
603
  if (run.runType === "ci_repair" || run.runType === "queue_repair") {
1049
- this.db.upsertIssueWithLease(lease, {
604
+ this.db.issueSessions.upsertIssueWithLease(lease, {
1050
605
  projectId: issue.projectId,
1051
606
  linearIssueId: issue.linearIssueId,
1052
607
  lastAttemptedFailureHeadSha: null,
@@ -1138,140 +693,25 @@ export class RunOrchestrator {
1138
693
  }
1139
694
  // Handle completed turn discovered during reconciliation
1140
695
  if (latestTurn?.status === "completed") {
1141
- const trackedIssue = this.db.issueToTrackedIssue(issue);
1142
- const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.listThreadEvents(run.id)));
1143
- const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
1144
- const verifiedRepairError = await this.verifyReactiveRunAdvancedBranch(run, freshIssue);
1145
- if (verifiedRepairError) {
1146
- const holdState = resolveRecoverablePostRunState(freshIssue) ?? "failed";
1147
- this.failRunAndClear(run, verifiedRepairError, holdState);
1148
- this.feed?.publish({
1149
- level: "warn",
1150
- kind: "turn",
1151
- issueKey: issue.issueKey,
1152
- projectId: run.projectId,
1153
- stage: run.runType,
1154
- status: "branch_not_advanced",
1155
- summary: verifiedRepairError,
1156
- });
1157
- const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
1158
- void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
1159
- void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
1160
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1161
- return;
1162
- }
1163
- const missingReviewFixHeadError = await this.verifyReviewFixAdvancedHead(run, freshIssue);
1164
- if (missingReviewFixHeadError) {
1165
- this.failRunAndClear(run, missingReviewFixHeadError, "escalated");
1166
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
1167
- this.feed?.publish({
1168
- level: "error",
1169
- kind: "turn",
1170
- issueKey: freshIssue.issueKey,
1171
- projectId: run.projectId,
1172
- stage: run.runType,
1173
- status: "same_head_review_handoff_blocked",
1174
- summary: missingReviewFixHeadError,
1175
- });
1176
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
1177
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
1178
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1179
- return;
1180
- }
1181
- const publishedOutcomeError = await this.verifyPublishedRunOutcome(run, freshIssue);
1182
- if (publishedOutcomeError) {
1183
- this.failRunAndClear(run, publishedOutcomeError, "failed");
1184
- this.feed?.publish({
1185
- level: "warn",
1186
- kind: "turn",
1187
- issueKey: issue.issueKey,
1188
- projectId: run.projectId,
1189
- stage: run.runType,
1190
- status: "publish_incomplete",
1191
- summary: publishedOutcomeError,
1192
- });
1193
- const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
1194
- void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
1195
- void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
1196
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1197
- return;
1198
- }
1199
- const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, freshIssue);
1200
- const postRunFollowUp = await this.resolvePostRunFollowUp(run, refreshedIssue);
1201
- const postRunState = postRunFollowUp?.factoryState ?? resolveCompletedRunState(refreshedIssue, run);
1202
- const reconciled = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
1203
- this.db.finishRun(run.id, {
1204
- status: "completed",
1205
- ...(run.threadId ? { threadId: run.threadId } : {}),
1206
- ...(latestTurn.id ? { turnId: latestTurn.id } : {}),
1207
- summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
1208
- reportJson: JSON.stringify(report),
1209
- });
1210
- this.db.upsertIssue({
1211
- projectId: run.projectId,
1212
- linearIssueId: run.linearIssueId,
1213
- activeRunId: null,
1214
- ...(postRunState ? { factoryState: postRunState } : {}),
1215
- pendingRunType: null,
1216
- pendingRunContextJson: null,
1217
- ...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
1218
- ? {
1219
- lastGitHubFailureSource: null,
1220
- lastGitHubFailureHeadSha: null,
1221
- lastGitHubFailureSignature: null,
1222
- lastGitHubFailureCheckName: null,
1223
- lastGitHubFailureCheckUrl: null,
1224
- lastGitHubFailureContextJson: null,
1225
- lastGitHubFailureAt: null,
1226
- lastQueueIncidentJson: null,
1227
- lastAttemptedFailureHeadSha: null,
1228
- lastAttemptedFailureSignature: null,
1229
- }
1230
- : {})),
1231
- });
1232
- if (postRunFollowUp) {
1233
- return this.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
1234
- }
1235
- return true;
696
+ await this.runFinalizer.finalizeCompletedRun({
697
+ source: "reconciliation",
698
+ run,
699
+ issue,
700
+ thread,
701
+ threadId: run.threadId,
702
+ ...(latestTurn.id ? { completedTurnId: latestTurn.id } : {}),
703
+ withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
704
+ releaseLease: (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId),
705
+ failRunAndClear: (targetRun, message, nextState) => this.failRunAndClear(targetRun, message, nextState),
706
+ verifyReactiveRunAdvancedBranch: (targetRun, targetIssue) => this.verifyReactiveRunAdvancedBranch(targetRun, targetIssue),
707
+ verifyReviewFixAdvancedHead: (targetRun, targetIssue) => this.verifyReviewFixAdvancedHead(targetRun, targetIssue),
708
+ verifyPublishedRunOutcome: (targetRun, targetIssue) => this.verifyPublishedRunOutcome(targetRun, targetIssue),
709
+ refreshIssueAfterReactivePublish: (targetRun, targetIssue) => this.refreshIssueAfterReactivePublish(targetRun, targetIssue),
710
+ resolvePostRunFollowUp: (targetRun, targetIssue) => this.resolvePostRunFollowUp(targetRun, targetIssue),
711
+ resolveCompletedRunState,
712
+ resolveRecoverableRunState: resolveRecoverablePostRunState,
713
+ appendWakeEventWithLease: (lease, targetIssue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, runType, context, dedupeScope),
1236
714
  });
1237
- if (!reconciled) {
1238
- this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping reconciled completion writes after losing issue-session lease");
1239
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1240
- return;
1241
- }
1242
- if (postRunFollowUp) {
1243
- this.feed?.publish({
1244
- level: "info",
1245
- kind: "stage",
1246
- issueKey: issue.issueKey,
1247
- projectId: run.projectId,
1248
- stage: postRunFollowUp.factoryState,
1249
- status: "follow_up_queued",
1250
- summary: postRunFollowUp.summary,
1251
- });
1252
- this.enqueueIssue(run.projectId, run.linearIssueId);
1253
- }
1254
- if (postRunState) {
1255
- this.feed?.publish({
1256
- level: "info",
1257
- kind: "turn",
1258
- issueKey: issue.issueKey,
1259
- projectId: run.projectId,
1260
- stage: run.runType,
1261
- status: "completed",
1262
- summary: `Reconciliation: ${run.runType} completed \u2192 ${postRunState}`,
1263
- });
1264
- }
1265
- const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
1266
- const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
1267
- void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
1268
- runType: run.runType,
1269
- completionSummary,
1270
- postRunState: updatedIssue.factoryState,
1271
- ...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
1272
- }));
1273
- void this.linearSync.syncSession(updatedIssue);
1274
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1275
715
  return;
1276
716
  }
1277
717
  if (acquiredRecoveryLease)
@@ -1279,69 +719,21 @@ export class RunOrchestrator {
1279
719
  }
1280
720
  // ─── Internal helpers ─────────────────────────────────────────────
1281
721
  escalate(issue, runType, reason) {
1282
- this.logger.warn({ issueKey: issue.issueKey, runType, reason }, "Escalating to human");
1283
- const escalated = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
1284
- if (issue.activeRunId) {
1285
- this.db.finishRunWithLease(lease, issue.activeRunId, { status: "released" });
1286
- }
1287
- this.db.clearPendingIssueSessionEventsWithLease(lease);
1288
- this.db.upsertIssueWithLease(lease, {
1289
- projectId: issue.projectId,
1290
- linearIssueId: issue.linearIssueId,
1291
- pendingRunType: null,
1292
- pendingRunContextJson: null,
1293
- activeRunId: null,
1294
- factoryState: "escalated",
1295
- });
1296
- return true;
1297
- });
1298
- if (!escalated) {
1299
- this.logger.warn({ issueKey: issue.issueKey, runType }, "Skipping escalation write after losing issue-session lease");
1300
- this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
1301
- return;
1302
- }
1303
- this.feed?.publish({
1304
- level: "error",
1305
- kind: "workflow",
1306
- issueKey: issue.issueKey,
1307
- projectId: issue.projectId,
1308
- stage: runType,
1309
- status: "escalated",
1310
- summary: `Escalated: ${reason}`,
1311
- });
1312
- const escalatedIssue = this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
1313
- void this.linearSync.emitActivity(escalatedIssue, {
1314
- type: "error",
1315
- body: `PatchRelay needs human help to continue.\n\n${reason}`,
722
+ this.runRecovery.escalate({
723
+ issue,
724
+ runType,
725
+ reason,
726
+ withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
1316
727
  });
1317
- void this.linearSync.syncSession(escalatedIssue);
1318
- this.releaseIssueSessionLease(issue.projectId, issue.linearIssueId);
1319
728
  }
1320
729
  failRunAndClear(run, message, nextState = "failed") {
1321
- const updated = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (lease) => {
1322
- this.db.finishRun(run.id, { status: "failed", failureReason: message });
1323
- if (nextState === "failed" || nextState === "escalated" || nextState === "awaiting_input" || nextState === "done") {
1324
- this.db.clearPendingIssueSessionEventsWithLease(lease);
1325
- }
1326
- this.db.upsertIssue({
1327
- projectId: run.projectId,
1328
- linearIssueId: run.linearIssueId,
1329
- activeRunId: null,
1330
- factoryState: nextState,
1331
- });
1332
- const branchOwner = this.resolveBranchOwnerForStateTransition(nextState);
1333
- if (branchOwner) {
1334
- const lease = this.getHeldIssueSessionLease(run.projectId, run.linearIssueId);
1335
- if (lease) {
1336
- this.db.setBranchOwnerWithLease(lease, branchOwner);
1337
- }
1338
- }
1339
- return true;
730
+ this.runRecovery.failRunAndClear({
731
+ run,
732
+ message,
733
+ nextState,
734
+ withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
735
+ getHeldLease: (projectId, linearIssueId) => this.getHeldIssueSessionLease(projectId, linearIssueId),
1340
736
  });
1341
- if (!updated) {
1342
- this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping failure cleanup after losing issue-session lease");
1343
- }
1344
- this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
1345
737
  }
1346
738
  resolveBranchOwnerForStateTransition(newState, pendingRunType) {
1347
739
  return resolveBranchOwnerForStateTransition(newState, pendingRunType);
@@ -1689,20 +1081,11 @@ export class RunOrchestrator {
1689
1081
  }
1690
1082
  throw new Error(`Failed to read thread ${threadId}`);
1691
1083
  }
1692
- issueSessionLeaseKey(projectId, linearIssueId) {
1693
- return `${projectId}:${linearIssueId}`;
1694
- }
1695
1084
  getHeldIssueSessionLease(projectId, linearIssueId) {
1696
- const leaseId = this.activeSessionLeases.get(this.issueSessionLeaseKey(projectId, linearIssueId));
1697
- if (!leaseId)
1698
- return undefined;
1699
- return { projectId, linearIssueId, leaseId };
1085
+ return this.leaseService.getHeldLease(projectId, linearIssueId);
1700
1086
  }
1701
1087
  withHeldIssueSessionLease(projectId, linearIssueId, fn) {
1702
- const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
1703
- if (!lease)
1704
- return undefined;
1705
- return this.db.withIssueSessionLease(projectId, linearIssueId, lease.leaseId, () => fn(lease));
1088
+ return this.leaseService.withHeldLease(projectId, linearIssueId, fn);
1706
1089
  }
1707
1090
  upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
1708
1091
  const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
@@ -1710,7 +1093,7 @@ export class RunOrchestrator {
1710
1093
  this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write without a held issue-session lease");
1711
1094
  return undefined;
1712
1095
  }
1713
- const updated = this.db.upsertIssueWithLease(lease, params);
1096
+ const updated = this.db.issueSessions.upsertIssueWithLease(lease, params);
1714
1097
  if (!updated) {
1715
1098
  this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
1716
1099
  }
@@ -1726,113 +1109,22 @@ export class RunOrchestrator {
1726
1109
  throw error;
1727
1110
  }
1728
1111
  acquireIssueSessionLease(projectId, linearIssueId) {
1729
- const leaseId = randomUUID();
1730
- const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
1731
- const acquired = this.db.acquireIssueSessionLease({
1732
- projectId,
1733
- linearIssueId,
1734
- leaseId,
1735
- workerId: this.workerId,
1736
- leasedUntil,
1737
- });
1738
- if (!acquired)
1739
- return undefined;
1740
- this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
1741
- return leaseId;
1112
+ return this.leaseService.acquire(projectId, linearIssueId);
1742
1113
  }
1743
1114
  forceAcquireIssueSessionLease(projectId, linearIssueId) {
1744
- const leaseId = randomUUID();
1745
- const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
1746
- const acquired = this.db.forceAcquireIssueSessionLease({
1747
- projectId,
1748
- linearIssueId,
1749
- leaseId,
1750
- workerId: this.workerId,
1751
- leasedUntil,
1752
- });
1753
- if (!acquired)
1754
- return undefined;
1755
- this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
1756
- return leaseId;
1115
+ return this.leaseService.forceAcquire(projectId, linearIssueId);
1757
1116
  }
1758
1117
  claimLeaseForReconciliation(projectId, linearIssueId) {
1759
- const key = this.issueSessionLeaseKey(projectId, linearIssueId);
1760
- if (this.activeSessionLeases.has(key)) {
1761
- return "owned";
1762
- }
1763
- const session = this.db.getIssueSession(projectId, linearIssueId);
1764
- if (!session)
1765
- return "skip";
1766
- const leasedUntilMs = session.leasedUntil ? Date.parse(session.leasedUntil) : undefined;
1767
- if (leasedUntilMs !== undefined && Number.isFinite(leasedUntilMs) && leasedUntilMs > Date.now()) {
1768
- return "skip";
1769
- }
1770
- return this.acquireIssueSessionLease(projectId, linearIssueId) ? true : "skip";
1118
+ return this.leaseService.claimForReconciliation(projectId, linearIssueId);
1771
1119
  }
1772
1120
  async reclaimForeignRecoveryLeaseIfSafe(run, issue) {
1773
- const key = this.issueSessionLeaseKey(run.projectId, run.linearIssueId);
1774
- if (this.activeSessionLeases.has(key)) {
1775
- return false;
1776
- }
1777
- const session = this.db.getIssueSession(run.projectId, run.linearIssueId);
1778
- if (!session?.leaseId || !session.workerId || session.workerId === this.workerId) {
1779
- return false;
1780
- }
1781
- if (issue.activeRunId !== run.id) {
1782
- return false;
1783
- }
1784
- let safeToReclaim = !run.threadId;
1785
- if (!safeToReclaim && run.threadId) {
1786
- try {
1787
- const thread = await this.readThreadWithRetry(run.threadId, 1);
1788
- const latestTurn = getThreadTurns(thread).at(-1);
1789
- safeToReclaim = thread.status === "notLoaded"
1790
- || latestTurn?.status === "interrupted"
1791
- || latestTurn?.status === "completed";
1792
- }
1793
- catch {
1794
- safeToReclaim = true;
1795
- }
1796
- }
1797
- if (!safeToReclaim) {
1798
- return false;
1799
- }
1800
- const leaseId = this.forceAcquireIssueSessionLease(run.projectId, run.linearIssueId);
1801
- if (!leaseId) {
1802
- return false;
1803
- }
1804
- this.logger.info({
1805
- issueKey: issue.issueKey,
1806
- runId: run.id,
1807
- previousWorkerId: session.workerId,
1808
- previousLeaseId: session.leaseId,
1809
- reclaimedLeaseId: leaseId,
1810
- }, "Reclaimed foreign issue-session lease for active-run recovery");
1811
- return true;
1121
+ return await this.leaseService.reclaimForeignRecoveryLeaseIfSafe(run, issue);
1812
1122
  }
1813
1123
  heartbeatIssueSessionLease(projectId, linearIssueId) {
1814
- const key = this.issueSessionLeaseKey(projectId, linearIssueId);
1815
- const leaseId = this.activeSessionLeases.get(key) ?? this.db.getIssueSession(projectId, linearIssueId)?.leaseId;
1816
- if (!leaseId)
1817
- return false;
1818
- const renewed = this.db.renewIssueSessionLease({
1819
- projectId,
1820
- linearIssueId,
1821
- leaseId,
1822
- leasedUntil: new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString(),
1823
- });
1824
- if (renewed) {
1825
- this.activeSessionLeases.set(key, leaseId);
1826
- return true;
1827
- }
1828
- this.activeSessionLeases.delete(key);
1829
- return false;
1124
+ return this.leaseService.heartbeat(projectId, linearIssueId);
1830
1125
  }
1831
1126
  releaseIssueSessionLease(projectId, linearIssueId) {
1832
- const key = this.issueSessionLeaseKey(projectId, linearIssueId);
1833
- const leaseId = this.activeSessionLeases.get(key);
1834
- this.db.releaseIssueSessionLease(projectId, linearIssueId, leaseId);
1835
- this.activeSessionLeases.delete(key);
1127
+ this.leaseService.release(projectId, linearIssueId);
1836
1128
  }
1837
1129
  }
1838
1130
  /**