patchrelay 0.36.8 → 0.36.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/cluster-health.js +1 -1
- package/dist/cli/data.js +12 -10
- package/dist/cli/formatters/text.js +3 -1
- package/dist/db/issue-session-store.js +15 -23
- package/dist/db/issue-store.js +559 -0
- package/dist/db/run-store.js +10 -12
- package/dist/db.js +37 -625
- package/dist/github-webhook-handler.js +36 -20
- package/dist/idle-reconciliation.js +26 -15
- package/dist/interrupted-run-recovery.js +176 -0
- package/dist/issue-query-service.js +4 -4
- package/dist/issue-session-projector.js +114 -0
- package/dist/linear-session-sync.js +6 -6
- package/dist/queue-health-monitor.js +3 -3
- package/dist/run-completion-policy.js +412 -0
- package/dist/run-finalizer.js +34 -23
- package/dist/run-launcher.js +5 -5
- package/dist/run-orchestrator.js +46 -684
- package/dist/run-recovery-service.js +26 -18
- package/dist/run-wake-planner.js +1 -1
- package/dist/service.js +9 -9
- package/dist/webhook-handler.js +5 -5
- package/dist/webhooks/agent-session-handler.js +7 -7
- package/dist/webhooks/comment-wake-handler.js +1 -1
- package/dist/webhooks/desired-stage-recorder.js +5 -5
- package/dist/webhooks/issue-removal-handler.js +3 -3
- package/dist/worktree-manager.js +69 -0
- package/dist/zombie-recovery.js +13 -0
- package/package.json +1 -1
package/dist/run-orchestrator.js
CHANGED
|
@@ -1,36 +1,32 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TERMINAL_STATES } from "./factory-state.js";
|
|
2
2
|
import { extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread } from "./run-reporting.js";
|
|
3
3
|
import { buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
4
4
|
import { WorktreeManager } from "./worktree-manager.js";
|
|
5
5
|
import { resolveAuthoritativeLinearStopState, resolvePreferredCompletedLinearState } from "./linear-workflow.js";
|
|
6
|
-
import { execCommand } from "./utils.js";
|
|
7
6
|
import { getThreadTurns } from "./codex-thread-utils.js";
|
|
8
|
-
import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
9
7
|
import { QueueHealthMonitor } from "./queue-health-monitor.js";
|
|
10
|
-
import { resolveImplementationDeliveryMode, } from "./prompting/patchrelay.js";
|
|
11
8
|
import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
|
|
12
9
|
import { LinearSessionSync } from "./linear-session-sync.js";
|
|
13
10
|
import { IssueSessionLeaseService } from "./issue-session-lease-service.js";
|
|
11
|
+
import { InterruptedRunRecovery, resolveRecoverablePostRunState } from "./interrupted-run-recovery.js";
|
|
12
|
+
import { RunCompletionPolicy } from "./run-completion-policy.js";
|
|
14
13
|
import { RunFinalizer } from "./run-finalizer.js";
|
|
15
14
|
import { RunLauncher } from "./run-launcher.js";
|
|
16
15
|
import { RunRecoveryService } from "./run-recovery-service.js";
|
|
17
16
|
import { RunWakePlanner } from "./run-wake-planner.js";
|
|
17
|
+
import { getRemainingZombieRecoveryDelayMs } from "./zombie-recovery.js";
|
|
18
18
|
function lowerCaseFirst(value) {
|
|
19
19
|
return value ? `${value.slice(0, 1).toLowerCase()}${value.slice(1)}` : value;
|
|
20
20
|
}
|
|
21
21
|
function isRequestedChangesRunType(runType) {
|
|
22
22
|
return runType === "review_fix" || runType === "branch_upkeep";
|
|
23
23
|
}
|
|
24
|
-
function
|
|
25
|
-
if (
|
|
26
|
-
return
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
: "address_review_feedback";
|
|
31
|
-
}
|
|
32
|
-
function isBranchUpkeepRequired(context) {
|
|
33
|
-
return context?.branchUpkeepRequired === true;
|
|
24
|
+
function shouldDelayZombieRecoveryLaunch(issue, issueSession, runType) {
|
|
25
|
+
if (issue.zombieRecoveryAttempts <= 0)
|
|
26
|
+
return 0;
|
|
27
|
+
if (issueSession?.lastRunType !== runType)
|
|
28
|
+
return 0;
|
|
29
|
+
return getRemainingZombieRecoveryDelayMs(issue.lastZombieRecoveryAt, issue.zombieRecoveryAttempts);
|
|
34
30
|
}
|
|
35
31
|
export class RunOrchestrator {
|
|
36
32
|
config;
|
|
@@ -52,6 +48,8 @@ export class RunOrchestrator {
|
|
|
52
48
|
runLauncher;
|
|
53
49
|
runRecovery;
|
|
54
50
|
runWakePlanner;
|
|
51
|
+
interruptedRunRecovery;
|
|
52
|
+
runCompletionPolicy;
|
|
55
53
|
activeSessionLeases;
|
|
56
54
|
botIdentity;
|
|
57
55
|
constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
|
|
@@ -66,9 +64,11 @@ export class RunOrchestrator {
|
|
|
66
64
|
this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
|
|
67
65
|
this.leaseService = new IssueSessionLeaseService(db, logger, this.workerId, (threadId, maxRetries) => this.readThreadWithRetry(threadId, maxRetries));
|
|
68
66
|
this.activeSessionLeases = this.leaseService.activeSessionLeases;
|
|
69
|
-
this.
|
|
67
|
+
this.runCompletionPolicy = new RunCompletionPolicy(config, db, logger, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn));
|
|
68
|
+
this.runFinalizer = new RunFinalizer(db, logger, this.linearSync, this.enqueueIssue, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (lease, issue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope), (run, message, nextState) => this.failRunAndClear(run, message, nextState), this.runCompletionPolicy, feed);
|
|
70
69
|
this.runLauncher = new RunLauncher(config, db, codex, logger, this.worktreeManager);
|
|
71
|
-
this.runRecovery = new RunRecoveryService(db, logger, this.linearSync, (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (projectId, issueId) => this.enqueueIssue(projectId, issueId), (newState, pendingRunType) => this.resolveBranchOwnerForStateTransition(newState, pendingRunType), feed);
|
|
70
|
+
this.runRecovery = new RunRecoveryService(db, logger, this.linearSync, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.getHeldIssueSessionLease(projectId, linearIssueId), (lease, issue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (projectId, issueId) => this.enqueueIssue(projectId, issueId), (newState, pendingRunType) => this.resolveBranchOwnerForStateTransition(newState, pendingRunType), feed);
|
|
71
|
+
this.interruptedRunRecovery = new InterruptedRunRecovery(db, logger, this.linearSync, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (run, message, nextState) => this.failRunAndClear(run, message, nextState), (issue) => this.restoreIdleWorktree(issue), this.runCompletionPolicy, (projectId, issueId) => this.enqueueIssue(projectId, issueId), feed);
|
|
72
72
|
this.runWakePlanner = new RunWakePlanner(db);
|
|
73
73
|
this.idleReconciler = new IdleIssueReconciler(db, config, {
|
|
74
74
|
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
@@ -95,7 +95,7 @@ export class RunOrchestrator {
|
|
|
95
95
|
if (this.leaseService.hasLocalLease(item.projectId, item.issueId)) {
|
|
96
96
|
return;
|
|
97
97
|
}
|
|
98
|
-
const issue = this.db.getIssue(item.projectId, item.issueId);
|
|
98
|
+
const issue = this.db.issues.getIssue(item.projectId, item.issueId);
|
|
99
99
|
if (!issue || issue.activeRunId !== undefined)
|
|
100
100
|
return;
|
|
101
101
|
const issueSession = this.db.issueSessions.getIssueSession(item.projectId, item.issueId);
|
|
@@ -116,8 +116,14 @@ export class RunOrchestrator {
|
|
|
116
116
|
return;
|
|
117
117
|
}
|
|
118
118
|
const { runType, context, resumeThread } = wake;
|
|
119
|
+
const remainingZombieDelayMs = shouldDelayZombieRecoveryLaunch(issue, issueSession, runType);
|
|
120
|
+
if (remainingZombieDelayMs > 0) {
|
|
121
|
+
this.logger.debug({ issueKey: issue.issueKey, runType, remainingZombieDelayMs }, "Deferring recovered run launch until zombie backoff elapses");
|
|
122
|
+
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
119
125
|
const effectiveContext = isRequestedChangesRunType(runType)
|
|
120
|
-
? await this.resolveRequestedChangesWakeContext(issue, runType, context
|
|
126
|
+
? await this.runCompletionPolicy.resolveRequestedChangesWakeContext(issue, runType, context)
|
|
121
127
|
: context;
|
|
122
128
|
const sourceHeadSha = typeof effectiveContext?.failureHeadSha === "string"
|
|
123
129
|
? effectiveContext.failureHeadSha
|
|
@@ -179,8 +185,6 @@ export class RunOrchestrator {
|
|
|
179
185
|
leaseId,
|
|
180
186
|
...(this.botIdentity ? { botIdentity: this.botIdentity } : {}),
|
|
181
187
|
assertLaunchLease: (targetRun, phase) => this.assertLaunchLease(targetRun, phase),
|
|
182
|
-
resetWorktreeToTrackedBranch: (targetWorktreePath, targetBranchName, targetIssue) => this.resetWorktreeToTrackedBranch(targetWorktreePath, targetBranchName, targetIssue),
|
|
183
|
-
freshenWorktree: (targetWorktreePath, targetProject, targetIssue) => this.freshenWorktree(targetWorktreePath, targetProject, targetIssue),
|
|
184
188
|
linearSync: this.linearSync,
|
|
185
189
|
releaseLease: (projectId, issueId) => this.releaseIssueSessionLease(projectId, issueId),
|
|
186
190
|
isRequestedChangesRunType,
|
|
@@ -203,98 +207,15 @@ export class RunOrchestrator {
|
|
|
203
207
|
}
|
|
204
208
|
this.logger.info({ issueKey: issue.issueKey, runType, threadId, turnId }, `Started ${runType} run`);
|
|
205
209
|
// Emit Linear activity + plan
|
|
206
|
-
const freshIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
210
|
+
const freshIssue = this.db.issues.getIssue(item.projectId, item.issueId) ?? issue;
|
|
207
211
|
void this.linearSync.emitActivity(freshIssue, buildRunStartedActivity(runType));
|
|
208
212
|
void this.linearSync.syncSession(freshIssue, { activeRunType: runType });
|
|
209
213
|
}
|
|
210
|
-
// ─── Pre-run branch freshening ────────────────────────────────────
|
|
211
|
-
/**
|
|
212
|
-
* Fetch origin and rebase the worktree onto the latest base branch.
|
|
213
|
-
*
|
|
214
|
-
* Risks mitigated:
|
|
215
|
-
* - Dirty worktree from interrupted run → stash before, pop after
|
|
216
|
-
* - Conflicts → abort rebase, throw so the run fails with a clear reason
|
|
217
|
-
* - Already up-to-date → no-op
|
|
218
|
-
* - Keep publishing explicit: the orchestrator updates the local worktree
|
|
219
|
-
* only; the agent/run owns any later branch push.
|
|
220
|
-
*/
|
|
221
|
-
async freshenWorktree(worktreePath, project, issue) {
|
|
222
|
-
const gitBin = this.config.runner.gitBin;
|
|
223
|
-
const baseBranch = project.github?.baseBranch ?? "main";
|
|
224
|
-
// Stash any uncommitted changes from a previous interrupted run
|
|
225
|
-
const stashResult = await execCommand(gitBin, ["-C", worktreePath, "stash"], { timeoutMs: 30_000 });
|
|
226
|
-
const didStash = stashResult.exitCode === 0 && !stashResult.stdout?.includes("No local changes");
|
|
227
|
-
// Fetch latest base
|
|
228
|
-
const fetchResult = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", baseBranch], { timeoutMs: 60_000 });
|
|
229
|
-
if (fetchResult.exitCode !== 0) {
|
|
230
|
-
this.logger.warn({ issueKey: issue.issueKey, stderr: fetchResult.stderr?.slice(0, 300) }, "Pre-run fetch failed, proceeding with current base");
|
|
231
|
-
if (didStash)
|
|
232
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
233
|
-
return;
|
|
234
|
-
}
|
|
235
|
-
// Check if rebase is needed: is HEAD already on top of origin/baseBranch?
|
|
236
|
-
const mergeBaseResult = await execCommand(gitBin, ["-C", worktreePath, "merge-base", "--is-ancestor", `origin/${baseBranch}`, "HEAD"], { timeoutMs: 10_000 });
|
|
237
|
-
if (mergeBaseResult.exitCode === 0) {
|
|
238
|
-
// Already up-to-date — no rebase needed
|
|
239
|
-
this.logger.debug({ issueKey: issue.issueKey }, "Pre-run freshen: branch already up to date");
|
|
240
|
-
if (didStash)
|
|
241
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
// Rebase onto latest base
|
|
245
|
-
const rebaseResult = await execCommand(gitBin, ["-C", worktreePath, "rebase", `origin/${baseBranch}`], { timeoutMs: 120_000 });
|
|
246
|
-
if (rebaseResult.exitCode !== 0) {
|
|
247
|
-
// Abort the failed rebase and restore state — then let the agent run
|
|
248
|
-
// proceed. The agent can resolve the conflict itself (the workflow
|
|
249
|
-
// prompt tells it to rebase and handle conflicts).
|
|
250
|
-
await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
|
|
251
|
-
if (didStash)
|
|
252
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
253
|
-
this.logger.warn({ issueKey: issue.issueKey, baseBranch }, "Pre-run freshen: rebase conflict, agent will resolve");
|
|
254
|
-
return;
|
|
255
|
-
}
|
|
256
|
-
this.logger.info({ issueKey: issue.issueKey, baseBranch }, "Pre-run freshen: rebased locally onto latest base");
|
|
257
|
-
// Restore stashed changes
|
|
258
|
-
if (didStash)
|
|
259
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
260
|
-
}
|
|
261
214
|
async resetWorktreeToTrackedBranch(worktreePath, branchName, issue) {
|
|
262
|
-
|
|
263
|
-
const branchFetch = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", branchName], { timeoutMs: 60_000 });
|
|
264
|
-
const hasRemoteBranch = branchFetch.exitCode === 0;
|
|
265
|
-
await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
|
|
266
|
-
await execCommand(gitBin, ["-C", worktreePath, "merge", "--abort"], { timeoutMs: 10_000 });
|
|
267
|
-
await execCommand(gitBin, ["-C", worktreePath, "cherry-pick", "--abort"], { timeoutMs: 10_000 });
|
|
268
|
-
await execCommand(gitBin, ["-C", worktreePath, "am", "--abort"], { timeoutMs: 10_000 });
|
|
269
|
-
await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", "HEAD"], { timeoutMs: 30_000 });
|
|
270
|
-
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
271
|
-
const checkoutTarget = hasRemoteBranch ? `origin/${branchName}` : branchName;
|
|
272
|
-
const checkoutResult = await execCommand(gitBin, ["-C", worktreePath, "checkout", "-B", branchName, checkoutTarget], { timeoutMs: 30_000 });
|
|
273
|
-
if (checkoutResult.exitCode !== 0) {
|
|
274
|
-
throw new Error(`Failed to restore ${branchName} worktree state: ${checkoutResult.stderr?.slice(0, 300) ?? "git checkout failed"}`);
|
|
275
|
-
}
|
|
276
|
-
const resetTarget = hasRemoteBranch ? `origin/${branchName}` : "HEAD";
|
|
277
|
-
const resetResult = await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", resetTarget], { timeoutMs: 30_000 });
|
|
278
|
-
if (resetResult.exitCode !== 0) {
|
|
279
|
-
throw new Error(`Failed to reset ${branchName} worktree state: ${resetResult.stderr?.slice(0, 300) ?? "git reset failed"}`);
|
|
280
|
-
}
|
|
281
|
-
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
282
|
-
this.logger.debug({ issueKey: issue.issueKey, branchName, hasRemoteBranch }, "Reset issue worktree to tracked branch state");
|
|
215
|
+
await this.worktreeManager.resetWorktreeToTrackedBranch(worktreePath, branchName, issue, this.logger);
|
|
283
216
|
}
|
|
284
217
|
async restoreIdleWorktree(issue) {
|
|
285
|
-
|
|
286
|
-
return;
|
|
287
|
-
try {
|
|
288
|
-
await this.resetWorktreeToTrackedBranch(issue.worktreePath, issue.branchName, issue);
|
|
289
|
-
}
|
|
290
|
-
catch (error) {
|
|
291
|
-
this.logger.warn({
|
|
292
|
-
issueKey: issue.issueKey,
|
|
293
|
-
branchName: issue.branchName,
|
|
294
|
-
worktreePath: issue.worktreePath,
|
|
295
|
-
error: error instanceof Error ? error.message : String(error),
|
|
296
|
-
}, "Failed to restore idle worktree after interrupted run");
|
|
297
|
-
}
|
|
218
|
+
await this.worktreeManager.restoreIdleWorktree(issue, this.logger);
|
|
298
219
|
}
|
|
299
220
|
// ─── Notification handler ─────────────────────────────────────────
|
|
300
221
|
async handleCodexNotification(notification) {
|
|
@@ -331,7 +252,7 @@ export class RunOrchestrator {
|
|
|
331
252
|
this.linearSync.maybeEmitProgress(notification, run);
|
|
332
253
|
// Sync codex plan to Linear session when it updates
|
|
333
254
|
if (notification.method === "turn/plan/updated") {
|
|
334
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
255
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
335
256
|
if (issue) {
|
|
336
257
|
void this.linearSync.syncCodexPlan(issue, notification.params);
|
|
337
258
|
}
|
|
@@ -339,7 +260,7 @@ export class RunOrchestrator {
|
|
|
339
260
|
if (notification.method !== "turn/completed")
|
|
340
261
|
return;
|
|
341
262
|
const thread = await this.readThreadWithRetry(threadId);
|
|
342
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
263
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
343
264
|
if (!issue)
|
|
344
265
|
return;
|
|
345
266
|
const completedTurnId = extractTurnId(notification.params);
|
|
@@ -375,7 +296,7 @@ export class RunOrchestrator {
|
|
|
375
296
|
status: "failed",
|
|
376
297
|
summary: `Turn failed for ${run.runType}`,
|
|
377
298
|
});
|
|
378
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
299
|
+
const failedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
379
300
|
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType));
|
|
380
301
|
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
381
302
|
this.linearSync.clearProgress(run.id);
|
|
@@ -390,23 +311,13 @@ export class RunOrchestrator {
|
|
|
390
311
|
thread,
|
|
391
312
|
threadId,
|
|
392
313
|
...(completedTurnId ? { completedTurnId } : {}),
|
|
393
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
394
|
-
releaseLease: (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId),
|
|
395
|
-
failRunAndClear: (targetRun, message, nextState) => this.failRunAndClear(targetRun, message, nextState),
|
|
396
|
-
verifyReactiveRunAdvancedBranch: (targetRun, targetIssue) => this.verifyReactiveRunAdvancedBranch(targetRun, targetIssue),
|
|
397
|
-
verifyReviewFixAdvancedHead: (targetRun, targetIssue) => this.verifyReviewFixAdvancedHead(targetRun, targetIssue),
|
|
398
|
-
verifyPublishedRunOutcome: (targetRun, targetIssue) => this.verifyPublishedRunOutcome(targetRun, targetIssue),
|
|
399
|
-
refreshIssueAfterReactivePublish: (targetRun, targetIssue) => this.refreshIssueAfterReactivePublish(targetRun, targetIssue),
|
|
400
|
-
resolvePostRunFollowUp: (targetRun, targetIssue) => this.resolvePostRunFollowUp(targetRun, targetIssue),
|
|
401
|
-
resolveCompletedRunState,
|
|
402
314
|
resolveRecoverableRunState: resolveRecoverablePostRunState,
|
|
403
|
-
appendWakeEventWithLease: (lease, targetIssue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, runType, context, dedupeScope),
|
|
404
315
|
});
|
|
405
316
|
this.activeThreadId = undefined;
|
|
406
317
|
}
|
|
407
318
|
// ─── Active status for query ──────────────────────────────────────
|
|
408
319
|
async getActiveRunStatus(issueKey) {
|
|
409
|
-
const issue = this.db.getIssueByKey(issueKey);
|
|
320
|
+
const issue = this.db.issues.getIssueByKey(issueKey);
|
|
410
321
|
if (!issue?.activeRunId)
|
|
411
322
|
return undefined;
|
|
412
323
|
const run = this.db.runs.getRunById(issue.activeRunId);
|
|
@@ -434,7 +345,7 @@ export class RunOrchestrator {
|
|
|
434
345
|
await this.reconcileMergedLinearCompletion();
|
|
435
346
|
}
|
|
436
347
|
async reconcileMergedLinearCompletion() {
|
|
437
|
-
for (const issue of this.db.listIssues()) {
|
|
348
|
+
for (const issue of this.db.issues.listIssues()) {
|
|
438
349
|
if (issue.prState !== "merged")
|
|
439
350
|
continue;
|
|
440
351
|
if (issue.currentLinearStateType?.trim().toLowerCase() === "completed")
|
|
@@ -449,7 +360,7 @@ export class RunOrchestrator {
|
|
|
449
360
|
continue;
|
|
450
361
|
const normalizedCurrent = liveIssue.stateName?.trim().toLowerCase();
|
|
451
362
|
if (normalizedCurrent === targetState.trim().toLowerCase()) {
|
|
452
|
-
this.db.upsertIssue({
|
|
363
|
+
this.db.issues.upsertIssue({
|
|
453
364
|
projectId: issue.projectId,
|
|
454
365
|
linearIssueId: issue.linearIssueId,
|
|
455
366
|
...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
|
|
@@ -458,7 +369,7 @@ export class RunOrchestrator {
|
|
|
458
369
|
continue;
|
|
459
370
|
}
|
|
460
371
|
const updated = await linear.setIssueState(issue.linearIssueId, targetState);
|
|
461
|
-
this.db.upsertIssue({
|
|
372
|
+
this.db.issues.upsertIssue({
|
|
462
373
|
projectId: issue.projectId,
|
|
463
374
|
linearIssueId: issue.linearIssueId,
|
|
464
375
|
...(updated.stateName ? { currentLinearState: updated.stateName } : {}),
|
|
@@ -485,12 +396,10 @@ export class RunOrchestrator {
|
|
|
485
396
|
runType,
|
|
486
397
|
reason,
|
|
487
398
|
isRequestedChangesRunType,
|
|
488
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
489
|
-
appendWakeEventWithLease: (lease, targetIssue, pendingRunType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, pendingRunType, context, dedupeScope),
|
|
490
399
|
});
|
|
491
400
|
}
|
|
492
401
|
async reconcileRun(run) {
|
|
493
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
402
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
494
403
|
if (!issue)
|
|
495
404
|
return;
|
|
496
405
|
let recoveryLease = this.claimLeaseForReconciliation(run.projectId, run.linearIssueId);
|
|
@@ -505,10 +414,10 @@ export class RunOrchestrator {
|
|
|
505
414
|
if (TERMINAL_STATES.has(issue.factoryState)) {
|
|
506
415
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
507
416
|
this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
508
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
417
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
509
418
|
});
|
|
510
419
|
this.logger.info({ issueKey: issue.issueKey, runId: run.id, factoryState: issue.factoryState }, "Reconciliation: released run on terminal issue");
|
|
511
|
-
const releasedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
420
|
+
const releasedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
512
421
|
void this.linearSync.syncSession(releasedIssue, { activeRunType: run.runType });
|
|
513
422
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
514
423
|
return;
|
|
@@ -518,10 +427,10 @@ export class RunOrchestrator {
|
|
|
518
427
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
|
|
519
428
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
520
429
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
521
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
430
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
522
431
|
});
|
|
523
432
|
this.recoverOrEscalate(issue, run.runType, "zombie");
|
|
524
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
433
|
+
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
525
434
|
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "The Codex turn never started before PatchRelay restarted."));
|
|
526
435
|
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
527
436
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
@@ -536,10 +445,10 @@ export class RunOrchestrator {
|
|
|
536
445
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
|
|
537
446
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
538
447
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
539
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
448
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
540
449
|
});
|
|
541
450
|
this.recoverOrEscalate(issue, run.runType, "stale_thread");
|
|
542
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
451
|
+
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
543
452
|
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "PatchRelay lost the active Codex thread after restart and needs to recover."));
|
|
544
453
|
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
545
454
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
@@ -554,7 +463,7 @@ export class RunOrchestrator {
|
|
|
554
463
|
if (stopState?.isFinal) {
|
|
555
464
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
556
465
|
this.db.runs.finishRun(run.id, { status: "released" });
|
|
557
|
-
this.db.upsertIssue({
|
|
466
|
+
this.db.issues.upsertIssue({
|
|
558
467
|
projectId: run.projectId,
|
|
559
468
|
linearIssueId: run.linearIssueId,
|
|
560
469
|
activeRunId: null,
|
|
@@ -571,7 +480,7 @@ export class RunOrchestrator {
|
|
|
571
480
|
status: "reconciled",
|
|
572
481
|
summary: `Linear state ${stopState.stateName} \u2192 done`,
|
|
573
482
|
});
|
|
574
|
-
const doneIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
483
|
+
const doneIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
575
484
|
void this.linearSync.syncSession(doneIssue, { activeRunType: run.runType });
|
|
576
485
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
577
486
|
return;
|
|
@@ -583,112 +492,7 @@ export class RunOrchestrator {
|
|
|
583
492
|
// The agent may have partially completed work (commits, PR) before interruption.
|
|
584
493
|
// Reactive loops (CI repair, review fix) will handle follow-up if needed.
|
|
585
494
|
if (latestTurn?.status === "interrupted") {
|
|
586
|
-
this.
|
|
587
|
-
// Interrupted runs are not real failures — undo the budget increment.
|
|
588
|
-
const repairedCounters = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
589
|
-
if (run.runType === "ci_repair" && issue.ciRepairAttempts > 0) {
|
|
590
|
-
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
591
|
-
projectId: issue.projectId,
|
|
592
|
-
linearIssueId: issue.linearIssueId,
|
|
593
|
-
ciRepairAttempts: issue.ciRepairAttempts - 1,
|
|
594
|
-
});
|
|
595
|
-
}
|
|
596
|
-
else if (run.runType === "queue_repair" && issue.queueRepairAttempts > 0) {
|
|
597
|
-
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
598
|
-
projectId: issue.projectId,
|
|
599
|
-
linearIssueId: issue.linearIssueId,
|
|
600
|
-
queueRepairAttempts: issue.queueRepairAttempts - 1,
|
|
601
|
-
});
|
|
602
|
-
}
|
|
603
|
-
if (run.runType === "ci_repair" || run.runType === "queue_repair") {
|
|
604
|
-
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
605
|
-
projectId: issue.projectId,
|
|
606
|
-
linearIssueId: issue.linearIssueId,
|
|
607
|
-
lastAttemptedFailureHeadSha: null,
|
|
608
|
-
lastAttemptedFailureSignature: null,
|
|
609
|
-
});
|
|
610
|
-
}
|
|
611
|
-
return true;
|
|
612
|
-
});
|
|
613
|
-
if (!repairedCounters) {
|
|
614
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping interrupted-run recovery after losing issue-session lease");
|
|
615
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
616
|
-
return;
|
|
617
|
-
}
|
|
618
|
-
if (isRequestedChangesRunType(run.runType)) {
|
|
619
|
-
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
|
|
620
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
621
|
-
const retryContext = project
|
|
622
|
-
? await this.resolveRequestedChangesWakeContext(refreshedIssue, run.runType, run.runType === "branch_upkeep"
|
|
623
|
-
? {
|
|
624
|
-
branchUpkeepRequired: true,
|
|
625
|
-
reviewFixMode: "branch_upkeep",
|
|
626
|
-
wakeReason: "branch_upkeep",
|
|
627
|
-
}
|
|
628
|
-
: undefined, project)
|
|
629
|
-
: undefined;
|
|
630
|
-
const retryRunType = resolveRequestedChangesMode(run.runType, retryContext) === "branch_upkeep"
|
|
631
|
-
? "branch_upkeep"
|
|
632
|
-
: "review_fix";
|
|
633
|
-
const recoveredState = resolveRecoverablePostRunState(refreshedIssue) ?? "failed";
|
|
634
|
-
const interruptedMessage = "Requested-changes run was interrupted before PatchRelay could verify that a new PR head was published";
|
|
635
|
-
this.failRunAndClear(run, interruptedMessage, recoveredState);
|
|
636
|
-
await this.restoreIdleWorktree(issue);
|
|
637
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
638
|
-
if (recoveredState === "changes_requested") {
|
|
639
|
-
this.db.upsertIssue({
|
|
640
|
-
projectId: run.projectId,
|
|
641
|
-
linearIssueId: run.linearIssueId,
|
|
642
|
-
pendingRunType: retryRunType,
|
|
643
|
-
pendingRunContextJson: retryContext ? JSON.stringify(retryContext) : null,
|
|
644
|
-
});
|
|
645
|
-
this.feed?.publish({
|
|
646
|
-
level: "warn",
|
|
647
|
-
kind: "workflow",
|
|
648
|
-
issueKey: issue.issueKey,
|
|
649
|
-
projectId: run.projectId,
|
|
650
|
-
stage: run.runType,
|
|
651
|
-
status: "retry_queued",
|
|
652
|
-
summary: "Requested-changes run was interrupted; PatchRelay will retry from fresh GitHub truth",
|
|
653
|
-
});
|
|
654
|
-
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
655
|
-
}
|
|
656
|
-
else {
|
|
657
|
-
this.feed?.publish({
|
|
658
|
-
level: "error",
|
|
659
|
-
kind: "workflow",
|
|
660
|
-
issueKey: issue.issueKey,
|
|
661
|
-
projectId: run.projectId,
|
|
662
|
-
stage: run.runType,
|
|
663
|
-
status: "escalated",
|
|
664
|
-
summary: interruptedMessage,
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
|
-
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, interruptedMessage));
|
|
668
|
-
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
669
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
670
|
-
return;
|
|
671
|
-
}
|
|
672
|
-
const recoveredState = resolveRecoverablePostRunState(this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
|
|
673
|
-
this.failRunAndClear(run, "Codex turn was interrupted", recoveredState);
|
|
674
|
-
await this.restoreIdleWorktree(issue);
|
|
675
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
676
|
-
if (recoveredState) {
|
|
677
|
-
this.feed?.publish({
|
|
678
|
-
level: "info",
|
|
679
|
-
kind: "stage",
|
|
680
|
-
issueKey: issue.issueKey,
|
|
681
|
-
projectId: run.projectId,
|
|
682
|
-
stage: recoveredState,
|
|
683
|
-
status: "reconciled",
|
|
684
|
-
summary: `Interrupted ${run.runType} recovered \u2192 ${recoveredState}`,
|
|
685
|
-
});
|
|
686
|
-
}
|
|
687
|
-
else {
|
|
688
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, "The Codex turn was interrupted."));
|
|
689
|
-
}
|
|
690
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
691
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
495
|
+
await this.interruptedRunRecovery.handle(run, issue);
|
|
692
496
|
return;
|
|
693
497
|
}
|
|
694
498
|
// Handle completed turn discovered during reconciliation
|
|
@@ -700,17 +504,7 @@ export class RunOrchestrator {
|
|
|
700
504
|
thread,
|
|
701
505
|
threadId: run.threadId,
|
|
702
506
|
...(latestTurn.id ? { completedTurnId: latestTurn.id } : {}),
|
|
703
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
704
|
-
releaseLease: (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId),
|
|
705
|
-
failRunAndClear: (targetRun, message, nextState) => this.failRunAndClear(targetRun, message, nextState),
|
|
706
|
-
verifyReactiveRunAdvancedBranch: (targetRun, targetIssue) => this.verifyReactiveRunAdvancedBranch(targetRun, targetIssue),
|
|
707
|
-
verifyReviewFixAdvancedHead: (targetRun, targetIssue) => this.verifyReviewFixAdvancedHead(targetRun, targetIssue),
|
|
708
|
-
verifyPublishedRunOutcome: (targetRun, targetIssue) => this.verifyPublishedRunOutcome(targetRun, targetIssue),
|
|
709
|
-
refreshIssueAfterReactivePublish: (targetRun, targetIssue) => this.refreshIssueAfterReactivePublish(targetRun, targetIssue),
|
|
710
|
-
resolvePostRunFollowUp: (targetRun, targetIssue) => this.resolvePostRunFollowUp(targetRun, targetIssue),
|
|
711
|
-
resolveCompletedRunState,
|
|
712
507
|
resolveRecoverableRunState: resolveRecoverablePostRunState,
|
|
713
|
-
appendWakeEventWithLease: (lease, targetIssue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, runType, context, dedupeScope),
|
|
714
508
|
});
|
|
715
509
|
return;
|
|
716
510
|
}
|
|
@@ -723,7 +517,6 @@ export class RunOrchestrator {
|
|
|
723
517
|
issue,
|
|
724
518
|
runType,
|
|
725
519
|
reason,
|
|
726
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
727
520
|
});
|
|
728
521
|
}
|
|
729
522
|
failRunAndClear(run, message, nextState = "failed") {
|
|
@@ -731,342 +524,13 @@ export class RunOrchestrator {
|
|
|
731
524
|
run,
|
|
732
525
|
message,
|
|
733
526
|
nextState,
|
|
734
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
735
|
-
getHeldLease: (projectId, linearIssueId) => this.getHeldIssueSessionLease(projectId, linearIssueId),
|
|
736
527
|
});
|
|
737
528
|
}
|
|
738
529
|
resolveBranchOwnerForStateTransition(newState, pendingRunType) {
|
|
739
530
|
return resolveBranchOwnerForStateTransition(newState, pendingRunType);
|
|
740
531
|
}
|
|
741
|
-
async
|
|
742
|
-
|
|
743
|
-
return undefined;
|
|
744
|
-
}
|
|
745
|
-
if (!issue.prNumber || issue.prState !== "open" || !issue.lastGitHubFailureHeadSha) {
|
|
746
|
-
return undefined;
|
|
747
|
-
}
|
|
748
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
749
|
-
if (!project?.github?.repoFullName) {
|
|
750
|
-
return undefined;
|
|
751
|
-
}
|
|
752
|
-
try {
|
|
753
|
-
const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
|
|
754
|
-
if (!pr || pr.state?.toUpperCase() !== "OPEN")
|
|
755
|
-
return undefined;
|
|
756
|
-
if (!pr.headRefOid || pr.headRefOid !== issue.lastGitHubFailureHeadSha)
|
|
757
|
-
return undefined;
|
|
758
|
-
return `Repair finished but PR #${issue.prNumber} is still on failing head ${issue.lastGitHubFailureHeadSha.slice(0, 8)}`;
|
|
759
|
-
}
|
|
760
|
-
catch (error) {
|
|
761
|
-
this.logger.debug({
|
|
762
|
-
issueKey: issue.issueKey,
|
|
763
|
-
prNumber: issue.prNumber,
|
|
764
|
-
error: error instanceof Error ? error.message : String(error),
|
|
765
|
-
}, "Failed to verify PR head advancement after repair");
|
|
766
|
-
return undefined;
|
|
767
|
-
}
|
|
768
|
-
}
|
|
769
|
-
async verifyReviewFixAdvancedHead(run, issue) {
|
|
770
|
-
if (!isRequestedChangesRunType(run.runType)) {
|
|
771
|
-
return undefined;
|
|
772
|
-
}
|
|
773
|
-
if (!issue.prNumber || issue.prState !== "open") {
|
|
774
|
-
return undefined;
|
|
775
|
-
}
|
|
776
|
-
if (!run.sourceHeadSha) {
|
|
777
|
-
return `Requested-changes run finished for PR #${issue.prNumber} without a recorded starting head SHA. PatchRelay cannot verify that a new head was published.`;
|
|
778
|
-
}
|
|
779
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
780
|
-
if (!project?.github?.repoFullName) {
|
|
781
|
-
return undefined;
|
|
782
|
-
}
|
|
783
|
-
try {
|
|
784
|
-
const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
|
|
785
|
-
if (!pr || pr.state?.toUpperCase() !== "OPEN")
|
|
786
|
-
return undefined;
|
|
787
|
-
if (!pr.headRefOid) {
|
|
788
|
-
return `Requested-changes run finished for PR #${issue.prNumber} but GitHub did not report a current head SHA.`;
|
|
789
|
-
}
|
|
790
|
-
if (pr.headRefOid === run.sourceHeadSha) {
|
|
791
|
-
return `Requested-changes run finished for PR #${issue.prNumber} without pushing a new head; PatchRelay must not hand the same SHA back to review.`;
|
|
792
|
-
}
|
|
793
|
-
return undefined;
|
|
794
|
-
}
|
|
795
|
-
catch (error) {
|
|
796
|
-
this.logger.debug({
|
|
797
|
-
issueKey: issue.issueKey,
|
|
798
|
-
prNumber: issue.prNumber,
|
|
799
|
-
error: error instanceof Error ? error.message : String(error),
|
|
800
|
-
}, "Failed to verify PR head advancement after requested-changes work");
|
|
801
|
-
return undefined;
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
|
-
async refreshIssueAfterReactivePublish(run, issue) {
|
|
805
|
-
if (run.runType !== "ci_repair" && run.runType !== "queue_repair" && !isRequestedChangesRunType(run.runType)) {
|
|
806
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
807
|
-
}
|
|
808
|
-
if (!issue.prNumber) {
|
|
809
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
810
|
-
}
|
|
811
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
812
|
-
const repoFullName = project?.github?.repoFullName;
|
|
813
|
-
if (!repoFullName) {
|
|
814
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
815
|
-
}
|
|
816
|
-
try {
|
|
817
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
818
|
-
if (!pr) {
|
|
819
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
820
|
-
}
|
|
821
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
822
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
823
|
-
const gateCheckName = project?.gateChecks?.find((entry) => entry.trim())?.trim() ?? "verify";
|
|
824
|
-
const headAdvanced = Boolean(pr.headRefOid && pr.headRefOid !== issue.lastGitHubFailureHeadSha);
|
|
825
|
-
const reviewFixHeadAdvanced = isRequestedChangesRunType(run.runType)
|
|
826
|
-
&& Boolean(pr.headRefOid && run.sourceHeadSha && pr.headRefOid !== run.sourceHeadSha);
|
|
827
|
-
this.upsertIssueIfLeaseHeld(run.projectId, run.linearIssueId, {
|
|
828
|
-
projectId: run.projectId,
|
|
829
|
-
linearIssueId: run.linearIssueId,
|
|
830
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
831
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
832
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
833
|
-
...((headAdvanced || reviewFixHeadAdvanced)
|
|
834
|
-
? {
|
|
835
|
-
prCheckStatus: "pending",
|
|
836
|
-
lastGitHubFailureSource: null,
|
|
837
|
-
lastGitHubFailureHeadSha: null,
|
|
838
|
-
lastGitHubFailureSignature: null,
|
|
839
|
-
lastGitHubFailureCheckName: null,
|
|
840
|
-
lastGitHubFailureCheckUrl: null,
|
|
841
|
-
lastGitHubFailureContextJson: null,
|
|
842
|
-
lastGitHubFailureAt: null,
|
|
843
|
-
lastQueueIncidentJson: null,
|
|
844
|
-
lastAttemptedFailureHeadSha: null,
|
|
845
|
-
lastAttemptedFailureSignature: null,
|
|
846
|
-
lastGitHubCiSnapshotHeadSha: pr.headRefOid ?? null,
|
|
847
|
-
lastGitHubCiSnapshotGateCheckName: gateCheckName,
|
|
848
|
-
lastGitHubCiSnapshotGateCheckStatus: "pending",
|
|
849
|
-
lastGitHubCiSnapshotJson: null,
|
|
850
|
-
lastGitHubCiSnapshotSettledAt: null,
|
|
851
|
-
}
|
|
852
|
-
: {}),
|
|
853
|
-
}, "reactive publish refresh");
|
|
854
|
-
}
|
|
855
|
-
catch (error) {
|
|
856
|
-
this.logger.debug({
|
|
857
|
-
issueKey: issue.issueKey,
|
|
858
|
-
prNumber: issue.prNumber,
|
|
859
|
-
error: error instanceof Error ? error.message : String(error),
|
|
860
|
-
}, "Failed to refresh PR state after reactive publish");
|
|
861
|
-
}
|
|
862
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
863
|
-
}
|
|
864
|
-
async loadRemotePrState(repoFullName, prNumber) {
|
|
865
|
-
const { stdout, exitCode } = await execCommand("gh", [
|
|
866
|
-
"pr", "view", String(prNumber),
|
|
867
|
-
"--repo", repoFullName,
|
|
868
|
-
"--json", "headRefOid,state,reviewDecision,mergeStateStatus",
|
|
869
|
-
], { timeoutMs: 10_000 });
|
|
870
|
-
if (exitCode !== 0)
|
|
871
|
-
return undefined;
|
|
872
|
-
return JSON.parse(stdout);
|
|
873
|
-
}
|
|
874
|
-
async resolveRequestedChangesWakeContext(issue, runType, context, project) {
|
|
875
|
-
if (runType === "branch_upkeep" || isBranchUpkeepRequired(context)) {
|
|
876
|
-
return context;
|
|
877
|
-
}
|
|
878
|
-
if (!issue.prNumber || issue.prState !== "open" || issue.prReviewState !== "changes_requested") {
|
|
879
|
-
return context;
|
|
880
|
-
}
|
|
881
|
-
const repoFullName = project.github?.repoFullName;
|
|
882
|
-
if (!repoFullName) {
|
|
883
|
-
return context;
|
|
884
|
-
}
|
|
885
|
-
try {
|
|
886
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
887
|
-
if (!pr)
|
|
888
|
-
return context;
|
|
889
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
890
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
891
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
892
|
-
projectId: issue.projectId,
|
|
893
|
-
linearIssueId: issue.linearIssueId,
|
|
894
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
895
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
896
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
897
|
-
}, "review-fix wake refresh");
|
|
898
|
-
if (nextPrState !== "open")
|
|
899
|
-
return context;
|
|
900
|
-
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
901
|
-
return context;
|
|
902
|
-
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
903
|
-
return context;
|
|
904
|
-
return buildReviewFixBranchUpkeepContext(issue.prNumber, project.github?.baseBranch ?? "main", pr, context);
|
|
905
|
-
}
|
|
906
|
-
catch (error) {
|
|
907
|
-
this.logger.debug({
|
|
908
|
-
issueKey: issue.issueKey,
|
|
909
|
-
prNumber: issue.prNumber,
|
|
910
|
-
error: error instanceof Error ? error.message : String(error),
|
|
911
|
-
}, "Failed to resolve requested-changes wake context");
|
|
912
|
-
return context;
|
|
913
|
-
}
|
|
914
|
-
}
|
|
915
|
-
async resolvePostRunFollowUp(run, issue, projectOverride) {
|
|
916
|
-
if (run.runType !== "review_fix") {
|
|
917
|
-
return undefined;
|
|
918
|
-
}
|
|
919
|
-
if (!issue.prNumber || issue.prState !== "open") {
|
|
920
|
-
return undefined;
|
|
921
|
-
}
|
|
922
|
-
if (issue.prReviewState !== "changes_requested") {
|
|
923
|
-
return undefined;
|
|
924
|
-
}
|
|
925
|
-
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
926
|
-
const repoFullName = project?.github?.repoFullName;
|
|
927
|
-
if (!repoFullName) {
|
|
928
|
-
return undefined;
|
|
929
|
-
}
|
|
930
|
-
try {
|
|
931
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
932
|
-
if (!pr)
|
|
933
|
-
return undefined;
|
|
934
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
935
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
936
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
937
|
-
projectId: issue.projectId,
|
|
938
|
-
linearIssueId: issue.linearIssueId,
|
|
939
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
940
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
941
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
942
|
-
}, "post-run follow-up refresh");
|
|
943
|
-
if (nextPrState !== "open")
|
|
944
|
-
return undefined;
|
|
945
|
-
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
946
|
-
return undefined;
|
|
947
|
-
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
948
|
-
return undefined;
|
|
949
|
-
return {
|
|
950
|
-
pendingRunType: "branch_upkeep",
|
|
951
|
-
factoryState: "changes_requested",
|
|
952
|
-
context: buildReviewFixBranchUpkeepContext(issue.prNumber, project?.github?.baseBranch ?? "main", pr),
|
|
953
|
-
summary: `PR #${issue.prNumber} is still dirty after review fix; queued branch upkeep`,
|
|
954
|
-
};
|
|
955
|
-
}
|
|
956
|
-
catch (error) {
|
|
957
|
-
this.logger.debug({
|
|
958
|
-
issueKey: issue.issueKey,
|
|
959
|
-
prNumber: issue.prNumber,
|
|
960
|
-
error: error instanceof Error ? error.message : String(error),
|
|
961
|
-
}, "Failed to resolve post-run PR upkeep");
|
|
962
|
-
return undefined;
|
|
963
|
-
}
|
|
964
|
-
}
|
|
965
|
-
async verifyPublishedRunOutcome(run, issue, projectOverride) {
|
|
966
|
-
if (run.runType !== "implementation") {
|
|
967
|
-
return undefined;
|
|
968
|
-
}
|
|
969
|
-
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
970
|
-
const baseBranch = project?.github?.baseBranch ?? "main";
|
|
971
|
-
const deliveryMode = resolveImplementationDeliveryMode(issue, undefined, run.promptText);
|
|
972
|
-
if (deliveryMode === "linear_only") {
|
|
973
|
-
if (issue.prNumber !== undefined) {
|
|
974
|
-
return `Planning-only implementation should not open a PR, but PR #${issue.prNumber} was observed`;
|
|
975
|
-
}
|
|
976
|
-
return this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
977
|
-
}
|
|
978
|
-
if (issue.prNumber && issue.prState && issue.prState !== "closed") {
|
|
979
|
-
return undefined;
|
|
980
|
-
}
|
|
981
|
-
if (project?.github?.repoFullName && issue.branchName) {
|
|
982
|
-
try {
|
|
983
|
-
const { stdout, exitCode } = await execCommand("gh", [
|
|
984
|
-
"pr",
|
|
985
|
-
"list",
|
|
986
|
-
"--repo",
|
|
987
|
-
project.github.repoFullName,
|
|
988
|
-
"--head",
|
|
989
|
-
issue.branchName,
|
|
990
|
-
"--state",
|
|
991
|
-
"all",
|
|
992
|
-
"--json",
|
|
993
|
-
"number,url,state,author,headRefOid",
|
|
994
|
-
], { timeoutMs: 10_000 });
|
|
995
|
-
if (exitCode === 0) {
|
|
996
|
-
const matches = JSON.parse(stdout);
|
|
997
|
-
const pr = matches[0];
|
|
998
|
-
if (pr?.number) {
|
|
999
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1000
|
-
projectId: issue.projectId,
|
|
1001
|
-
linearIssueId: issue.linearIssueId,
|
|
1002
|
-
prNumber: pr.number,
|
|
1003
|
-
...(pr.url ? { prUrl: pr.url } : {}),
|
|
1004
|
-
...(pr.state ? { prState: pr.state.toLowerCase() } : {}),
|
|
1005
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1006
|
-
...(pr.author?.login ? { prAuthorLogin: pr.author.login } : {}),
|
|
1007
|
-
}, "published PR verification refresh");
|
|
1008
|
-
return undefined;
|
|
1009
|
-
}
|
|
1010
|
-
}
|
|
1011
|
-
}
|
|
1012
|
-
catch (error) {
|
|
1013
|
-
this.logger.debug({
|
|
1014
|
-
issueKey: issue.issueKey,
|
|
1015
|
-
branchName: issue.branchName,
|
|
1016
|
-
repoFullName: project.github.repoFullName,
|
|
1017
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1018
|
-
}, "Failed to verify published PR state after implementation");
|
|
1019
|
-
}
|
|
1020
|
-
}
|
|
1021
|
-
const details = await this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
1022
|
-
return details ?? `Implementation completed without opening a PR for branch ${issue.branchName ?? issue.linearIssueId}`;
|
|
1023
|
-
}
|
|
1024
|
-
async describeLocalImplementationOutcome(issue, baseBranch, deliveryMode = "publish_pr") {
|
|
1025
|
-
if (!issue.worktreePath) {
|
|
1026
|
-
return undefined;
|
|
1027
|
-
}
|
|
1028
|
-
try {
|
|
1029
|
-
const status = await execCommand(this.config.runner.gitBin, [
|
|
1030
|
-
"-C",
|
|
1031
|
-
issue.worktreePath,
|
|
1032
|
-
"status",
|
|
1033
|
-
"--short",
|
|
1034
|
-
], { timeoutMs: 10_000 });
|
|
1035
|
-
const dirtyEntries = status.exitCode === 0
|
|
1036
|
-
? status.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
1037
|
-
: [];
|
|
1038
|
-
if (dirtyEntries.length > 0) {
|
|
1039
|
-
if (deliveryMode === "linear_only") {
|
|
1040
|
-
return `Planning-only implementation should not modify the repo; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1041
|
-
}
|
|
1042
|
-
return `Implementation completed without opening a PR; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1043
|
-
}
|
|
1044
|
-
}
|
|
1045
|
-
catch {
|
|
1046
|
-
// Best effort only.
|
|
1047
|
-
}
|
|
1048
|
-
try {
|
|
1049
|
-
const ahead = await execCommand(this.config.runner.gitBin, [
|
|
1050
|
-
"-C",
|
|
1051
|
-
issue.worktreePath,
|
|
1052
|
-
"rev-list",
|
|
1053
|
-
"--count",
|
|
1054
|
-
`origin/${baseBranch}..HEAD`,
|
|
1055
|
-
], { timeoutMs: 10_000 });
|
|
1056
|
-
if (ahead.exitCode === 0) {
|
|
1057
|
-
const count = Number(ahead.stdout.trim());
|
|
1058
|
-
if (Number.isFinite(count) && count > 0) {
|
|
1059
|
-
if (deliveryMode === "linear_only") {
|
|
1060
|
-
return `Planning-only implementation should not create repo commits; worktree is ${count} local commit(s) ahead of origin/${baseBranch}`;
|
|
1061
|
-
}
|
|
1062
|
-
return `Implementation completed with ${count} local commit(s) ahead of origin/${baseBranch} but no PR was observed`;
|
|
1063
|
-
}
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
|
-
catch {
|
|
1067
|
-
// Best effort only.
|
|
1068
|
-
}
|
|
1069
|
-
return undefined;
|
|
532
|
+
async resolveRequestedChangesWakeContext(issue, runType, context) {
|
|
533
|
+
return await this.runCompletionPolicy.resolveRequestedChangesWakeContext(issue, runType, context);
|
|
1070
534
|
}
|
|
1071
535
|
async readThreadWithRetry(threadId, maxRetries = 3) {
|
|
1072
536
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
@@ -1087,18 +551,6 @@ export class RunOrchestrator {
|
|
|
1087
551
|
withHeldIssueSessionLease(projectId, linearIssueId, fn) {
|
|
1088
552
|
return this.leaseService.withHeldLease(projectId, linearIssueId, fn);
|
|
1089
553
|
}
|
|
1090
|
-
upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
|
|
1091
|
-
const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
|
|
1092
|
-
if (!lease) {
|
|
1093
|
-
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write without a held issue-session lease");
|
|
1094
|
-
return undefined;
|
|
1095
|
-
}
|
|
1096
|
-
const updated = this.db.issueSessions.upsertIssueWithLease(lease, params);
|
|
1097
|
-
if (!updated) {
|
|
1098
|
-
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
|
|
1099
|
-
}
|
|
1100
|
-
return updated;
|
|
1101
|
-
}
|
|
1102
554
|
assertLaunchLease(run, phase) {
|
|
1103
555
|
if (this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
1104
556
|
return;
|
|
@@ -1108,12 +560,6 @@ export class RunOrchestrator {
|
|
|
1108
560
|
this.logger.warn({ runId: run.id, issueId: run.linearIssueId, phase }, "Aborting run launch after losing issue-session lease");
|
|
1109
561
|
throw error;
|
|
1110
562
|
}
|
|
1111
|
-
acquireIssueSessionLease(projectId, linearIssueId) {
|
|
1112
|
-
return this.leaseService.acquire(projectId, linearIssueId);
|
|
1113
|
-
}
|
|
1114
|
-
forceAcquireIssueSessionLease(projectId, linearIssueId) {
|
|
1115
|
-
return this.leaseService.forceAcquire(projectId, linearIssueId);
|
|
1116
|
-
}
|
|
1117
563
|
claimLeaseForReconciliation(projectId, linearIssueId) {
|
|
1118
564
|
return this.leaseService.claimForReconciliation(projectId, linearIssueId);
|
|
1119
565
|
}
|
|
@@ -1127,87 +573,3 @@ export class RunOrchestrator {
|
|
|
1127
573
|
this.leaseService.release(projectId, linearIssueId);
|
|
1128
574
|
}
|
|
1129
575
|
}
|
|
1130
|
-
/**
|
|
1131
|
-
* Determine post-run factory state from current PR metadata.
|
|
1132
|
-
* Used by both the normal completion path and reconciliation.
|
|
1133
|
-
*/
|
|
1134
|
-
function resolvePostRunState(issue) {
|
|
1135
|
-
if (ACTIVE_RUN_STATES.has(issue.factoryState) && issue.prNumber) {
|
|
1136
|
-
// Check merged first — a merged PR is both approved and merged,
|
|
1137
|
-
// and "done" must take priority over "awaiting_queue".
|
|
1138
|
-
if (issue.prState === "merged")
|
|
1139
|
-
return "done";
|
|
1140
|
-
if (issue.prReviewState === "approved")
|
|
1141
|
-
return "awaiting_queue";
|
|
1142
|
-
return "pr_open";
|
|
1143
|
-
}
|
|
1144
|
-
return undefined;
|
|
1145
|
-
}
|
|
1146
|
-
function resolveCompletedRunState(issue, run) {
|
|
1147
|
-
if (run.runType === "implementation" && resolveImplementationDeliveryMode(issue, undefined, run.promptText) === "linear_only") {
|
|
1148
|
-
return "done";
|
|
1149
|
-
}
|
|
1150
|
-
return resolvePostRunState(issue);
|
|
1151
|
-
}
|
|
1152
|
-
function resolveRecoverablePostRunState(issue) {
|
|
1153
|
-
if (!issue.prNumber) {
|
|
1154
|
-
return resolvePostRunState(issue);
|
|
1155
|
-
}
|
|
1156
|
-
if (issue.prState === "merged")
|
|
1157
|
-
return "done";
|
|
1158
|
-
if (issue.prState === "open") {
|
|
1159
|
-
const reactiveIntent = deriveIssueSessionReactiveIntent({
|
|
1160
|
-
prNumber: issue.prNumber,
|
|
1161
|
-
prState: issue.prState,
|
|
1162
|
-
prReviewState: issue.prReviewState,
|
|
1163
|
-
prCheckStatus: issue.prCheckStatus,
|
|
1164
|
-
latestFailureSource: issue.lastGitHubFailureSource,
|
|
1165
|
-
});
|
|
1166
|
-
if (reactiveIntent)
|
|
1167
|
-
return reactiveIntent.compatibilityFactoryState;
|
|
1168
|
-
if (issue.prReviewState === "approved")
|
|
1169
|
-
return "awaiting_queue";
|
|
1170
|
-
return "pr_open";
|
|
1171
|
-
}
|
|
1172
|
-
return resolvePostRunState(issue);
|
|
1173
|
-
}
|
|
1174
|
-
function normalizeRemotePrState(value) {
|
|
1175
|
-
const normalized = value?.trim().toUpperCase();
|
|
1176
|
-
if (normalized === "OPEN")
|
|
1177
|
-
return "open";
|
|
1178
|
-
if (normalized === "CLOSED")
|
|
1179
|
-
return "closed";
|
|
1180
|
-
if (normalized === "MERGED")
|
|
1181
|
-
return "merged";
|
|
1182
|
-
return undefined;
|
|
1183
|
-
}
|
|
1184
|
-
function normalizeRemoteReviewDecision(value) {
|
|
1185
|
-
const normalized = value?.trim().toUpperCase();
|
|
1186
|
-
if (normalized === "APPROVED")
|
|
1187
|
-
return "approved";
|
|
1188
|
-
if (normalized === "CHANGES_REQUESTED")
|
|
1189
|
-
return "changes_requested";
|
|
1190
|
-
if (normalized === "REVIEW_REQUIRED")
|
|
1191
|
-
return "commented";
|
|
1192
|
-
return undefined;
|
|
1193
|
-
}
|
|
1194
|
-
function isDirtyMergeStateStatus(value) {
|
|
1195
|
-
return value?.trim().toUpperCase() === "DIRTY";
|
|
1196
|
-
}
|
|
1197
|
-
function buildReviewFixBranchUpkeepContext(prNumber, baseBranch, pr, context) {
|
|
1198
|
-
const promptContext = [
|
|
1199
|
-
`The requested code change may already be present, but GitHub still reports PR #${prNumber} as ${String(pr.mergeStateStatus)} against latest ${baseBranch}.`,
|
|
1200
|
-
`This turn is branch upkeep on the existing PR branch: update onto latest ${baseBranch}, resolve any conflicts, rerun the narrowest relevant verification, and push a newer head.`,
|
|
1201
|
-
"Do not stop just because the requested code change is already present. Review can only move forward after a new pushed head.",
|
|
1202
|
-
].join(" ");
|
|
1203
|
-
return {
|
|
1204
|
-
...(context ?? {}),
|
|
1205
|
-
branchUpkeepRequired: true,
|
|
1206
|
-
reviewFixMode: "branch_upkeep",
|
|
1207
|
-
wakeReason: "branch_upkeep",
|
|
1208
|
-
promptContext,
|
|
1209
|
-
...(pr.mergeStateStatus ? { mergeStateStatus: pr.mergeStateStatus } : {}),
|
|
1210
|
-
...(pr.headRefOid ? { failingHeadSha: pr.headRefOid } : {}),
|
|
1211
|
-
baseBranch,
|
|
1212
|
-
};
|
|
1213
|
-
}
|