patchrelay 0.36.8 → 0.36.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/cluster-health.js +1 -1
- package/dist/cli/data.js +9 -9
- package/dist/db/issue-session-store.js +15 -23
- package/dist/db/issue-store.js +559 -0
- package/dist/db/run-store.js +10 -12
- package/dist/db.js +37 -625
- package/dist/github-webhook-handler.js +19 -19
- package/dist/idle-reconciliation.js +15 -15
- package/dist/interrupted-run-recovery.js +176 -0
- package/dist/issue-query-service.js +4 -4
- package/dist/issue-session-projector.js +114 -0
- package/dist/linear-session-sync.js +6 -6
- package/dist/queue-health-monitor.js +3 -3
- package/dist/run-completion-policy.js +412 -0
- package/dist/run-finalizer.js +34 -23
- package/dist/run-launcher.js +5 -5
- package/dist/run-orchestrator.js +33 -685
- package/dist/run-recovery-service.js +19 -13
- package/dist/run-wake-planner.js +1 -1
- package/dist/service.js +9 -9
- package/dist/webhook-handler.js +5 -5
- package/dist/webhooks/agent-session-handler.js +7 -7
- package/dist/webhooks/comment-wake-handler.js +1 -1
- package/dist/webhooks/desired-stage-recorder.js +5 -5
- package/dist/webhooks/issue-removal-handler.js +3 -3
- package/dist/worktree-manager.js +69 -0
- package/package.json +1 -1
package/dist/run-orchestrator.js
CHANGED
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { TERMINAL_STATES } from "./factory-state.js";
|
|
2
2
|
import { extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread } from "./run-reporting.js";
|
|
3
3
|
import { buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
4
4
|
import { WorktreeManager } from "./worktree-manager.js";
|
|
5
5
|
import { resolveAuthoritativeLinearStopState, resolvePreferredCompletedLinearState } from "./linear-workflow.js";
|
|
6
|
-
import { execCommand } from "./utils.js";
|
|
7
6
|
import { getThreadTurns } from "./codex-thread-utils.js";
|
|
8
|
-
import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
9
7
|
import { QueueHealthMonitor } from "./queue-health-monitor.js";
|
|
10
|
-
import { resolveImplementationDeliveryMode, } from "./prompting/patchrelay.js";
|
|
11
8
|
import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
|
|
12
9
|
import { LinearSessionSync } from "./linear-session-sync.js";
|
|
13
10
|
import { IssueSessionLeaseService } from "./issue-session-lease-service.js";
|
|
11
|
+
import { InterruptedRunRecovery, resolveRecoverablePostRunState } from "./interrupted-run-recovery.js";
|
|
12
|
+
import { RunCompletionPolicy } from "./run-completion-policy.js";
|
|
14
13
|
import { RunFinalizer } from "./run-finalizer.js";
|
|
15
14
|
import { RunLauncher } from "./run-launcher.js";
|
|
16
15
|
import { RunRecoveryService } from "./run-recovery-service.js";
|
|
@@ -21,17 +20,6 @@ function lowerCaseFirst(value) {
|
|
|
21
20
|
function isRequestedChangesRunType(runType) {
|
|
22
21
|
return runType === "review_fix" || runType === "branch_upkeep";
|
|
23
22
|
}
|
|
24
|
-
function resolveRequestedChangesMode(runType, context) {
|
|
25
|
-
if (runType === "branch_upkeep") {
|
|
26
|
-
return "branch_upkeep";
|
|
27
|
-
}
|
|
28
|
-
return context?.reviewFixMode === "branch_upkeep" || context?.branchUpkeepRequired === true
|
|
29
|
-
? "branch_upkeep"
|
|
30
|
-
: "address_review_feedback";
|
|
31
|
-
}
|
|
32
|
-
function isBranchUpkeepRequired(context) {
|
|
33
|
-
return context?.branchUpkeepRequired === true;
|
|
34
|
-
}
|
|
35
23
|
export class RunOrchestrator {
|
|
36
24
|
config;
|
|
37
25
|
db;
|
|
@@ -52,6 +40,8 @@ export class RunOrchestrator {
|
|
|
52
40
|
runLauncher;
|
|
53
41
|
runRecovery;
|
|
54
42
|
runWakePlanner;
|
|
43
|
+
interruptedRunRecovery;
|
|
44
|
+
runCompletionPolicy;
|
|
55
45
|
activeSessionLeases;
|
|
56
46
|
botIdentity;
|
|
57
47
|
constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
|
|
@@ -66,9 +56,11 @@ export class RunOrchestrator {
|
|
|
66
56
|
this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
|
|
67
57
|
this.leaseService = new IssueSessionLeaseService(db, logger, this.workerId, (threadId, maxRetries) => this.readThreadWithRetry(threadId, maxRetries));
|
|
68
58
|
this.activeSessionLeases = this.leaseService.activeSessionLeases;
|
|
69
|
-
this.
|
|
59
|
+
this.runCompletionPolicy = new RunCompletionPolicy(config, db, logger, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn));
|
|
60
|
+
this.runFinalizer = new RunFinalizer(db, logger, this.linearSync, this.enqueueIssue, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (lease, issue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope), (run, message, nextState) => this.failRunAndClear(run, message, nextState), this.runCompletionPolicy, feed);
|
|
70
61
|
this.runLauncher = new RunLauncher(config, db, codex, logger, this.worktreeManager);
|
|
71
|
-
this.runRecovery = new RunRecoveryService(db, logger, this.linearSync, (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (projectId, issueId) => this.enqueueIssue(projectId, issueId), (newState, pendingRunType) => this.resolveBranchOwnerForStateTransition(newState, pendingRunType), feed);
|
|
62
|
+
this.runRecovery = new RunRecoveryService(db, logger, this.linearSync, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.getHeldIssueSessionLease(projectId, linearIssueId), (lease, issue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, issue, runType, context, dedupeScope), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (projectId, issueId) => this.enqueueIssue(projectId, issueId), (newState, pendingRunType) => this.resolveBranchOwnerForStateTransition(newState, pendingRunType), feed);
|
|
63
|
+
this.interruptedRunRecovery = new InterruptedRunRecovery(db, logger, this.linearSync, (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn), (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId), (run, message, nextState) => this.failRunAndClear(run, message, nextState), (issue) => this.restoreIdleWorktree(issue), this.runCompletionPolicy, (projectId, issueId) => this.enqueueIssue(projectId, issueId), feed);
|
|
72
64
|
this.runWakePlanner = new RunWakePlanner(db);
|
|
73
65
|
this.idleReconciler = new IdleIssueReconciler(db, config, {
|
|
74
66
|
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
@@ -95,7 +87,7 @@ export class RunOrchestrator {
|
|
|
95
87
|
if (this.leaseService.hasLocalLease(item.projectId, item.issueId)) {
|
|
96
88
|
return;
|
|
97
89
|
}
|
|
98
|
-
const issue = this.db.getIssue(item.projectId, item.issueId);
|
|
90
|
+
const issue = this.db.issues.getIssue(item.projectId, item.issueId);
|
|
99
91
|
if (!issue || issue.activeRunId !== undefined)
|
|
100
92
|
return;
|
|
101
93
|
const issueSession = this.db.issueSessions.getIssueSession(item.projectId, item.issueId);
|
|
@@ -117,7 +109,7 @@ export class RunOrchestrator {
|
|
|
117
109
|
}
|
|
118
110
|
const { runType, context, resumeThread } = wake;
|
|
119
111
|
const effectiveContext = isRequestedChangesRunType(runType)
|
|
120
|
-
? await this.resolveRequestedChangesWakeContext(issue, runType, context
|
|
112
|
+
? await this.runCompletionPolicy.resolveRequestedChangesWakeContext(issue, runType, context)
|
|
121
113
|
: context;
|
|
122
114
|
const sourceHeadSha = typeof effectiveContext?.failureHeadSha === "string"
|
|
123
115
|
? effectiveContext.failureHeadSha
|
|
@@ -179,8 +171,6 @@ export class RunOrchestrator {
|
|
|
179
171
|
leaseId,
|
|
180
172
|
...(this.botIdentity ? { botIdentity: this.botIdentity } : {}),
|
|
181
173
|
assertLaunchLease: (targetRun, phase) => this.assertLaunchLease(targetRun, phase),
|
|
182
|
-
resetWorktreeToTrackedBranch: (targetWorktreePath, targetBranchName, targetIssue) => this.resetWorktreeToTrackedBranch(targetWorktreePath, targetBranchName, targetIssue),
|
|
183
|
-
freshenWorktree: (targetWorktreePath, targetProject, targetIssue) => this.freshenWorktree(targetWorktreePath, targetProject, targetIssue),
|
|
184
174
|
linearSync: this.linearSync,
|
|
185
175
|
releaseLease: (projectId, issueId) => this.releaseIssueSessionLease(projectId, issueId),
|
|
186
176
|
isRequestedChangesRunType,
|
|
@@ -203,98 +193,15 @@ export class RunOrchestrator {
|
|
|
203
193
|
}
|
|
204
194
|
this.logger.info({ issueKey: issue.issueKey, runType, threadId, turnId }, `Started ${runType} run`);
|
|
205
195
|
// Emit Linear activity + plan
|
|
206
|
-
const freshIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
196
|
+
const freshIssue = this.db.issues.getIssue(item.projectId, item.issueId) ?? issue;
|
|
207
197
|
void this.linearSync.emitActivity(freshIssue, buildRunStartedActivity(runType));
|
|
208
198
|
void this.linearSync.syncSession(freshIssue, { activeRunType: runType });
|
|
209
199
|
}
|
|
210
|
-
// ─── Pre-run branch freshening ────────────────────────────────────
|
|
211
|
-
/**
|
|
212
|
-
* Fetch origin and rebase the worktree onto the latest base branch.
|
|
213
|
-
*
|
|
214
|
-
* Risks mitigated:
|
|
215
|
-
* - Dirty worktree from interrupted run → stash before, pop after
|
|
216
|
-
* - Conflicts → abort rebase, throw so the run fails with a clear reason
|
|
217
|
-
* - Already up-to-date → no-op
|
|
218
|
-
* - Keep publishing explicit: the orchestrator updates the local worktree
|
|
219
|
-
* only; the agent/run owns any later branch push.
|
|
220
|
-
*/
|
|
221
|
-
async freshenWorktree(worktreePath, project, issue) {
|
|
222
|
-
const gitBin = this.config.runner.gitBin;
|
|
223
|
-
const baseBranch = project.github?.baseBranch ?? "main";
|
|
224
|
-
// Stash any uncommitted changes from a previous interrupted run
|
|
225
|
-
const stashResult = await execCommand(gitBin, ["-C", worktreePath, "stash"], { timeoutMs: 30_000 });
|
|
226
|
-
const didStash = stashResult.exitCode === 0 && !stashResult.stdout?.includes("No local changes");
|
|
227
|
-
// Fetch latest base
|
|
228
|
-
const fetchResult = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", baseBranch], { timeoutMs: 60_000 });
|
|
229
|
-
if (fetchResult.exitCode !== 0) {
|
|
230
|
-
this.logger.warn({ issueKey: issue.issueKey, stderr: fetchResult.stderr?.slice(0, 300) }, "Pre-run fetch failed, proceeding with current base");
|
|
231
|
-
if (didStash)
|
|
232
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
233
|
-
return;
|
|
234
|
-
}
|
|
235
|
-
// Check if rebase is needed: is HEAD already on top of origin/baseBranch?
|
|
236
|
-
const mergeBaseResult = await execCommand(gitBin, ["-C", worktreePath, "merge-base", "--is-ancestor", `origin/${baseBranch}`, "HEAD"], { timeoutMs: 10_000 });
|
|
237
|
-
if (mergeBaseResult.exitCode === 0) {
|
|
238
|
-
// Already up-to-date — no rebase needed
|
|
239
|
-
this.logger.debug({ issueKey: issue.issueKey }, "Pre-run freshen: branch already up to date");
|
|
240
|
-
if (didStash)
|
|
241
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
242
|
-
return;
|
|
243
|
-
}
|
|
244
|
-
// Rebase onto latest base
|
|
245
|
-
const rebaseResult = await execCommand(gitBin, ["-C", worktreePath, "rebase", `origin/${baseBranch}`], { timeoutMs: 120_000 });
|
|
246
|
-
if (rebaseResult.exitCode !== 0) {
|
|
247
|
-
// Abort the failed rebase and restore state — then let the agent run
|
|
248
|
-
// proceed. The agent can resolve the conflict itself (the workflow
|
|
249
|
-
// prompt tells it to rebase and handle conflicts).
|
|
250
|
-
await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
|
|
251
|
-
if (didStash)
|
|
252
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
253
|
-
this.logger.warn({ issueKey: issue.issueKey, baseBranch }, "Pre-run freshen: rebase conflict, agent will resolve");
|
|
254
|
-
return;
|
|
255
|
-
}
|
|
256
|
-
this.logger.info({ issueKey: issue.issueKey, baseBranch }, "Pre-run freshen: rebased locally onto latest base");
|
|
257
|
-
// Restore stashed changes
|
|
258
|
-
if (didStash)
|
|
259
|
-
await execCommand(gitBin, ["-C", worktreePath, "stash", "pop"], { timeoutMs: 10_000 });
|
|
260
|
-
}
|
|
261
200
|
async resetWorktreeToTrackedBranch(worktreePath, branchName, issue) {
|
|
262
|
-
|
|
263
|
-
const branchFetch = await execCommand(gitBin, ["-C", worktreePath, "fetch", "origin", branchName], { timeoutMs: 60_000 });
|
|
264
|
-
const hasRemoteBranch = branchFetch.exitCode === 0;
|
|
265
|
-
await execCommand(gitBin, ["-C", worktreePath, "rebase", "--abort"], { timeoutMs: 10_000 });
|
|
266
|
-
await execCommand(gitBin, ["-C", worktreePath, "merge", "--abort"], { timeoutMs: 10_000 });
|
|
267
|
-
await execCommand(gitBin, ["-C", worktreePath, "cherry-pick", "--abort"], { timeoutMs: 10_000 });
|
|
268
|
-
await execCommand(gitBin, ["-C", worktreePath, "am", "--abort"], { timeoutMs: 10_000 });
|
|
269
|
-
await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", "HEAD"], { timeoutMs: 30_000 });
|
|
270
|
-
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
271
|
-
const checkoutTarget = hasRemoteBranch ? `origin/${branchName}` : branchName;
|
|
272
|
-
const checkoutResult = await execCommand(gitBin, ["-C", worktreePath, "checkout", "-B", branchName, checkoutTarget], { timeoutMs: 30_000 });
|
|
273
|
-
if (checkoutResult.exitCode !== 0) {
|
|
274
|
-
throw new Error(`Failed to restore ${branchName} worktree state: ${checkoutResult.stderr?.slice(0, 300) ?? "git checkout failed"}`);
|
|
275
|
-
}
|
|
276
|
-
const resetTarget = hasRemoteBranch ? `origin/${branchName}` : "HEAD";
|
|
277
|
-
const resetResult = await execCommand(gitBin, ["-C", worktreePath, "reset", "--hard", resetTarget], { timeoutMs: 30_000 });
|
|
278
|
-
if (resetResult.exitCode !== 0) {
|
|
279
|
-
throw new Error(`Failed to reset ${branchName} worktree state: ${resetResult.stderr?.slice(0, 300) ?? "git reset failed"}`);
|
|
280
|
-
}
|
|
281
|
-
await execCommand(gitBin, ["-C", worktreePath, "clean", "-fd"], { timeoutMs: 30_000 });
|
|
282
|
-
this.logger.debug({ issueKey: issue.issueKey, branchName, hasRemoteBranch }, "Reset issue worktree to tracked branch state");
|
|
201
|
+
await this.worktreeManager.resetWorktreeToTrackedBranch(worktreePath, branchName, issue, this.logger);
|
|
283
202
|
}
|
|
284
203
|
async restoreIdleWorktree(issue) {
|
|
285
|
-
|
|
286
|
-
return;
|
|
287
|
-
try {
|
|
288
|
-
await this.resetWorktreeToTrackedBranch(issue.worktreePath, issue.branchName, issue);
|
|
289
|
-
}
|
|
290
|
-
catch (error) {
|
|
291
|
-
this.logger.warn({
|
|
292
|
-
issueKey: issue.issueKey,
|
|
293
|
-
branchName: issue.branchName,
|
|
294
|
-
worktreePath: issue.worktreePath,
|
|
295
|
-
error: error instanceof Error ? error.message : String(error),
|
|
296
|
-
}, "Failed to restore idle worktree after interrupted run");
|
|
297
|
-
}
|
|
204
|
+
await this.worktreeManager.restoreIdleWorktree(issue, this.logger);
|
|
298
205
|
}
|
|
299
206
|
// ─── Notification handler ─────────────────────────────────────────
|
|
300
207
|
async handleCodexNotification(notification) {
|
|
@@ -331,7 +238,7 @@ export class RunOrchestrator {
|
|
|
331
238
|
this.linearSync.maybeEmitProgress(notification, run);
|
|
332
239
|
// Sync codex plan to Linear session when it updates
|
|
333
240
|
if (notification.method === "turn/plan/updated") {
|
|
334
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
241
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
335
242
|
if (issue) {
|
|
336
243
|
void this.linearSync.syncCodexPlan(issue, notification.params);
|
|
337
244
|
}
|
|
@@ -339,7 +246,7 @@ export class RunOrchestrator {
|
|
|
339
246
|
if (notification.method !== "turn/completed")
|
|
340
247
|
return;
|
|
341
248
|
const thread = await this.readThreadWithRetry(threadId);
|
|
342
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
249
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
343
250
|
if (!issue)
|
|
344
251
|
return;
|
|
345
252
|
const completedTurnId = extractTurnId(notification.params);
|
|
@@ -375,7 +282,7 @@ export class RunOrchestrator {
|
|
|
375
282
|
status: "failed",
|
|
376
283
|
summary: `Turn failed for ${run.runType}`,
|
|
377
284
|
});
|
|
378
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
285
|
+
const failedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
379
286
|
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType));
|
|
380
287
|
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
381
288
|
this.linearSync.clearProgress(run.id);
|
|
@@ -390,23 +297,13 @@ export class RunOrchestrator {
|
|
|
390
297
|
thread,
|
|
391
298
|
threadId,
|
|
392
299
|
...(completedTurnId ? { completedTurnId } : {}),
|
|
393
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
394
|
-
releaseLease: (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId),
|
|
395
|
-
failRunAndClear: (targetRun, message, nextState) => this.failRunAndClear(targetRun, message, nextState),
|
|
396
|
-
verifyReactiveRunAdvancedBranch: (targetRun, targetIssue) => this.verifyReactiveRunAdvancedBranch(targetRun, targetIssue),
|
|
397
|
-
verifyReviewFixAdvancedHead: (targetRun, targetIssue) => this.verifyReviewFixAdvancedHead(targetRun, targetIssue),
|
|
398
|
-
verifyPublishedRunOutcome: (targetRun, targetIssue) => this.verifyPublishedRunOutcome(targetRun, targetIssue),
|
|
399
|
-
refreshIssueAfterReactivePublish: (targetRun, targetIssue) => this.refreshIssueAfterReactivePublish(targetRun, targetIssue),
|
|
400
|
-
resolvePostRunFollowUp: (targetRun, targetIssue) => this.resolvePostRunFollowUp(targetRun, targetIssue),
|
|
401
|
-
resolveCompletedRunState,
|
|
402
300
|
resolveRecoverableRunState: resolveRecoverablePostRunState,
|
|
403
|
-
appendWakeEventWithLease: (lease, targetIssue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, runType, context, dedupeScope),
|
|
404
301
|
});
|
|
405
302
|
this.activeThreadId = undefined;
|
|
406
303
|
}
|
|
407
304
|
// ─── Active status for query ──────────────────────────────────────
|
|
408
305
|
async getActiveRunStatus(issueKey) {
|
|
409
|
-
const issue = this.db.getIssueByKey(issueKey);
|
|
306
|
+
const issue = this.db.issues.getIssueByKey(issueKey);
|
|
410
307
|
if (!issue?.activeRunId)
|
|
411
308
|
return undefined;
|
|
412
309
|
const run = this.db.runs.getRunById(issue.activeRunId);
|
|
@@ -434,7 +331,7 @@ export class RunOrchestrator {
|
|
|
434
331
|
await this.reconcileMergedLinearCompletion();
|
|
435
332
|
}
|
|
436
333
|
async reconcileMergedLinearCompletion() {
|
|
437
|
-
for (const issue of this.db.listIssues()) {
|
|
334
|
+
for (const issue of this.db.issues.listIssues()) {
|
|
438
335
|
if (issue.prState !== "merged")
|
|
439
336
|
continue;
|
|
440
337
|
if (issue.currentLinearStateType?.trim().toLowerCase() === "completed")
|
|
@@ -449,7 +346,7 @@ export class RunOrchestrator {
|
|
|
449
346
|
continue;
|
|
450
347
|
const normalizedCurrent = liveIssue.stateName?.trim().toLowerCase();
|
|
451
348
|
if (normalizedCurrent === targetState.trim().toLowerCase()) {
|
|
452
|
-
this.db.upsertIssue({
|
|
349
|
+
this.db.issues.upsertIssue({
|
|
453
350
|
projectId: issue.projectId,
|
|
454
351
|
linearIssueId: issue.linearIssueId,
|
|
455
352
|
...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
|
|
@@ -458,7 +355,7 @@ export class RunOrchestrator {
|
|
|
458
355
|
continue;
|
|
459
356
|
}
|
|
460
357
|
const updated = await linear.setIssueState(issue.linearIssueId, targetState);
|
|
461
|
-
this.db.upsertIssue({
|
|
358
|
+
this.db.issues.upsertIssue({
|
|
462
359
|
projectId: issue.projectId,
|
|
463
360
|
linearIssueId: issue.linearIssueId,
|
|
464
361
|
...(updated.stateName ? { currentLinearState: updated.stateName } : {}),
|
|
@@ -485,12 +382,10 @@ export class RunOrchestrator {
|
|
|
485
382
|
runType,
|
|
486
383
|
reason,
|
|
487
384
|
isRequestedChangesRunType,
|
|
488
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
489
|
-
appendWakeEventWithLease: (lease, targetIssue, pendingRunType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, pendingRunType, context, dedupeScope),
|
|
490
385
|
});
|
|
491
386
|
}
|
|
492
387
|
async reconcileRun(run) {
|
|
493
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
388
|
+
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
494
389
|
if (!issue)
|
|
495
390
|
return;
|
|
496
391
|
let recoveryLease = this.claimLeaseForReconciliation(run.projectId, run.linearIssueId);
|
|
@@ -505,10 +400,10 @@ export class RunOrchestrator {
|
|
|
505
400
|
if (TERMINAL_STATES.has(issue.factoryState)) {
|
|
506
401
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
507
402
|
this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
508
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
403
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
509
404
|
});
|
|
510
405
|
this.logger.info({ issueKey: issue.issueKey, runId: run.id, factoryState: issue.factoryState }, "Reconciliation: released run on terminal issue");
|
|
511
|
-
const releasedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
406
|
+
const releasedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
512
407
|
void this.linearSync.syncSession(releasedIssue, { activeRunType: run.runType });
|
|
513
408
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
514
409
|
return;
|
|
@@ -518,10 +413,10 @@ export class RunOrchestrator {
|
|
|
518
413
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
|
|
519
414
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
520
415
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
521
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
416
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
522
417
|
});
|
|
523
418
|
this.recoverOrEscalate(issue, run.runType, "zombie");
|
|
524
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
419
|
+
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
525
420
|
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "The Codex turn never started before PatchRelay restarted."));
|
|
526
421
|
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
527
422
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
@@ -536,10 +431,10 @@ export class RunOrchestrator {
|
|
|
536
431
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
|
|
537
432
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
538
433
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
539
|
-
this.db.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
434
|
+
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
540
435
|
});
|
|
541
436
|
this.recoverOrEscalate(issue, run.runType, "stale_thread");
|
|
542
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
437
|
+
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
543
438
|
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, "PatchRelay lost the active Codex thread after restart and needs to recover."));
|
|
544
439
|
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
545
440
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
@@ -554,7 +449,7 @@ export class RunOrchestrator {
|
|
|
554
449
|
if (stopState?.isFinal) {
|
|
555
450
|
this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, () => {
|
|
556
451
|
this.db.runs.finishRun(run.id, { status: "released" });
|
|
557
|
-
this.db.upsertIssue({
|
|
452
|
+
this.db.issues.upsertIssue({
|
|
558
453
|
projectId: run.projectId,
|
|
559
454
|
linearIssueId: run.linearIssueId,
|
|
560
455
|
activeRunId: null,
|
|
@@ -571,7 +466,7 @@ export class RunOrchestrator {
|
|
|
571
466
|
status: "reconciled",
|
|
572
467
|
summary: `Linear state ${stopState.stateName} \u2192 done`,
|
|
573
468
|
});
|
|
574
|
-
const doneIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
469
|
+
const doneIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
575
470
|
void this.linearSync.syncSession(doneIssue, { activeRunType: run.runType });
|
|
576
471
|
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
577
472
|
return;
|
|
@@ -583,112 +478,7 @@ export class RunOrchestrator {
|
|
|
583
478
|
// The agent may have partially completed work (commits, PR) before interruption.
|
|
584
479
|
// Reactive loops (CI repair, review fix) will handle follow-up if needed.
|
|
585
480
|
if (latestTurn?.status === "interrupted") {
|
|
586
|
-
this.
|
|
587
|
-
// Interrupted runs are not real failures — undo the budget increment.
|
|
588
|
-
const repairedCounters = this.withHeldIssueSessionLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
589
|
-
if (run.runType === "ci_repair" && issue.ciRepairAttempts > 0) {
|
|
590
|
-
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
591
|
-
projectId: issue.projectId,
|
|
592
|
-
linearIssueId: issue.linearIssueId,
|
|
593
|
-
ciRepairAttempts: issue.ciRepairAttempts - 1,
|
|
594
|
-
});
|
|
595
|
-
}
|
|
596
|
-
else if (run.runType === "queue_repair" && issue.queueRepairAttempts > 0) {
|
|
597
|
-
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
598
|
-
projectId: issue.projectId,
|
|
599
|
-
linearIssueId: issue.linearIssueId,
|
|
600
|
-
queueRepairAttempts: issue.queueRepairAttempts - 1,
|
|
601
|
-
});
|
|
602
|
-
}
|
|
603
|
-
if (run.runType === "ci_repair" || run.runType === "queue_repair") {
|
|
604
|
-
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
605
|
-
projectId: issue.projectId,
|
|
606
|
-
linearIssueId: issue.linearIssueId,
|
|
607
|
-
lastAttemptedFailureHeadSha: null,
|
|
608
|
-
lastAttemptedFailureSignature: null,
|
|
609
|
-
});
|
|
610
|
-
}
|
|
611
|
-
return true;
|
|
612
|
-
});
|
|
613
|
-
if (!repairedCounters) {
|
|
614
|
-
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping interrupted-run recovery after losing issue-session lease");
|
|
615
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
616
|
-
return;
|
|
617
|
-
}
|
|
618
|
-
if (isRequestedChangesRunType(run.runType)) {
|
|
619
|
-
const refreshedIssue = await this.refreshIssueAfterReactivePublish(run, this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
|
|
620
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
621
|
-
const retryContext = project
|
|
622
|
-
? await this.resolveRequestedChangesWakeContext(refreshedIssue, run.runType, run.runType === "branch_upkeep"
|
|
623
|
-
? {
|
|
624
|
-
branchUpkeepRequired: true,
|
|
625
|
-
reviewFixMode: "branch_upkeep",
|
|
626
|
-
wakeReason: "branch_upkeep",
|
|
627
|
-
}
|
|
628
|
-
: undefined, project)
|
|
629
|
-
: undefined;
|
|
630
|
-
const retryRunType = resolveRequestedChangesMode(run.runType, retryContext) === "branch_upkeep"
|
|
631
|
-
? "branch_upkeep"
|
|
632
|
-
: "review_fix";
|
|
633
|
-
const recoveredState = resolveRecoverablePostRunState(refreshedIssue) ?? "failed";
|
|
634
|
-
const interruptedMessage = "Requested-changes run was interrupted before PatchRelay could verify that a new PR head was published";
|
|
635
|
-
this.failRunAndClear(run, interruptedMessage, recoveredState);
|
|
636
|
-
await this.restoreIdleWorktree(issue);
|
|
637
|
-
const recoveredIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
638
|
-
if (recoveredState === "changes_requested") {
|
|
639
|
-
this.db.upsertIssue({
|
|
640
|
-
projectId: run.projectId,
|
|
641
|
-
linearIssueId: run.linearIssueId,
|
|
642
|
-
pendingRunType: retryRunType,
|
|
643
|
-
pendingRunContextJson: retryContext ? JSON.stringify(retryContext) : null,
|
|
644
|
-
});
|
|
645
|
-
this.feed?.publish({
|
|
646
|
-
level: "warn",
|
|
647
|
-
kind: "workflow",
|
|
648
|
-
issueKey: issue.issueKey,
|
|
649
|
-
projectId: run.projectId,
|
|
650
|
-
stage: run.runType,
|
|
651
|
-
status: "retry_queued",
|
|
652
|
-
summary: "Requested-changes run was interrupted; PatchRelay will retry from fresh GitHub truth",
|
|
653
|
-
});
|
|
654
|
-
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
655
|
-
}
|
|
656
|
-
else {
|
|
657
|
-
this.feed?.publish({
|
|
658
|
-
level: "error",
|
|
659
|
-
kind: "workflow",
|
|
660
|
-
issueKey: issue.issueKey,
|
|
661
|
-
projectId: run.projectId,
|
|
662
|
-
stage: run.runType,
|
|
663
|
-
status: "escalated",
|
|
664
|
-
summary: interruptedMessage,
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
|
-
void this.linearSync.emitActivity(recoveredIssue, buildRunFailureActivity(run.runType, interruptedMessage));
|
|
668
|
-
void this.linearSync.syncSession(recoveredIssue, { activeRunType: run.runType });
|
|
669
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
670
|
-
return;
|
|
671
|
-
}
|
|
672
|
-
const recoveredState = resolveRecoverablePostRunState(this.db.getIssue(run.projectId, run.linearIssueId) ?? issue);
|
|
673
|
-
this.failRunAndClear(run, "Codex turn was interrupted", recoveredState);
|
|
674
|
-
await this.restoreIdleWorktree(issue);
|
|
675
|
-
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
676
|
-
if (recoveredState) {
|
|
677
|
-
this.feed?.publish({
|
|
678
|
-
level: "info",
|
|
679
|
-
kind: "stage",
|
|
680
|
-
issueKey: issue.issueKey,
|
|
681
|
-
projectId: run.projectId,
|
|
682
|
-
stage: recoveredState,
|
|
683
|
-
status: "reconciled",
|
|
684
|
-
summary: `Interrupted ${run.runType} recovered \u2192 ${recoveredState}`,
|
|
685
|
-
});
|
|
686
|
-
}
|
|
687
|
-
else {
|
|
688
|
-
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, "The Codex turn was interrupted."));
|
|
689
|
-
}
|
|
690
|
-
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
691
|
-
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
481
|
+
await this.interruptedRunRecovery.handle(run, issue);
|
|
692
482
|
return;
|
|
693
483
|
}
|
|
694
484
|
// Handle completed turn discovered during reconciliation
|
|
@@ -700,17 +490,7 @@ export class RunOrchestrator {
|
|
|
700
490
|
thread,
|
|
701
491
|
threadId: run.threadId,
|
|
702
492
|
...(latestTurn.id ? { completedTurnId: latestTurn.id } : {}),
|
|
703
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
704
|
-
releaseLease: (projectId, linearIssueId) => this.releaseIssueSessionLease(projectId, linearIssueId),
|
|
705
|
-
failRunAndClear: (targetRun, message, nextState) => this.failRunAndClear(targetRun, message, nextState),
|
|
706
|
-
verifyReactiveRunAdvancedBranch: (targetRun, targetIssue) => this.verifyReactiveRunAdvancedBranch(targetRun, targetIssue),
|
|
707
|
-
verifyReviewFixAdvancedHead: (targetRun, targetIssue) => this.verifyReviewFixAdvancedHead(targetRun, targetIssue),
|
|
708
|
-
verifyPublishedRunOutcome: (targetRun, targetIssue) => this.verifyPublishedRunOutcome(targetRun, targetIssue),
|
|
709
|
-
refreshIssueAfterReactivePublish: (targetRun, targetIssue) => this.refreshIssueAfterReactivePublish(targetRun, targetIssue),
|
|
710
|
-
resolvePostRunFollowUp: (targetRun, targetIssue) => this.resolvePostRunFollowUp(targetRun, targetIssue),
|
|
711
|
-
resolveCompletedRunState,
|
|
712
493
|
resolveRecoverableRunState: resolveRecoverablePostRunState,
|
|
713
|
-
appendWakeEventWithLease: (lease, targetIssue, runType, context, dedupeScope) => this.appendWakeEventWithLease(lease, targetIssue, runType, context, dedupeScope),
|
|
714
494
|
});
|
|
715
495
|
return;
|
|
716
496
|
}
|
|
@@ -723,7 +503,6 @@ export class RunOrchestrator {
|
|
|
723
503
|
issue,
|
|
724
504
|
runType,
|
|
725
505
|
reason,
|
|
726
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
727
506
|
});
|
|
728
507
|
}
|
|
729
508
|
failRunAndClear(run, message, nextState = "failed") {
|
|
@@ -731,342 +510,13 @@ export class RunOrchestrator {
|
|
|
731
510
|
run,
|
|
732
511
|
message,
|
|
733
512
|
nextState,
|
|
734
|
-
withHeldLease: (projectId, linearIssueId, fn) => this.withHeldIssueSessionLease(projectId, linearIssueId, fn),
|
|
735
|
-
getHeldLease: (projectId, linearIssueId) => this.getHeldIssueSessionLease(projectId, linearIssueId),
|
|
736
513
|
});
|
|
737
514
|
}
|
|
738
515
|
resolveBranchOwnerForStateTransition(newState, pendingRunType) {
|
|
739
516
|
return resolveBranchOwnerForStateTransition(newState, pendingRunType);
|
|
740
517
|
}
|
|
741
|
-
async
|
|
742
|
-
|
|
743
|
-
return undefined;
|
|
744
|
-
}
|
|
745
|
-
if (!issue.prNumber || issue.prState !== "open" || !issue.lastGitHubFailureHeadSha) {
|
|
746
|
-
return undefined;
|
|
747
|
-
}
|
|
748
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
749
|
-
if (!project?.github?.repoFullName) {
|
|
750
|
-
return undefined;
|
|
751
|
-
}
|
|
752
|
-
try {
|
|
753
|
-
const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
|
|
754
|
-
if (!pr || pr.state?.toUpperCase() !== "OPEN")
|
|
755
|
-
return undefined;
|
|
756
|
-
if (!pr.headRefOid || pr.headRefOid !== issue.lastGitHubFailureHeadSha)
|
|
757
|
-
return undefined;
|
|
758
|
-
return `Repair finished but PR #${issue.prNumber} is still on failing head ${issue.lastGitHubFailureHeadSha.slice(0, 8)}`;
|
|
759
|
-
}
|
|
760
|
-
catch (error) {
|
|
761
|
-
this.logger.debug({
|
|
762
|
-
issueKey: issue.issueKey,
|
|
763
|
-
prNumber: issue.prNumber,
|
|
764
|
-
error: error instanceof Error ? error.message : String(error),
|
|
765
|
-
}, "Failed to verify PR head advancement after repair");
|
|
766
|
-
return undefined;
|
|
767
|
-
}
|
|
768
|
-
}
|
|
769
|
-
async verifyReviewFixAdvancedHead(run, issue) {
|
|
770
|
-
if (!isRequestedChangesRunType(run.runType)) {
|
|
771
|
-
return undefined;
|
|
772
|
-
}
|
|
773
|
-
if (!issue.prNumber || issue.prState !== "open") {
|
|
774
|
-
return undefined;
|
|
775
|
-
}
|
|
776
|
-
if (!run.sourceHeadSha) {
|
|
777
|
-
return `Requested-changes run finished for PR #${issue.prNumber} without a recorded starting head SHA. PatchRelay cannot verify that a new head was published.`;
|
|
778
|
-
}
|
|
779
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
780
|
-
if (!project?.github?.repoFullName) {
|
|
781
|
-
return undefined;
|
|
782
|
-
}
|
|
783
|
-
try {
|
|
784
|
-
const pr = await this.loadRemotePrState(project.github.repoFullName, issue.prNumber);
|
|
785
|
-
if (!pr || pr.state?.toUpperCase() !== "OPEN")
|
|
786
|
-
return undefined;
|
|
787
|
-
if (!pr.headRefOid) {
|
|
788
|
-
return `Requested-changes run finished for PR #${issue.prNumber} but GitHub did not report a current head SHA.`;
|
|
789
|
-
}
|
|
790
|
-
if (pr.headRefOid === run.sourceHeadSha) {
|
|
791
|
-
return `Requested-changes run finished for PR #${issue.prNumber} without pushing a new head; PatchRelay must not hand the same SHA back to review.`;
|
|
792
|
-
}
|
|
793
|
-
return undefined;
|
|
794
|
-
}
|
|
795
|
-
catch (error) {
|
|
796
|
-
this.logger.debug({
|
|
797
|
-
issueKey: issue.issueKey,
|
|
798
|
-
prNumber: issue.prNumber,
|
|
799
|
-
error: error instanceof Error ? error.message : String(error),
|
|
800
|
-
}, "Failed to verify PR head advancement after requested-changes work");
|
|
801
|
-
return undefined;
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
|
-
async refreshIssueAfterReactivePublish(run, issue) {
|
|
805
|
-
if (run.runType !== "ci_repair" && run.runType !== "queue_repair" && !isRequestedChangesRunType(run.runType)) {
|
|
806
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
807
|
-
}
|
|
808
|
-
if (!issue.prNumber) {
|
|
809
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
810
|
-
}
|
|
811
|
-
const project = this.config.projects.find((entry) => entry.id === run.projectId);
|
|
812
|
-
const repoFullName = project?.github?.repoFullName;
|
|
813
|
-
if (!repoFullName) {
|
|
814
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
815
|
-
}
|
|
816
|
-
try {
|
|
817
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
818
|
-
if (!pr) {
|
|
819
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
820
|
-
}
|
|
821
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
822
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
823
|
-
const gateCheckName = project?.gateChecks?.find((entry) => entry.trim())?.trim() ?? "verify";
|
|
824
|
-
const headAdvanced = Boolean(pr.headRefOid && pr.headRefOid !== issue.lastGitHubFailureHeadSha);
|
|
825
|
-
const reviewFixHeadAdvanced = isRequestedChangesRunType(run.runType)
|
|
826
|
-
&& Boolean(pr.headRefOid && run.sourceHeadSha && pr.headRefOid !== run.sourceHeadSha);
|
|
827
|
-
this.upsertIssueIfLeaseHeld(run.projectId, run.linearIssueId, {
|
|
828
|
-
projectId: run.projectId,
|
|
829
|
-
linearIssueId: run.linearIssueId,
|
|
830
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
831
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
832
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
833
|
-
...((headAdvanced || reviewFixHeadAdvanced)
|
|
834
|
-
? {
|
|
835
|
-
prCheckStatus: "pending",
|
|
836
|
-
lastGitHubFailureSource: null,
|
|
837
|
-
lastGitHubFailureHeadSha: null,
|
|
838
|
-
lastGitHubFailureSignature: null,
|
|
839
|
-
lastGitHubFailureCheckName: null,
|
|
840
|
-
lastGitHubFailureCheckUrl: null,
|
|
841
|
-
lastGitHubFailureContextJson: null,
|
|
842
|
-
lastGitHubFailureAt: null,
|
|
843
|
-
lastQueueIncidentJson: null,
|
|
844
|
-
lastAttemptedFailureHeadSha: null,
|
|
845
|
-
lastAttemptedFailureSignature: null,
|
|
846
|
-
lastGitHubCiSnapshotHeadSha: pr.headRefOid ?? null,
|
|
847
|
-
lastGitHubCiSnapshotGateCheckName: gateCheckName,
|
|
848
|
-
lastGitHubCiSnapshotGateCheckStatus: "pending",
|
|
849
|
-
lastGitHubCiSnapshotJson: null,
|
|
850
|
-
lastGitHubCiSnapshotSettledAt: null,
|
|
851
|
-
}
|
|
852
|
-
: {}),
|
|
853
|
-
}, "reactive publish refresh");
|
|
854
|
-
}
|
|
855
|
-
catch (error) {
|
|
856
|
-
this.logger.debug({
|
|
857
|
-
issueKey: issue.issueKey,
|
|
858
|
-
prNumber: issue.prNumber,
|
|
859
|
-
error: error instanceof Error ? error.message : String(error),
|
|
860
|
-
}, "Failed to refresh PR state after reactive publish");
|
|
861
|
-
}
|
|
862
|
-
return this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
863
|
-
}
|
|
864
|
-
async loadRemotePrState(repoFullName, prNumber) {
|
|
865
|
-
const { stdout, exitCode } = await execCommand("gh", [
|
|
866
|
-
"pr", "view", String(prNumber),
|
|
867
|
-
"--repo", repoFullName,
|
|
868
|
-
"--json", "headRefOid,state,reviewDecision,mergeStateStatus",
|
|
869
|
-
], { timeoutMs: 10_000 });
|
|
870
|
-
if (exitCode !== 0)
|
|
871
|
-
return undefined;
|
|
872
|
-
return JSON.parse(stdout);
|
|
873
|
-
}
|
|
874
|
-
async resolveRequestedChangesWakeContext(issue, runType, context, project) {
|
|
875
|
-
if (runType === "branch_upkeep" || isBranchUpkeepRequired(context)) {
|
|
876
|
-
return context;
|
|
877
|
-
}
|
|
878
|
-
if (!issue.prNumber || issue.prState !== "open" || issue.prReviewState !== "changes_requested") {
|
|
879
|
-
return context;
|
|
880
|
-
}
|
|
881
|
-
const repoFullName = project.github?.repoFullName;
|
|
882
|
-
if (!repoFullName) {
|
|
883
|
-
return context;
|
|
884
|
-
}
|
|
885
|
-
try {
|
|
886
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
887
|
-
if (!pr)
|
|
888
|
-
return context;
|
|
889
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
890
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
891
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
892
|
-
projectId: issue.projectId,
|
|
893
|
-
linearIssueId: issue.linearIssueId,
|
|
894
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
895
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
896
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
897
|
-
}, "review-fix wake refresh");
|
|
898
|
-
if (nextPrState !== "open")
|
|
899
|
-
return context;
|
|
900
|
-
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
901
|
-
return context;
|
|
902
|
-
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
903
|
-
return context;
|
|
904
|
-
return buildReviewFixBranchUpkeepContext(issue.prNumber, project.github?.baseBranch ?? "main", pr, context);
|
|
905
|
-
}
|
|
906
|
-
catch (error) {
|
|
907
|
-
this.logger.debug({
|
|
908
|
-
issueKey: issue.issueKey,
|
|
909
|
-
prNumber: issue.prNumber,
|
|
910
|
-
error: error instanceof Error ? error.message : String(error),
|
|
911
|
-
}, "Failed to resolve requested-changes wake context");
|
|
912
|
-
return context;
|
|
913
|
-
}
|
|
914
|
-
}
|
|
915
|
-
async resolvePostRunFollowUp(run, issue, projectOverride) {
|
|
916
|
-
if (run.runType !== "review_fix") {
|
|
917
|
-
return undefined;
|
|
918
|
-
}
|
|
919
|
-
if (!issue.prNumber || issue.prState !== "open") {
|
|
920
|
-
return undefined;
|
|
921
|
-
}
|
|
922
|
-
if (issue.prReviewState !== "changes_requested") {
|
|
923
|
-
return undefined;
|
|
924
|
-
}
|
|
925
|
-
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
926
|
-
const repoFullName = project?.github?.repoFullName;
|
|
927
|
-
if (!repoFullName) {
|
|
928
|
-
return undefined;
|
|
929
|
-
}
|
|
930
|
-
try {
|
|
931
|
-
const pr = await this.loadRemotePrState(repoFullName, issue.prNumber);
|
|
932
|
-
if (!pr)
|
|
933
|
-
return undefined;
|
|
934
|
-
const nextPrState = normalizeRemotePrState(pr.state);
|
|
935
|
-
const nextReviewState = normalizeRemoteReviewDecision(pr.reviewDecision);
|
|
936
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
937
|
-
projectId: issue.projectId,
|
|
938
|
-
linearIssueId: issue.linearIssueId,
|
|
939
|
-
...(nextPrState ? { prState: nextPrState } : {}),
|
|
940
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
941
|
-
...(nextReviewState ? { prReviewState: nextReviewState } : {}),
|
|
942
|
-
}, "post-run follow-up refresh");
|
|
943
|
-
if (nextPrState !== "open")
|
|
944
|
-
return undefined;
|
|
945
|
-
if (nextReviewState && nextReviewState !== "changes_requested")
|
|
946
|
-
return undefined;
|
|
947
|
-
if (!isDirtyMergeStateStatus(pr.mergeStateStatus))
|
|
948
|
-
return undefined;
|
|
949
|
-
return {
|
|
950
|
-
pendingRunType: "branch_upkeep",
|
|
951
|
-
factoryState: "changes_requested",
|
|
952
|
-
context: buildReviewFixBranchUpkeepContext(issue.prNumber, project?.github?.baseBranch ?? "main", pr),
|
|
953
|
-
summary: `PR #${issue.prNumber} is still dirty after review fix; queued branch upkeep`,
|
|
954
|
-
};
|
|
955
|
-
}
|
|
956
|
-
catch (error) {
|
|
957
|
-
this.logger.debug({
|
|
958
|
-
issueKey: issue.issueKey,
|
|
959
|
-
prNumber: issue.prNumber,
|
|
960
|
-
error: error instanceof Error ? error.message : String(error),
|
|
961
|
-
}, "Failed to resolve post-run PR upkeep");
|
|
962
|
-
return undefined;
|
|
963
|
-
}
|
|
964
|
-
}
|
|
965
|
-
async verifyPublishedRunOutcome(run, issue, projectOverride) {
|
|
966
|
-
if (run.runType !== "implementation") {
|
|
967
|
-
return undefined;
|
|
968
|
-
}
|
|
969
|
-
const project = projectOverride ?? this.config.projects.find((entry) => entry.id === run.projectId);
|
|
970
|
-
const baseBranch = project?.github?.baseBranch ?? "main";
|
|
971
|
-
const deliveryMode = resolveImplementationDeliveryMode(issue, undefined, run.promptText);
|
|
972
|
-
if (deliveryMode === "linear_only") {
|
|
973
|
-
if (issue.prNumber !== undefined) {
|
|
974
|
-
return `Planning-only implementation should not open a PR, but PR #${issue.prNumber} was observed`;
|
|
975
|
-
}
|
|
976
|
-
return this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
977
|
-
}
|
|
978
|
-
if (issue.prNumber && issue.prState && issue.prState !== "closed") {
|
|
979
|
-
return undefined;
|
|
980
|
-
}
|
|
981
|
-
if (project?.github?.repoFullName && issue.branchName) {
|
|
982
|
-
try {
|
|
983
|
-
const { stdout, exitCode } = await execCommand("gh", [
|
|
984
|
-
"pr",
|
|
985
|
-
"list",
|
|
986
|
-
"--repo",
|
|
987
|
-
project.github.repoFullName,
|
|
988
|
-
"--head",
|
|
989
|
-
issue.branchName,
|
|
990
|
-
"--state",
|
|
991
|
-
"all",
|
|
992
|
-
"--json",
|
|
993
|
-
"number,url,state,author,headRefOid",
|
|
994
|
-
], { timeoutMs: 10_000 });
|
|
995
|
-
if (exitCode === 0) {
|
|
996
|
-
const matches = JSON.parse(stdout);
|
|
997
|
-
const pr = matches[0];
|
|
998
|
-
if (pr?.number) {
|
|
999
|
-
this.upsertIssueIfLeaseHeld(issue.projectId, issue.linearIssueId, {
|
|
1000
|
-
projectId: issue.projectId,
|
|
1001
|
-
linearIssueId: issue.linearIssueId,
|
|
1002
|
-
prNumber: pr.number,
|
|
1003
|
-
...(pr.url ? { prUrl: pr.url } : {}),
|
|
1004
|
-
...(pr.state ? { prState: pr.state.toLowerCase() } : {}),
|
|
1005
|
-
...(pr.headRefOid ? { prHeadSha: pr.headRefOid } : {}),
|
|
1006
|
-
...(pr.author?.login ? { prAuthorLogin: pr.author.login } : {}),
|
|
1007
|
-
}, "published PR verification refresh");
|
|
1008
|
-
return undefined;
|
|
1009
|
-
}
|
|
1010
|
-
}
|
|
1011
|
-
}
|
|
1012
|
-
catch (error) {
|
|
1013
|
-
this.logger.debug({
|
|
1014
|
-
issueKey: issue.issueKey,
|
|
1015
|
-
branchName: issue.branchName,
|
|
1016
|
-
repoFullName: project.github.repoFullName,
|
|
1017
|
-
error: error instanceof Error ? error.message : String(error),
|
|
1018
|
-
}, "Failed to verify published PR state after implementation");
|
|
1019
|
-
}
|
|
1020
|
-
}
|
|
1021
|
-
const details = await this.describeLocalImplementationOutcome(issue, baseBranch, deliveryMode);
|
|
1022
|
-
return details ?? `Implementation completed without opening a PR for branch ${issue.branchName ?? issue.linearIssueId}`;
|
|
1023
|
-
}
|
|
1024
|
-
async describeLocalImplementationOutcome(issue, baseBranch, deliveryMode = "publish_pr") {
|
|
1025
|
-
if (!issue.worktreePath) {
|
|
1026
|
-
return undefined;
|
|
1027
|
-
}
|
|
1028
|
-
try {
|
|
1029
|
-
const status = await execCommand(this.config.runner.gitBin, [
|
|
1030
|
-
"-C",
|
|
1031
|
-
issue.worktreePath,
|
|
1032
|
-
"status",
|
|
1033
|
-
"--short",
|
|
1034
|
-
], { timeoutMs: 10_000 });
|
|
1035
|
-
const dirtyEntries = status.exitCode === 0
|
|
1036
|
-
? status.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
1037
|
-
: [];
|
|
1038
|
-
if (dirtyEntries.length > 0) {
|
|
1039
|
-
if (deliveryMode === "linear_only") {
|
|
1040
|
-
return `Planning-only implementation should not modify the repo; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1041
|
-
}
|
|
1042
|
-
return `Implementation completed without opening a PR; worktree still has ${dirtyEntries.length} uncommitted change(s)`;
|
|
1043
|
-
}
|
|
1044
|
-
}
|
|
1045
|
-
catch {
|
|
1046
|
-
// Best effort only.
|
|
1047
|
-
}
|
|
1048
|
-
try {
|
|
1049
|
-
const ahead = await execCommand(this.config.runner.gitBin, [
|
|
1050
|
-
"-C",
|
|
1051
|
-
issue.worktreePath,
|
|
1052
|
-
"rev-list",
|
|
1053
|
-
"--count",
|
|
1054
|
-
`origin/${baseBranch}..HEAD`,
|
|
1055
|
-
], { timeoutMs: 10_000 });
|
|
1056
|
-
if (ahead.exitCode === 0) {
|
|
1057
|
-
const count = Number(ahead.stdout.trim());
|
|
1058
|
-
if (Number.isFinite(count) && count > 0) {
|
|
1059
|
-
if (deliveryMode === "linear_only") {
|
|
1060
|
-
return `Planning-only implementation should not create repo commits; worktree is ${count} local commit(s) ahead of origin/${baseBranch}`;
|
|
1061
|
-
}
|
|
1062
|
-
return `Implementation completed with ${count} local commit(s) ahead of origin/${baseBranch} but no PR was observed`;
|
|
1063
|
-
}
|
|
1064
|
-
}
|
|
1065
|
-
}
|
|
1066
|
-
catch {
|
|
1067
|
-
// Best effort only.
|
|
1068
|
-
}
|
|
1069
|
-
return undefined;
|
|
518
|
+
async resolveRequestedChangesWakeContext(issue, runType, context) {
|
|
519
|
+
return await this.runCompletionPolicy.resolveRequestedChangesWakeContext(issue, runType, context);
|
|
1070
520
|
}
|
|
1071
521
|
async readThreadWithRetry(threadId, maxRetries = 3) {
|
|
1072
522
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
@@ -1087,18 +537,6 @@ export class RunOrchestrator {
|
|
|
1087
537
|
withHeldIssueSessionLease(projectId, linearIssueId, fn) {
|
|
1088
538
|
return this.leaseService.withHeldLease(projectId, linearIssueId, fn);
|
|
1089
539
|
}
|
|
1090
|
-
upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
|
|
1091
|
-
const lease = this.getHeldIssueSessionLease(projectId, linearIssueId);
|
|
1092
|
-
if (!lease) {
|
|
1093
|
-
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write without a held issue-session lease");
|
|
1094
|
-
return undefined;
|
|
1095
|
-
}
|
|
1096
|
-
const updated = this.db.issueSessions.upsertIssueWithLease(lease, params);
|
|
1097
|
-
if (!updated) {
|
|
1098
|
-
this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
|
|
1099
|
-
}
|
|
1100
|
-
return updated;
|
|
1101
|
-
}
|
|
1102
540
|
assertLaunchLease(run, phase) {
|
|
1103
541
|
if (this.heartbeatIssueSessionLease(run.projectId, run.linearIssueId)) {
|
|
1104
542
|
return;
|
|
@@ -1108,12 +546,6 @@ export class RunOrchestrator {
|
|
|
1108
546
|
this.logger.warn({ runId: run.id, issueId: run.linearIssueId, phase }, "Aborting run launch after losing issue-session lease");
|
|
1109
547
|
throw error;
|
|
1110
548
|
}
|
|
1111
|
-
acquireIssueSessionLease(projectId, linearIssueId) {
|
|
1112
|
-
return this.leaseService.acquire(projectId, linearIssueId);
|
|
1113
|
-
}
|
|
1114
|
-
forceAcquireIssueSessionLease(projectId, linearIssueId) {
|
|
1115
|
-
return this.leaseService.forceAcquire(projectId, linearIssueId);
|
|
1116
|
-
}
|
|
1117
549
|
claimLeaseForReconciliation(projectId, linearIssueId) {
|
|
1118
550
|
return this.leaseService.claimForReconciliation(projectId, linearIssueId);
|
|
1119
551
|
}
|
|
@@ -1127,87 +559,3 @@ export class RunOrchestrator {
|
|
|
1127
559
|
this.leaseService.release(projectId, linearIssueId);
|
|
1128
560
|
}
|
|
1129
561
|
}
|
|
1130
|
-
/**
|
|
1131
|
-
* Determine post-run factory state from current PR metadata.
|
|
1132
|
-
* Used by both the normal completion path and reconciliation.
|
|
1133
|
-
*/
|
|
1134
|
-
function resolvePostRunState(issue) {
|
|
1135
|
-
if (ACTIVE_RUN_STATES.has(issue.factoryState) && issue.prNumber) {
|
|
1136
|
-
// Check merged first — a merged PR is both approved and merged,
|
|
1137
|
-
// and "done" must take priority over "awaiting_queue".
|
|
1138
|
-
if (issue.prState === "merged")
|
|
1139
|
-
return "done";
|
|
1140
|
-
if (issue.prReviewState === "approved")
|
|
1141
|
-
return "awaiting_queue";
|
|
1142
|
-
return "pr_open";
|
|
1143
|
-
}
|
|
1144
|
-
return undefined;
|
|
1145
|
-
}
|
|
1146
|
-
function resolveCompletedRunState(issue, run) {
|
|
1147
|
-
if (run.runType === "implementation" && resolveImplementationDeliveryMode(issue, undefined, run.promptText) === "linear_only") {
|
|
1148
|
-
return "done";
|
|
1149
|
-
}
|
|
1150
|
-
return resolvePostRunState(issue);
|
|
1151
|
-
}
|
|
1152
|
-
function resolveRecoverablePostRunState(issue) {
|
|
1153
|
-
if (!issue.prNumber) {
|
|
1154
|
-
return resolvePostRunState(issue);
|
|
1155
|
-
}
|
|
1156
|
-
if (issue.prState === "merged")
|
|
1157
|
-
return "done";
|
|
1158
|
-
if (issue.prState === "open") {
|
|
1159
|
-
const reactiveIntent = deriveIssueSessionReactiveIntent({
|
|
1160
|
-
prNumber: issue.prNumber,
|
|
1161
|
-
prState: issue.prState,
|
|
1162
|
-
prReviewState: issue.prReviewState,
|
|
1163
|
-
prCheckStatus: issue.prCheckStatus,
|
|
1164
|
-
latestFailureSource: issue.lastGitHubFailureSource,
|
|
1165
|
-
});
|
|
1166
|
-
if (reactiveIntent)
|
|
1167
|
-
return reactiveIntent.compatibilityFactoryState;
|
|
1168
|
-
if (issue.prReviewState === "approved")
|
|
1169
|
-
return "awaiting_queue";
|
|
1170
|
-
return "pr_open";
|
|
1171
|
-
}
|
|
1172
|
-
return resolvePostRunState(issue);
|
|
1173
|
-
}
|
|
1174
|
-
function normalizeRemotePrState(value) {
|
|
1175
|
-
const normalized = value?.trim().toUpperCase();
|
|
1176
|
-
if (normalized === "OPEN")
|
|
1177
|
-
return "open";
|
|
1178
|
-
if (normalized === "CLOSED")
|
|
1179
|
-
return "closed";
|
|
1180
|
-
if (normalized === "MERGED")
|
|
1181
|
-
return "merged";
|
|
1182
|
-
return undefined;
|
|
1183
|
-
}
|
|
1184
|
-
function normalizeRemoteReviewDecision(value) {
|
|
1185
|
-
const normalized = value?.trim().toUpperCase();
|
|
1186
|
-
if (normalized === "APPROVED")
|
|
1187
|
-
return "approved";
|
|
1188
|
-
if (normalized === "CHANGES_REQUESTED")
|
|
1189
|
-
return "changes_requested";
|
|
1190
|
-
if (normalized === "REVIEW_REQUIRED")
|
|
1191
|
-
return "commented";
|
|
1192
|
-
return undefined;
|
|
1193
|
-
}
|
|
1194
|
-
function isDirtyMergeStateStatus(value) {
|
|
1195
|
-
return value?.trim().toUpperCase() === "DIRTY";
|
|
1196
|
-
}
|
|
1197
|
-
function buildReviewFixBranchUpkeepContext(prNumber, baseBranch, pr, context) {
|
|
1198
|
-
const promptContext = [
|
|
1199
|
-
`The requested code change may already be present, but GitHub still reports PR #${prNumber} as ${String(pr.mergeStateStatus)} against latest ${baseBranch}.`,
|
|
1200
|
-
`This turn is branch upkeep on the existing PR branch: update onto latest ${baseBranch}, resolve any conflicts, rerun the narrowest relevant verification, and push a newer head.`,
|
|
1201
|
-
"Do not stop just because the requested code change is already present. Review can only move forward after a new pushed head.",
|
|
1202
|
-
].join(" ");
|
|
1203
|
-
return {
|
|
1204
|
-
...(context ?? {}),
|
|
1205
|
-
branchUpkeepRequired: true,
|
|
1206
|
-
reviewFixMode: "branch_upkeep",
|
|
1207
|
-
wakeReason: "branch_upkeep",
|
|
1208
|
-
promptContext,
|
|
1209
|
-
...(pr.mergeStateStatus ? { mergeStateStatus: pr.mergeStateStatus } : {}),
|
|
1210
|
-
...(pr.headRefOid ? { failingHeadSha: pr.headRefOid } : {}),
|
|
1211
|
-
baseBranch,
|
|
1212
|
-
};
|
|
1213
|
-
}
|