patchrelay 0.75.2 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-input-service.js +40 -26
- package/dist/build-info.json +3 -3
- package/dist/cli/data.js +3 -1
- package/dist/db/issue-session-store.js +44 -9
- package/dist/db/issue-store.js +31 -2
- package/dist/db/migrations.js +3 -0
- package/dist/factory-state.js +23 -0
- package/dist/github-webhook-reactive-run.js +15 -11
- package/dist/github-webhook-stack-coordination.js +8 -4
- package/dist/github-webhook-state-projector.js +204 -139
- package/dist/github-webhook-terminal-handler.js +37 -27
- package/dist/idle-reconciliation.js +122 -66
- package/dist/implementation-outcome-policy.js +5 -1
- package/dist/interrupted-run-recovery.js +46 -33
- package/dist/issue-session-projection-invalidator.js +9 -0
- package/dist/linear-agent-session-client.js +16 -8
- package/dist/linear-issue-projection.js +15 -11
- package/dist/linear-status-comment-sync.js +8 -4
- package/dist/linear-workflow-state-sync.js +9 -5
- package/dist/merged-linear-completion-reconciler.js +39 -17
- package/dist/no-pr-completion-check.js +51 -29
- package/dist/orchestration-parent-wake.js +15 -8
- package/dist/queue-health-monitor.js +17 -8
- package/dist/reactive-run-policy.js +5 -1
- package/dist/run-finalizer.js +61 -29
- package/dist/run-launcher.js +42 -12
- package/dist/run-notification-handler.js +19 -7
- package/dist/run-orchestrator.js +121 -18
- package/dist/run-reconciler.js +121 -50
- package/dist/run-recovery-service.js +70 -33
- package/dist/run-wake-planner.js +39 -29
- package/dist/service-issue-actions.js +45 -28
- package/dist/service-startup-recovery.js +61 -35
- package/dist/telemetry.js +9 -0
- package/dist/terminal-wake-reconciler.js +20 -3
- package/dist/webhooks/agent-session-handler.js +22 -12
- package/dist/webhooks/dependency-readiness-handler.js +17 -10
- package/dist/webhooks/desired-stage-recorder.js +32 -13
- package/dist/webhooks/issue-removal-handler.js +24 -13
- package/package.json +1 -1
package/dist/run-orchestrator.js
CHANGED
|
@@ -26,6 +26,11 @@ import { CodexThreadMaterializingError, isThreadMaterializingError } from "./cod
|
|
|
26
26
|
import { emitTelemetry, noopTelemetry } from "./telemetry.js";
|
|
27
27
|
import { LinearIssueProjectionService } from "./linear-issue-projection.js";
|
|
28
28
|
import { RunAdmissionController } from "./run-admission-controller.js";
|
|
29
|
+
const WRITER = "run-orchestrator";
|
|
30
|
+
// A terminal run must hold the active slot for at least this long before
|
|
31
|
+
// the orchestrator force-clears it, so we never race the normal
|
|
32
|
+
// notification-driven finalize that runs within seconds of completion.
|
|
33
|
+
const DANGLING_ACTIVE_RUN_MIN_AGE_MS = 2 * 60_000;
|
|
29
34
|
function lowerCaseFirst(value) {
|
|
30
35
|
return value ? `${value.slice(0, 1).toLowerCase()}${value.slice(1)}` : value;
|
|
31
36
|
}
|
|
@@ -220,14 +225,21 @@ export class RunOrchestrator {
|
|
|
220
225
|
try {
|
|
221
226
|
const triage = await this.issueTriage.classify({ issue, childIssues });
|
|
222
227
|
if (triage) {
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
228
|
+
// The triage verdict is an external classifier response; persist it
|
|
229
|
+
// unconditionally so a benign version bump during the (slow) triage
|
|
230
|
+
// call cannot discard the result.
|
|
231
|
+
const triageCommit = this.db.issueSessions.commitIssueState({
|
|
232
|
+
writer: WRITER,
|
|
233
|
+
update: {
|
|
234
|
+
projectId: issue.projectId,
|
|
235
|
+
linearIssueId: issue.linearIssueId,
|
|
236
|
+
issueClass: triage.issueClass,
|
|
237
|
+
issueClassSource: "triage",
|
|
238
|
+
issueTriageHash: triageHash,
|
|
239
|
+
issueTriageResultJson: JSON.stringify(triage),
|
|
240
|
+
},
|
|
230
241
|
});
|
|
242
|
+
return triageCommit.outcome === "applied" ? triageCommit.issue : issue;
|
|
231
243
|
}
|
|
232
244
|
}
|
|
233
245
|
catch (error) {
|
|
@@ -238,12 +250,22 @@ export class RunOrchestrator {
|
|
|
238
250
|
const fallbackClassification = classification.issueClassSource === "triage" && !triageCacheFresh
|
|
239
251
|
? { issueClass: "implementation", issueClassSource: "heuristic" }
|
|
240
252
|
: classification;
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
253
|
+
const fallbackCommit = this.db.issueSessions.commitIssueState({
|
|
254
|
+
writer: WRITER,
|
|
255
|
+
expectedVersion: issue.version,
|
|
256
|
+
update: {
|
|
257
|
+
projectId: issue.projectId,
|
|
258
|
+
linearIssueId: issue.linearIssueId,
|
|
259
|
+
issueClass: fallbackClassification.issueClass,
|
|
260
|
+
issueClassSource: fallbackClassification.issueClassSource,
|
|
261
|
+
},
|
|
262
|
+
// A concurrent writer is newer truth; the next pass reclassifies.
|
|
263
|
+
onConflict: () => undefined,
|
|
246
264
|
});
|
|
265
|
+
if (fallbackCommit.outcome === "applied") {
|
|
266
|
+
return fallbackCommit.issue;
|
|
267
|
+
}
|
|
268
|
+
return (fallbackCommit.outcome === "conflict_skipped" ? fallbackCommit.issue : undefined) ?? issue;
|
|
247
269
|
}
|
|
248
270
|
// ─── Run ────────────────────────────────────────────────────────
|
|
249
271
|
async run(item) {
|
|
@@ -305,7 +327,11 @@ export class RunOrchestrator {
|
|
|
305
327
|
return;
|
|
306
328
|
}
|
|
307
329
|
if (issue.prState === "merged") {
|
|
308
|
-
this.db.issueSessions.
|
|
330
|
+
this.db.issueSessions.commitIssueState({
|
|
331
|
+
writer: WRITER,
|
|
332
|
+
lease: { projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId },
|
|
333
|
+
update: { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" },
|
|
334
|
+
});
|
|
309
335
|
this.leaseService.release(item.projectId, item.issueId);
|
|
310
336
|
return;
|
|
311
337
|
}
|
|
@@ -475,11 +501,15 @@ export class RunOrchestrator {
|
|
|
475
501
|
}
|
|
476
502
|
// Reset zombie recovery counter — this run started successfully
|
|
477
503
|
if (issue.zombieRecoveryAttempts > 0) {
|
|
478
|
-
this.db.issueSessions.
|
|
479
|
-
|
|
480
|
-
linearIssueId: item.issueId,
|
|
481
|
-
|
|
482
|
-
|
|
504
|
+
this.db.issueSessions.commitIssueState({
|
|
505
|
+
writer: WRITER,
|
|
506
|
+
lease: { projectId: item.projectId, linearIssueId: item.issueId, leaseId },
|
|
507
|
+
update: {
|
|
508
|
+
projectId: item.projectId,
|
|
509
|
+
linearIssueId: item.issueId,
|
|
510
|
+
zombieRecoveryAttempts: 0,
|
|
511
|
+
lastZombieRecoveryAt: null,
|
|
512
|
+
},
|
|
483
513
|
});
|
|
484
514
|
}
|
|
485
515
|
this.logger.info({ issueKey: issue.issueKey, runType, threadId, turnId }, `Started ${runType} run`);
|
|
@@ -559,6 +589,10 @@ export class RunOrchestrator {
|
|
|
559
589
|
for (const run of this.db.runs.listRunningRuns()) {
|
|
560
590
|
await this.reconcileRun(run);
|
|
561
591
|
}
|
|
592
|
+
// Free any issue whose active slot is pinned to an already-terminal
|
|
593
|
+
// run (post-run finalize interrupted by restart). Must run before the
|
|
594
|
+
// idle reconciler so the freed issue is routed in this same pass.
|
|
595
|
+
this.finalizeDanglingActiveRuns();
|
|
562
596
|
// Preemptively detect stuck merge-queue PRs (conflicts visible on
|
|
563
597
|
// GitHub) and dispatch queue_repair before the Steward evicts.
|
|
564
598
|
await this.queueHealthMonitor.reconcile();
|
|
@@ -584,6 +618,75 @@ export class RunOrchestrator {
|
|
|
584
618
|
isRequestedChangesRunType,
|
|
585
619
|
});
|
|
586
620
|
}
|
|
621
|
+
// Clear a dangling active slot: an issue still pointing at an
|
|
622
|
+
// already-terminal run via `activeRunId`. The post-run finalize was
|
|
623
|
+
// interrupted (almost always a restart between marking the run
|
|
624
|
+
// terminal and clearing the slot), so the run can never drive the
|
|
625
|
+
// session forward, yet every idle/recovery pass skips the issue
|
|
626
|
+
// because `activeRunId` is set. We re-read under the issue-session
|
|
627
|
+
// lease and null the slot; the idle reconciler then routes the issue
|
|
628
|
+
// from GitHub truth (e.g. a missed changes_requested → review_fix).
|
|
629
|
+
finalizeDanglingActiveRuns() {
|
|
630
|
+
for (const issue of this.db.issues.listIssuesWithTerminalActiveRun()) {
|
|
631
|
+
if (issue.activeRunId === undefined)
|
|
632
|
+
continue;
|
|
633
|
+
const run = this.db.runs.getRunById(issue.activeRunId);
|
|
634
|
+
// The query already filters to terminal runs; this guards against a
|
|
635
|
+
// race where the run advanced back to active between query and read.
|
|
636
|
+
if (!run || run.status === "running" || run.status === "queued")
|
|
637
|
+
continue;
|
|
638
|
+
// Hold off until the run has been terminal long enough that the
|
|
639
|
+
// normal notification-driven finalize has demonstrably not run —
|
|
640
|
+
// avoids racing a live completion that is milliseconds from clearing
|
|
641
|
+
// the slot itself.
|
|
642
|
+
const endedAtMs = run.endedAt ? Date.parse(run.endedAt) : Number.NaN;
|
|
643
|
+
if (Number.isFinite(endedAtMs) && Date.now() - endedAtMs < DANGLING_ACTIVE_RUN_MIN_AGE_MS)
|
|
644
|
+
continue;
|
|
645
|
+
const lease = this.claimLeaseForReconciliation(run.projectId, run.linearIssueId);
|
|
646
|
+
// "skip" → a live lease owns the session (a real run is in flight);
|
|
647
|
+
// leave it alone. "owned" → an outer local scope holds it, so we
|
|
648
|
+
// must not release it here.
|
|
649
|
+
if (lease === "skip")
|
|
650
|
+
continue;
|
|
651
|
+
try {
|
|
652
|
+
const cleared = this.withHeldIssueSessionLease(run.projectId, run.linearIssueId, (held) => {
|
|
653
|
+
const fresh = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
654
|
+
if (!fresh || fresh.activeRunId !== run.id)
|
|
655
|
+
return false;
|
|
656
|
+
const danglingClear = {
|
|
657
|
+
projectId: run.projectId,
|
|
658
|
+
linearIssueId: run.linearIssueId,
|
|
659
|
+
activeRunId: null,
|
|
660
|
+
};
|
|
661
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
662
|
+
writer: WRITER,
|
|
663
|
+
lease: held,
|
|
664
|
+
expectedVersion: fresh.version,
|
|
665
|
+
update: danglingClear,
|
|
666
|
+
// Never clear a slot a concurrent writer re-pointed elsewhere.
|
|
667
|
+
onConflict: (current) => (current.activeRunId === run.id ? danglingClear : undefined),
|
|
668
|
+
});
|
|
669
|
+
return commit.outcome === "applied";
|
|
670
|
+
});
|
|
671
|
+
if (cleared) {
|
|
672
|
+
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, runStatus: run.status }, "Cleared dangling active-run slot left by a terminal run; idle reconcile will resume the issue");
|
|
673
|
+
this.feed?.publish({
|
|
674
|
+
level: "warn",
|
|
675
|
+
kind: "workflow",
|
|
676
|
+
issueKey: issue.issueKey,
|
|
677
|
+
projectId: run.projectId,
|
|
678
|
+
stage: run.runType,
|
|
679
|
+
status: "recovered",
|
|
680
|
+
summary: `Cleared stuck active slot: run #${run.id} was ${run.status} but still held the issue`,
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
finally {
|
|
685
|
+
if (lease !== "owned")
|
|
686
|
+
this.releaseIssueSessionLease(run.projectId, run.linearIssueId);
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
}
|
|
587
690
|
async reconcileRun(run) {
|
|
588
691
|
const issue = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
589
692
|
if (!issue)
|
package/dist/run-reconciler.js
CHANGED
|
@@ -8,6 +8,7 @@ import { resolveEffectiveActiveRun } from "./effective-active-run.js";
|
|
|
8
8
|
import { isThreadMaterializingError } from "./codex-thread-errors.js";
|
|
9
9
|
import { fetchPullRequestSnapshot } from "./reconcile-pr-fetch.js";
|
|
10
10
|
const THREAD_MATERIALIZATION_GRACE_MS = 10 * 60_000;
|
|
11
|
+
const WRITER = "run-reconciler";
|
|
11
12
|
function isWithinThreadMaterializationGrace(run, nowMs = Date.now()) {
|
|
12
13
|
const startedAtMs = Date.parse(run.startedAt);
|
|
13
14
|
if (!Number.isFinite(startedAtMs))
|
|
@@ -50,13 +51,26 @@ export class RunReconciler {
|
|
|
50
51
|
latestRun: run,
|
|
51
52
|
});
|
|
52
53
|
if (effectiveActiveRun?.id === run.id && issue.activeRunId !== run.id) {
|
|
53
|
-
|
|
54
|
+
const reattachUpdate = {
|
|
54
55
|
projectId: run.projectId,
|
|
55
56
|
linearIssueId: run.linearIssueId,
|
|
56
57
|
activeRunId: run.id,
|
|
57
58
|
...(run.threadId ? { threadId: run.threadId } : {}),
|
|
58
|
-
}
|
|
59
|
-
this.
|
|
59
|
+
};
|
|
60
|
+
const commit = this.withHeldLease(run.projectId, run.linearIssueId, () => this.db.issueSessions.commitIssueState({
|
|
61
|
+
writer: WRITER,
|
|
62
|
+
expectedVersion: issue.version,
|
|
63
|
+
update: reattachUpdate,
|
|
64
|
+
// Never steal the slot from a run that was attached concurrently.
|
|
65
|
+
onConflict: (current) => (current.activeRunId == null ? reattachUpdate : undefined),
|
|
66
|
+
}));
|
|
67
|
+
if (commit?.outcome === "applied") {
|
|
68
|
+
effectiveIssue = commit.issue;
|
|
69
|
+
this.logger.info({ issueKey: effectiveIssue.issueKey, runId: run.id, runType: run.runType }, "Reattached detached active run during reconciliation");
|
|
70
|
+
}
|
|
71
|
+
else if (commit?.outcome === "conflict_skipped" && commit.issue) {
|
|
72
|
+
effectiveIssue = commit.issue;
|
|
73
|
+
}
|
|
60
74
|
}
|
|
61
75
|
if (!effectiveIssue.delegatedToPatchRelay) {
|
|
62
76
|
const authority = await this.confirmDelegationAuthorityBeforeRelease(run, effectiveIssue);
|
|
@@ -69,9 +83,18 @@ export class RunReconciler {
|
|
|
69
83
|
}
|
|
70
84
|
}
|
|
71
85
|
if (TERMINAL_STATES.has(effectiveIssue.factoryState)) {
|
|
86
|
+
const terminalClear = { projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null };
|
|
72
87
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
88
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
89
|
+
writer: WRITER,
|
|
90
|
+
expectedVersion: effectiveIssue.version,
|
|
91
|
+
update: terminalClear,
|
|
92
|
+
// Re-check the release predicate against the fresh row.
|
|
93
|
+
onConflict: (current) => TERMINAL_STATES.has(current.factoryState) && current.activeRunId === run.id ? terminalClear : undefined,
|
|
94
|
+
});
|
|
95
|
+
if (commit.outcome !== "applied")
|
|
96
|
+
return;
|
|
73
97
|
this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
74
|
-
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
75
98
|
});
|
|
76
99
|
this.logger.info({ issueKey: effectiveIssue.issueKey, runId: run.id, factoryState: effectiveIssue.factoryState }, "Reconciliation: released run on terminal issue");
|
|
77
100
|
const releasedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? effectiveIssue;
|
|
@@ -88,9 +111,17 @@ export class RunReconciler {
|
|
|
88
111
|
return;
|
|
89
112
|
}
|
|
90
113
|
this.logger.warn({ issueKey: effectiveIssue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
|
|
114
|
+
const zombieClear = { projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null };
|
|
91
115
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
116
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
117
|
+
writer: WRITER,
|
|
118
|
+
expectedVersion: effectiveIssue.version,
|
|
119
|
+
update: zombieClear,
|
|
120
|
+
onConflict: (current) => (current.activeRunId === run.id ? zombieClear : undefined),
|
|
121
|
+
});
|
|
122
|
+
if (commit.outcome !== "applied")
|
|
123
|
+
return;
|
|
92
124
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
93
|
-
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
94
125
|
});
|
|
95
126
|
this.recoverOrEscalate(effectiveIssue, run.runType, "zombie");
|
|
96
127
|
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? effectiveIssue;
|
|
@@ -113,9 +144,17 @@ export class RunReconciler {
|
|
|
113
144
|
return;
|
|
114
145
|
}
|
|
115
146
|
this.logger.warn({ issueKey: effectiveIssue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
|
|
147
|
+
const staleClear = { projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null };
|
|
116
148
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
149
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
150
|
+
writer: WRITER,
|
|
151
|
+
expectedVersion: effectiveIssue.version,
|
|
152
|
+
update: staleClear,
|
|
153
|
+
onConflict: (current) => (current.activeRunId === run.id ? staleClear : undefined),
|
|
154
|
+
});
|
|
155
|
+
if (commit.outcome !== "applied")
|
|
156
|
+
return;
|
|
117
157
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
118
|
-
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
119
158
|
});
|
|
120
159
|
this.recoverOrEscalate(effectiveIssue, run.runType, "stale_thread");
|
|
121
160
|
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? effectiveIssue;
|
|
@@ -130,15 +169,25 @@ export class RunReconciler {
|
|
|
130
169
|
if (linearIssue) {
|
|
131
170
|
const stopState = resolveAuthoritativeLinearStopState(linearIssue);
|
|
132
171
|
if (stopState?.isFinal) {
|
|
172
|
+
const stopUpdate = {
|
|
173
|
+
projectId: run.projectId,
|
|
174
|
+
linearIssueId: run.linearIssueId,
|
|
175
|
+
activeRunId: null,
|
|
176
|
+
currentLinearState: stopState.stateName,
|
|
177
|
+
factoryState: "done",
|
|
178
|
+
};
|
|
133
179
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
134
|
-
this.db.
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
180
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
181
|
+
writer: WRITER,
|
|
182
|
+
expectedVersion: effectiveIssue.version,
|
|
183
|
+
// The Linear stop state is authoritative; only the run-slot
|
|
184
|
+
// ownership needs re-checking on conflict.
|
|
185
|
+
update: stopUpdate,
|
|
186
|
+
onConflict: (current) => (current.activeRunId === run.id ? stopUpdate : undefined),
|
|
141
187
|
});
|
|
188
|
+
if (commit.outcome !== "applied")
|
|
189
|
+
return;
|
|
190
|
+
this.db.runs.finishRun(run.id, { status: "released" });
|
|
142
191
|
});
|
|
143
192
|
this.feed?.publish({
|
|
144
193
|
level: "info",
|
|
@@ -198,21 +247,31 @@ export class RunReconciler {
|
|
|
198
247
|
return true;
|
|
199
248
|
}
|
|
200
249
|
releaseMergedRun(run, issue, reason) {
|
|
250
|
+
const mergedUpdate = {
|
|
251
|
+
projectId: run.projectId,
|
|
252
|
+
linearIssueId: run.linearIssueId,
|
|
253
|
+
activeRunId: null,
|
|
254
|
+
factoryState: "done",
|
|
255
|
+
prState: "merged",
|
|
256
|
+
pendingRunType: null,
|
|
257
|
+
pendingRunContextJson: null,
|
|
258
|
+
};
|
|
201
259
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
260
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
261
|
+
writer: WRITER,
|
|
262
|
+
expectedVersion: issue.version,
|
|
263
|
+
// The merge itself is external truth; only re-check that the run
|
|
264
|
+
// slot still belongs to this run before clearing it.
|
|
265
|
+
update: mergedUpdate,
|
|
266
|
+
onConflict: (current) => (current.activeRunId === run.id ? mergedUpdate : undefined),
|
|
267
|
+
});
|
|
268
|
+
if (commit.outcome !== "applied")
|
|
269
|
+
return;
|
|
202
270
|
this.db.issueSessions.clearPendingIssueSessionEvents(run.projectId, run.linearIssueId);
|
|
203
271
|
this.db.runs.finishRun(run.id, {
|
|
204
272
|
status: "released",
|
|
205
273
|
failureReason: reason,
|
|
206
274
|
});
|
|
207
|
-
this.db.issues.upsertIssue({
|
|
208
|
-
projectId: run.projectId,
|
|
209
|
-
linearIssueId: run.linearIssueId,
|
|
210
|
-
activeRunId: null,
|
|
211
|
-
factoryState: "done",
|
|
212
|
-
prState: "merged",
|
|
213
|
-
pendingRunType: null,
|
|
214
|
-
pendingRunContextJson: null,
|
|
215
|
-
});
|
|
216
275
|
});
|
|
217
276
|
this.feed?.publish({
|
|
218
277
|
level: "info",
|
|
@@ -287,19 +346,26 @@ export class RunReconciler {
|
|
|
287
346
|
},
|
|
288
347
|
});
|
|
289
348
|
if (delegated) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
349
|
+
// Live Linear is the authority on delegation; commit unconditionally.
|
|
350
|
+
const repairedIssue = this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
351
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
352
|
+
writer: WRITER,
|
|
353
|
+
update: {
|
|
354
|
+
projectId: run.projectId,
|
|
355
|
+
linearIssueId: run.linearIssueId,
|
|
356
|
+
delegatedToPatchRelay: true,
|
|
357
|
+
...(linearIssue.identifier ? { issueKey: linearIssue.identifier } : {}),
|
|
358
|
+
...(linearIssue.title ? { title: linearIssue.title } : {}),
|
|
359
|
+
...(linearIssue.description ? { description: linearIssue.description } : {}),
|
|
360
|
+
...(linearIssue.url ? { url: linearIssue.url } : {}),
|
|
361
|
+
...(linearIssue.priority != null ? { priority: linearIssue.priority } : {}),
|
|
362
|
+
...(linearIssue.estimate != null ? { estimate: linearIssue.estimate } : {}),
|
|
363
|
+
...(linearIssue.stateName ? { currentLinearState: linearIssue.stateName } : {}),
|
|
364
|
+
...(linearIssue.stateType ? { currentLinearStateType: linearIssue.stateType } : {}),
|
|
365
|
+
},
|
|
366
|
+
});
|
|
367
|
+
return commit.outcome === "applied" ? commit.issue : undefined;
|
|
368
|
+
}) ?? issue;
|
|
303
369
|
return { issue: repairedIssue, released: false };
|
|
304
370
|
}
|
|
305
371
|
appendRunReleasedAuthorityEvent(this.db, {
|
|
@@ -315,22 +381,27 @@ export class RunReconciler {
|
|
|
315
381
|
},
|
|
316
382
|
});
|
|
317
383
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
384
|
+
// Undelegation confirmed against live Linear — external truth, commit
|
|
385
|
+
// unconditionally; the run release rides in the same transaction.
|
|
386
|
+
this.db.issueSessions.commitIssueState({
|
|
387
|
+
writer: WRITER,
|
|
388
|
+
update: {
|
|
389
|
+
projectId: run.projectId,
|
|
390
|
+
linearIssueId: run.linearIssueId,
|
|
391
|
+
activeRunId: null,
|
|
392
|
+
factoryState: issue.factoryState,
|
|
393
|
+
delegatedToPatchRelay: false,
|
|
394
|
+
...(linearIssue.identifier ? { issueKey: linearIssue.identifier } : {}),
|
|
395
|
+
...(linearIssue.title ? { title: linearIssue.title } : {}),
|
|
396
|
+
...(linearIssue.description ? { description: linearIssue.description } : {}),
|
|
397
|
+
...(linearIssue.url ? { url: linearIssue.url } : {}),
|
|
398
|
+
...(linearIssue.priority != null ? { priority: linearIssue.priority } : {}),
|
|
399
|
+
...(linearIssue.estimate != null ? { estimate: linearIssue.estimate } : {}),
|
|
400
|
+
...(linearIssue.stateName ? { currentLinearState: linearIssue.stateName } : {}),
|
|
401
|
+
...(linearIssue.stateType ? { currentLinearStateType: linearIssue.stateType } : {}),
|
|
402
|
+
},
|
|
333
403
|
});
|
|
404
|
+
this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue was un-delegated during active run" });
|
|
334
405
|
});
|
|
335
406
|
return { issue, released: true };
|
|
336
407
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { buildRunFailureActivity } from "./linear-session-reporting.js";
|
|
2
2
|
import { getRemainingZombieRecoveryDelayMs } from "./zombie-recovery.js";
|
|
3
|
+
const WRITER = "run-recovery-service";
|
|
3
4
|
const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
|
|
4
5
|
export class RunRecoveryService {
|
|
5
6
|
db;
|
|
@@ -30,12 +31,16 @@ export class RunRecoveryService {
|
|
|
30
31
|
if (params.isRequestedChangesRunType(runType)) {
|
|
31
32
|
const updated = this.withHeldLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
32
33
|
this.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
|
|
33
|
-
this.db.issueSessions.
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
34
|
+
this.db.issueSessions.commitIssueState({
|
|
35
|
+
writer: WRITER,
|
|
36
|
+
lease,
|
|
37
|
+
update: {
|
|
38
|
+
projectId: fresh.projectId,
|
|
39
|
+
linearIssueId: fresh.linearIssueId,
|
|
40
|
+
pendingRunType: null,
|
|
41
|
+
pendingRunContextJson: null,
|
|
42
|
+
factoryState: "escalated",
|
|
43
|
+
},
|
|
39
44
|
});
|
|
40
45
|
return true;
|
|
41
46
|
});
|
|
@@ -59,12 +64,16 @@ export class RunRecoveryService {
|
|
|
59
64
|
}
|
|
60
65
|
if (fresh.prState === "merged") {
|
|
61
66
|
const updated = this.withHeldLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
62
|
-
this.db.issueSessions.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
this.db.issueSessions.commitIssueState({
|
|
68
|
+
writer: WRITER,
|
|
69
|
+
lease,
|
|
70
|
+
update: {
|
|
71
|
+
projectId: fresh.projectId,
|
|
72
|
+
linearIssueId: fresh.linearIssueId,
|
|
73
|
+
factoryState: "done",
|
|
74
|
+
zombieRecoveryAttempts: 0,
|
|
75
|
+
lastZombieRecoveryAt: null,
|
|
76
|
+
},
|
|
68
77
|
});
|
|
69
78
|
return true;
|
|
70
79
|
});
|
|
@@ -80,10 +89,14 @@ export class RunRecoveryService {
|
|
|
80
89
|
const attempts = fresh.zombieRecoveryAttempts + 1;
|
|
81
90
|
if (attempts > DEFAULT_ZOMBIE_RECOVERY_BUDGET) {
|
|
82
91
|
const updated = this.withHeldLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
83
|
-
this.db.issueSessions.
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
92
|
+
this.db.issueSessions.commitIssueState({
|
|
93
|
+
writer: WRITER,
|
|
94
|
+
lease,
|
|
95
|
+
update: {
|
|
96
|
+
projectId: fresh.projectId,
|
|
97
|
+
linearIssueId: fresh.linearIssueId,
|
|
98
|
+
factoryState: "escalated",
|
|
99
|
+
},
|
|
87
100
|
});
|
|
88
101
|
return true;
|
|
89
102
|
});
|
|
@@ -116,14 +129,23 @@ export class RunRecoveryService {
|
|
|
116
129
|
}
|
|
117
130
|
}
|
|
118
131
|
const requeued = this.withHeldLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
119
|
-
|
|
132
|
+
// `attempts` is read-modify-write against the fresh row read above; on
|
|
133
|
+
// conflict recompute the counter from the current row.
|
|
134
|
+
const buildRequeueUpdate = (record) => ({
|
|
120
135
|
projectId: fresh.projectId,
|
|
121
136
|
linearIssueId: fresh.linearIssueId,
|
|
122
137
|
pendingRunType: null,
|
|
123
138
|
pendingRunContextJson: null,
|
|
124
|
-
zombieRecoveryAttempts:
|
|
139
|
+
zombieRecoveryAttempts: record.zombieRecoveryAttempts + 1,
|
|
125
140
|
lastZombieRecoveryAt: new Date().toISOString(),
|
|
126
141
|
});
|
|
142
|
+
this.db.issueSessions.commitIssueState({
|
|
143
|
+
writer: WRITER,
|
|
144
|
+
lease,
|
|
145
|
+
expectedVersion: fresh.version,
|
|
146
|
+
update: buildRequeueUpdate(fresh),
|
|
147
|
+
onConflict: (current) => buildRequeueUpdate(current),
|
|
148
|
+
});
|
|
127
149
|
return this.appendWakeEventWithLease(lease, fresh, runType, undefined, `recovery:${attempts}`);
|
|
128
150
|
});
|
|
129
151
|
if (!requeued) {
|
|
@@ -138,18 +160,27 @@ export class RunRecoveryService {
|
|
|
138
160
|
const { issue, runType, reason } = params;
|
|
139
161
|
this.logger.warn({ issueKey: issue.issueKey, runType, reason }, "Escalating to human");
|
|
140
162
|
const escalated = this.withHeldLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
163
|
+
// Escalation is an operator-facing decision: the issue write and the
|
|
164
|
+
// run release ride in the held-lease transaction, with the run gated
|
|
165
|
+
// on the issue commit.
|
|
166
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
167
|
+
writer: WRITER,
|
|
168
|
+
lease,
|
|
169
|
+
update: {
|
|
170
|
+
projectId: issue.projectId,
|
|
171
|
+
linearIssueId: issue.linearIssueId,
|
|
172
|
+
pendingRunType: null,
|
|
173
|
+
pendingRunContextJson: null,
|
|
174
|
+
activeRunId: null,
|
|
175
|
+
factoryState: "escalated",
|
|
176
|
+
},
|
|
177
|
+
});
|
|
178
|
+
if (commit.outcome !== "applied")
|
|
179
|
+
return false;
|
|
141
180
|
if (issue.activeRunId) {
|
|
142
|
-
this.db.
|
|
181
|
+
this.db.runs.finishRun(issue.activeRunId, { status: "released" });
|
|
143
182
|
}
|
|
144
183
|
this.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
|
|
145
|
-
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
146
|
-
projectId: issue.projectId,
|
|
147
|
-
linearIssueId: issue.linearIssueId,
|
|
148
|
-
pendingRunType: null,
|
|
149
|
-
pendingRunContextJson: null,
|
|
150
|
-
activeRunId: null,
|
|
151
|
-
factoryState: "escalated",
|
|
152
|
-
});
|
|
153
184
|
return true;
|
|
154
185
|
});
|
|
155
186
|
if (!escalated) {
|
|
@@ -177,16 +208,22 @@ export class RunRecoveryService {
|
|
|
177
208
|
failRunAndClear(params) {
|
|
178
209
|
const { run, message, nextState } = params;
|
|
179
210
|
const updated = this.withHeldLease(run.projectId, run.linearIssueId, (lease) => {
|
|
211
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
212
|
+
writer: WRITER,
|
|
213
|
+
update: {
|
|
214
|
+
projectId: run.projectId,
|
|
215
|
+
linearIssueId: run.linearIssueId,
|
|
216
|
+
activeRunId: null,
|
|
217
|
+
factoryState: nextState,
|
|
218
|
+
},
|
|
219
|
+
});
|
|
220
|
+
if (commit.outcome !== "applied") {
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
180
223
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: message });
|
|
181
224
|
if (nextState === "failed" || nextState === "escalated" || nextState === "awaiting_input" || nextState === "done") {
|
|
182
225
|
this.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
|
|
183
226
|
}
|
|
184
|
-
this.db.issues.upsertIssue({
|
|
185
|
-
projectId: run.projectId,
|
|
186
|
-
linearIssueId: run.linearIssueId,
|
|
187
|
-
activeRunId: null,
|
|
188
|
-
factoryState: nextState,
|
|
189
|
-
});
|
|
190
227
|
return true;
|
|
191
228
|
});
|
|
192
229
|
if (!updated) {
|