patchrelay 0.75.3 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-input-service.js +40 -26
- package/dist/build-info.json +3 -3
- package/dist/cli/data.js +3 -1
- package/dist/db/issue-session-store.js +44 -9
- package/dist/db/issue-store.js +11 -2
- package/dist/db/migrations.js +3 -0
- package/dist/factory-state.js +23 -0
- package/dist/github-webhook-reactive-run.js +15 -11
- package/dist/github-webhook-stack-coordination.js +8 -4
- package/dist/github-webhook-state-projector.js +204 -139
- package/dist/github-webhook-terminal-handler.js +37 -27
- package/dist/idle-reconciliation.js +122 -66
- package/dist/implementation-outcome-policy.js +5 -1
- package/dist/interrupted-run-recovery.js +46 -33
- package/dist/issue-session-projection-invalidator.js +9 -0
- package/dist/linear-agent-session-client.js +16 -8
- package/dist/linear-issue-projection.js +15 -11
- package/dist/linear-status-comment-sync.js +8 -4
- package/dist/linear-workflow-state-sync.js +9 -5
- package/dist/merged-linear-completion-reconciler.js +39 -17
- package/dist/no-pr-completion-check.js +51 -29
- package/dist/orchestration-parent-wake.js +15 -8
- package/dist/queue-health-monitor.js +17 -8
- package/dist/reactive-run-policy.js +5 -1
- package/dist/run-finalizer.js +61 -29
- package/dist/run-launcher.js +42 -12
- package/dist/run-notification-handler.js +19 -7
- package/dist/run-orchestrator.js +54 -20
- package/dist/run-reconciler.js +121 -50
- package/dist/run-recovery-service.js +70 -33
- package/dist/run-wake-planner.js +39 -29
- package/dist/service-issue-actions.js +45 -28
- package/dist/service-startup-recovery.js +61 -35
- package/dist/telemetry.js +9 -0
- package/dist/terminal-wake-reconciler.js +20 -3
- package/dist/webhooks/agent-session-handler.js +22 -12
- package/dist/webhooks/dependency-readiness-handler.js +17 -10
- package/dist/webhooks/desired-stage-recorder.js +32 -13
- package/dist/webhooks/issue-removal-handler.js +24 -13
- package/package.json +1 -1
package/dist/run-orchestrator.js
CHANGED
|
@@ -26,6 +26,7 @@ import { CodexThreadMaterializingError, isThreadMaterializingError } from "./cod
|
|
|
26
26
|
import { emitTelemetry, noopTelemetry } from "./telemetry.js";
|
|
27
27
|
import { LinearIssueProjectionService } from "./linear-issue-projection.js";
|
|
28
28
|
import { RunAdmissionController } from "./run-admission-controller.js";
|
|
29
|
+
const WRITER = "run-orchestrator";
|
|
29
30
|
// A terminal run must hold the active slot for at least this long before
|
|
30
31
|
// the orchestrator force-clears it, so we never race the normal
|
|
31
32
|
// notification-driven finalize that runs within seconds of completion.
|
|
@@ -224,14 +225,21 @@ export class RunOrchestrator {
|
|
|
224
225
|
try {
|
|
225
226
|
const triage = await this.issueTriage.classify({ issue, childIssues });
|
|
226
227
|
if (triage) {
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
228
|
+
// The triage verdict is an external classifier response; persist it
|
|
229
|
+
// unconditionally so a benign version bump during the (slow) triage
|
|
230
|
+
// call cannot discard the result.
|
|
231
|
+
const triageCommit = this.db.issueSessions.commitIssueState({
|
|
232
|
+
writer: WRITER,
|
|
233
|
+
update: {
|
|
234
|
+
projectId: issue.projectId,
|
|
235
|
+
linearIssueId: issue.linearIssueId,
|
|
236
|
+
issueClass: triage.issueClass,
|
|
237
|
+
issueClassSource: "triage",
|
|
238
|
+
issueTriageHash: triageHash,
|
|
239
|
+
issueTriageResultJson: JSON.stringify(triage),
|
|
240
|
+
},
|
|
234
241
|
});
|
|
242
|
+
return triageCommit.outcome === "applied" ? triageCommit.issue : issue;
|
|
235
243
|
}
|
|
236
244
|
}
|
|
237
245
|
catch (error) {
|
|
@@ -242,12 +250,22 @@ export class RunOrchestrator {
|
|
|
242
250
|
const fallbackClassification = classification.issueClassSource === "triage" && !triageCacheFresh
|
|
243
251
|
? { issueClass: "implementation", issueClassSource: "heuristic" }
|
|
244
252
|
: classification;
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
253
|
+
const fallbackCommit = this.db.issueSessions.commitIssueState({
|
|
254
|
+
writer: WRITER,
|
|
255
|
+
expectedVersion: issue.version,
|
|
256
|
+
update: {
|
|
257
|
+
projectId: issue.projectId,
|
|
258
|
+
linearIssueId: issue.linearIssueId,
|
|
259
|
+
issueClass: fallbackClassification.issueClass,
|
|
260
|
+
issueClassSource: fallbackClassification.issueClassSource,
|
|
261
|
+
},
|
|
262
|
+
// A concurrent writer is newer truth; the next pass reclassifies.
|
|
263
|
+
onConflict: () => undefined,
|
|
250
264
|
});
|
|
265
|
+
if (fallbackCommit.outcome === "applied") {
|
|
266
|
+
return fallbackCommit.issue;
|
|
267
|
+
}
|
|
268
|
+
return (fallbackCommit.outcome === "conflict_skipped" ? fallbackCommit.issue : undefined) ?? issue;
|
|
251
269
|
}
|
|
252
270
|
// ─── Run ────────────────────────────────────────────────────────
|
|
253
271
|
async run(item) {
|
|
@@ -309,7 +327,11 @@ export class RunOrchestrator {
|
|
|
309
327
|
return;
|
|
310
328
|
}
|
|
311
329
|
if (issue.prState === "merged") {
|
|
312
|
-
this.db.issueSessions.
|
|
330
|
+
this.db.issueSessions.commitIssueState({
|
|
331
|
+
writer: WRITER,
|
|
332
|
+
lease: { projectId: issue.projectId, linearIssueId: issue.linearIssueId, leaseId },
|
|
333
|
+
update: { projectId: issue.projectId, linearIssueId: issue.linearIssueId, pendingRunType: null, factoryState: "done" },
|
|
334
|
+
});
|
|
313
335
|
this.leaseService.release(item.projectId, item.issueId);
|
|
314
336
|
return;
|
|
315
337
|
}
|
|
@@ -479,11 +501,15 @@ export class RunOrchestrator {
|
|
|
479
501
|
}
|
|
480
502
|
// Reset zombie recovery counter — this run started successfully
|
|
481
503
|
if (issue.zombieRecoveryAttempts > 0) {
|
|
482
|
-
this.db.issueSessions.
|
|
483
|
-
|
|
484
|
-
linearIssueId: item.issueId,
|
|
485
|
-
|
|
486
|
-
|
|
504
|
+
this.db.issueSessions.commitIssueState({
|
|
505
|
+
writer: WRITER,
|
|
506
|
+
lease: { projectId: item.projectId, linearIssueId: item.issueId, leaseId },
|
|
507
|
+
update: {
|
|
508
|
+
projectId: item.projectId,
|
|
509
|
+
linearIssueId: item.issueId,
|
|
510
|
+
zombieRecoveryAttempts: 0,
|
|
511
|
+
lastZombieRecoveryAt: null,
|
|
512
|
+
},
|
|
487
513
|
});
|
|
488
514
|
}
|
|
489
515
|
this.logger.info({ issueKey: issue.issueKey, runType, threadId, turnId }, `Started ${runType} run`);
|
|
@@ -627,12 +653,20 @@ export class RunOrchestrator {
|
|
|
627
653
|
const fresh = this.db.issues.getIssue(run.projectId, run.linearIssueId);
|
|
628
654
|
if (!fresh || fresh.activeRunId !== run.id)
|
|
629
655
|
return false;
|
|
630
|
-
|
|
656
|
+
const danglingClear = {
|
|
631
657
|
projectId: run.projectId,
|
|
632
658
|
linearIssueId: run.linearIssueId,
|
|
633
659
|
activeRunId: null,
|
|
660
|
+
};
|
|
661
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
662
|
+
writer: WRITER,
|
|
663
|
+
lease: held,
|
|
664
|
+
expectedVersion: fresh.version,
|
|
665
|
+
update: danglingClear,
|
|
666
|
+
// Never clear a slot a concurrent writer re-pointed elsewhere.
|
|
667
|
+
onConflict: (current) => (current.activeRunId === run.id ? danglingClear : undefined),
|
|
634
668
|
});
|
|
635
|
-
return
|
|
669
|
+
return commit.outcome === "applied";
|
|
636
670
|
});
|
|
637
671
|
if (cleared) {
|
|
638
672
|
this.logger.warn({ issueKey: issue.issueKey, runId: run.id, runType: run.runType, runStatus: run.status }, "Cleared dangling active-run slot left by a terminal run; idle reconcile will resume the issue");
|
package/dist/run-reconciler.js
CHANGED
|
@@ -8,6 +8,7 @@ import { resolveEffectiveActiveRun } from "./effective-active-run.js";
|
|
|
8
8
|
import { isThreadMaterializingError } from "./codex-thread-errors.js";
|
|
9
9
|
import { fetchPullRequestSnapshot } from "./reconcile-pr-fetch.js";
|
|
10
10
|
const THREAD_MATERIALIZATION_GRACE_MS = 10 * 60_000;
|
|
11
|
+
const WRITER = "run-reconciler";
|
|
11
12
|
function isWithinThreadMaterializationGrace(run, nowMs = Date.now()) {
|
|
12
13
|
const startedAtMs = Date.parse(run.startedAt);
|
|
13
14
|
if (!Number.isFinite(startedAtMs))
|
|
@@ -50,13 +51,26 @@ export class RunReconciler {
|
|
|
50
51
|
latestRun: run,
|
|
51
52
|
});
|
|
52
53
|
if (effectiveActiveRun?.id === run.id && issue.activeRunId !== run.id) {
|
|
53
|
-
|
|
54
|
+
const reattachUpdate = {
|
|
54
55
|
projectId: run.projectId,
|
|
55
56
|
linearIssueId: run.linearIssueId,
|
|
56
57
|
activeRunId: run.id,
|
|
57
58
|
...(run.threadId ? { threadId: run.threadId } : {}),
|
|
58
|
-
}
|
|
59
|
-
this.
|
|
59
|
+
};
|
|
60
|
+
const commit = this.withHeldLease(run.projectId, run.linearIssueId, () => this.db.issueSessions.commitIssueState({
|
|
61
|
+
writer: WRITER,
|
|
62
|
+
expectedVersion: issue.version,
|
|
63
|
+
update: reattachUpdate,
|
|
64
|
+
// Never steal the slot from a run that was attached concurrently.
|
|
65
|
+
onConflict: (current) => (current.activeRunId == null ? reattachUpdate : undefined),
|
|
66
|
+
}));
|
|
67
|
+
if (commit?.outcome === "applied") {
|
|
68
|
+
effectiveIssue = commit.issue;
|
|
69
|
+
this.logger.info({ issueKey: effectiveIssue.issueKey, runId: run.id, runType: run.runType }, "Reattached detached active run during reconciliation");
|
|
70
|
+
}
|
|
71
|
+
else if (commit?.outcome === "conflict_skipped" && commit.issue) {
|
|
72
|
+
effectiveIssue = commit.issue;
|
|
73
|
+
}
|
|
60
74
|
}
|
|
61
75
|
if (!effectiveIssue.delegatedToPatchRelay) {
|
|
62
76
|
const authority = await this.confirmDelegationAuthorityBeforeRelease(run, effectiveIssue);
|
|
@@ -69,9 +83,18 @@ export class RunReconciler {
|
|
|
69
83
|
}
|
|
70
84
|
}
|
|
71
85
|
if (TERMINAL_STATES.has(effectiveIssue.factoryState)) {
|
|
86
|
+
const terminalClear = { projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null };
|
|
72
87
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
88
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
89
|
+
writer: WRITER,
|
|
90
|
+
expectedVersion: effectiveIssue.version,
|
|
91
|
+
update: terminalClear,
|
|
92
|
+
// Re-check the release predicate against the fresh row.
|
|
93
|
+
onConflict: (current) => TERMINAL_STATES.has(current.factoryState) && current.activeRunId === run.id ? terminalClear : undefined,
|
|
94
|
+
});
|
|
95
|
+
if (commit.outcome !== "applied")
|
|
96
|
+
return;
|
|
73
97
|
this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue reached terminal state during active run" });
|
|
74
|
-
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
75
98
|
});
|
|
76
99
|
this.logger.info({ issueKey: effectiveIssue.issueKey, runId: run.id, factoryState: effectiveIssue.factoryState }, "Reconciliation: released run on terminal issue");
|
|
77
100
|
const releasedIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? effectiveIssue;
|
|
@@ -88,9 +111,17 @@ export class RunReconciler {
|
|
|
88
111
|
return;
|
|
89
112
|
}
|
|
90
113
|
this.logger.warn({ issueKey: effectiveIssue.issueKey, runId: run.id, runType: run.runType }, "Zombie run detected (no thread)");
|
|
114
|
+
const zombieClear = { projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null };
|
|
91
115
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
116
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
117
|
+
writer: WRITER,
|
|
118
|
+
expectedVersion: effectiveIssue.version,
|
|
119
|
+
update: zombieClear,
|
|
120
|
+
onConflict: (current) => (current.activeRunId === run.id ? zombieClear : undefined),
|
|
121
|
+
});
|
|
122
|
+
if (commit.outcome !== "applied")
|
|
123
|
+
return;
|
|
92
124
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Zombie: never started (no thread after restart)" });
|
|
93
|
-
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
94
125
|
});
|
|
95
126
|
this.recoverOrEscalate(effectiveIssue, run.runType, "zombie");
|
|
96
127
|
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? effectiveIssue;
|
|
@@ -113,9 +144,17 @@ export class RunReconciler {
|
|
|
113
144
|
return;
|
|
114
145
|
}
|
|
115
146
|
this.logger.warn({ issueKey: effectiveIssue.issueKey, runId: run.id, runType: run.runType, threadId: run.threadId }, "Stale thread during reconciliation");
|
|
147
|
+
const staleClear = { projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null };
|
|
116
148
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
149
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
150
|
+
writer: WRITER,
|
|
151
|
+
expectedVersion: effectiveIssue.version,
|
|
152
|
+
update: staleClear,
|
|
153
|
+
onConflict: (current) => (current.activeRunId === run.id ? staleClear : undefined),
|
|
154
|
+
});
|
|
155
|
+
if (commit.outcome !== "applied")
|
|
156
|
+
return;
|
|
117
157
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: "Stale thread after restart" });
|
|
118
|
-
this.db.issues.upsertIssue({ projectId: run.projectId, linearIssueId: run.linearIssueId, activeRunId: null });
|
|
119
158
|
});
|
|
120
159
|
this.recoverOrEscalate(effectiveIssue, run.runType, "stale_thread");
|
|
121
160
|
const recoveredIssue = this.db.issues.getIssue(run.projectId, run.linearIssueId) ?? effectiveIssue;
|
|
@@ -130,15 +169,25 @@ export class RunReconciler {
|
|
|
130
169
|
if (linearIssue) {
|
|
131
170
|
const stopState = resolveAuthoritativeLinearStopState(linearIssue);
|
|
132
171
|
if (stopState?.isFinal) {
|
|
172
|
+
const stopUpdate = {
|
|
173
|
+
projectId: run.projectId,
|
|
174
|
+
linearIssueId: run.linearIssueId,
|
|
175
|
+
activeRunId: null,
|
|
176
|
+
currentLinearState: stopState.stateName,
|
|
177
|
+
factoryState: "done",
|
|
178
|
+
};
|
|
133
179
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
134
|
-
this.db.
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
180
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
181
|
+
writer: WRITER,
|
|
182
|
+
expectedVersion: effectiveIssue.version,
|
|
183
|
+
// The Linear stop state is authoritative; only the run-slot
|
|
184
|
+
// ownership needs re-checking on conflict.
|
|
185
|
+
update: stopUpdate,
|
|
186
|
+
onConflict: (current) => (current.activeRunId === run.id ? stopUpdate : undefined),
|
|
141
187
|
});
|
|
188
|
+
if (commit.outcome !== "applied")
|
|
189
|
+
return;
|
|
190
|
+
this.db.runs.finishRun(run.id, { status: "released" });
|
|
142
191
|
});
|
|
143
192
|
this.feed?.publish({
|
|
144
193
|
level: "info",
|
|
@@ -198,21 +247,31 @@ export class RunReconciler {
|
|
|
198
247
|
return true;
|
|
199
248
|
}
|
|
200
249
|
releaseMergedRun(run, issue, reason) {
|
|
250
|
+
const mergedUpdate = {
|
|
251
|
+
projectId: run.projectId,
|
|
252
|
+
linearIssueId: run.linearIssueId,
|
|
253
|
+
activeRunId: null,
|
|
254
|
+
factoryState: "done",
|
|
255
|
+
prState: "merged",
|
|
256
|
+
pendingRunType: null,
|
|
257
|
+
pendingRunContextJson: null,
|
|
258
|
+
};
|
|
201
259
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
260
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
261
|
+
writer: WRITER,
|
|
262
|
+
expectedVersion: issue.version,
|
|
263
|
+
// The merge itself is external truth; only re-check that the run
|
|
264
|
+
// slot still belongs to this run before clearing it.
|
|
265
|
+
update: mergedUpdate,
|
|
266
|
+
onConflict: (current) => (current.activeRunId === run.id ? mergedUpdate : undefined),
|
|
267
|
+
});
|
|
268
|
+
if (commit.outcome !== "applied")
|
|
269
|
+
return;
|
|
202
270
|
this.db.issueSessions.clearPendingIssueSessionEvents(run.projectId, run.linearIssueId);
|
|
203
271
|
this.db.runs.finishRun(run.id, {
|
|
204
272
|
status: "released",
|
|
205
273
|
failureReason: reason,
|
|
206
274
|
});
|
|
207
|
-
this.db.issues.upsertIssue({
|
|
208
|
-
projectId: run.projectId,
|
|
209
|
-
linearIssueId: run.linearIssueId,
|
|
210
|
-
activeRunId: null,
|
|
211
|
-
factoryState: "done",
|
|
212
|
-
prState: "merged",
|
|
213
|
-
pendingRunType: null,
|
|
214
|
-
pendingRunContextJson: null,
|
|
215
|
-
});
|
|
216
275
|
});
|
|
217
276
|
this.feed?.publish({
|
|
218
277
|
level: "info",
|
|
@@ -287,19 +346,26 @@ export class RunReconciler {
|
|
|
287
346
|
},
|
|
288
347
|
});
|
|
289
348
|
if (delegated) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
349
|
+
// Live Linear is the authority on delegation; commit unconditionally.
|
|
350
|
+
const repairedIssue = this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
351
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
352
|
+
writer: WRITER,
|
|
353
|
+
update: {
|
|
354
|
+
projectId: run.projectId,
|
|
355
|
+
linearIssueId: run.linearIssueId,
|
|
356
|
+
delegatedToPatchRelay: true,
|
|
357
|
+
...(linearIssue.identifier ? { issueKey: linearIssue.identifier } : {}),
|
|
358
|
+
...(linearIssue.title ? { title: linearIssue.title } : {}),
|
|
359
|
+
...(linearIssue.description ? { description: linearIssue.description } : {}),
|
|
360
|
+
...(linearIssue.url ? { url: linearIssue.url } : {}),
|
|
361
|
+
...(linearIssue.priority != null ? { priority: linearIssue.priority } : {}),
|
|
362
|
+
...(linearIssue.estimate != null ? { estimate: linearIssue.estimate } : {}),
|
|
363
|
+
...(linearIssue.stateName ? { currentLinearState: linearIssue.stateName } : {}),
|
|
364
|
+
...(linearIssue.stateType ? { currentLinearStateType: linearIssue.stateType } : {}),
|
|
365
|
+
},
|
|
366
|
+
});
|
|
367
|
+
return commit.outcome === "applied" ? commit.issue : undefined;
|
|
368
|
+
}) ?? issue;
|
|
303
369
|
return { issue: repairedIssue, released: false };
|
|
304
370
|
}
|
|
305
371
|
appendRunReleasedAuthorityEvent(this.db, {
|
|
@@ -315,22 +381,27 @@ export class RunReconciler {
|
|
|
315
381
|
},
|
|
316
382
|
});
|
|
317
383
|
this.withHeldLease(run.projectId, run.linearIssueId, () => {
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
384
|
+
// Undelegation confirmed against live Linear — external truth, commit
|
|
385
|
+
// unconditionally; the run release rides in the same transaction.
|
|
386
|
+
this.db.issueSessions.commitIssueState({
|
|
387
|
+
writer: WRITER,
|
|
388
|
+
update: {
|
|
389
|
+
projectId: run.projectId,
|
|
390
|
+
linearIssueId: run.linearIssueId,
|
|
391
|
+
activeRunId: null,
|
|
392
|
+
factoryState: issue.factoryState,
|
|
393
|
+
delegatedToPatchRelay: false,
|
|
394
|
+
...(linearIssue.identifier ? { issueKey: linearIssue.identifier } : {}),
|
|
395
|
+
...(linearIssue.title ? { title: linearIssue.title } : {}),
|
|
396
|
+
...(linearIssue.description ? { description: linearIssue.description } : {}),
|
|
397
|
+
...(linearIssue.url ? { url: linearIssue.url } : {}),
|
|
398
|
+
...(linearIssue.priority != null ? { priority: linearIssue.priority } : {}),
|
|
399
|
+
...(linearIssue.estimate != null ? { estimate: linearIssue.estimate } : {}),
|
|
400
|
+
...(linearIssue.stateName ? { currentLinearState: linearIssue.stateName } : {}),
|
|
401
|
+
...(linearIssue.stateType ? { currentLinearStateType: linearIssue.stateType } : {}),
|
|
402
|
+
},
|
|
333
403
|
});
|
|
404
|
+
this.db.runs.finishRun(run.id, { status: "released", failureReason: "Issue was un-delegated during active run" });
|
|
334
405
|
});
|
|
335
406
|
return { issue, released: true };
|
|
336
407
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { buildRunFailureActivity } from "./linear-session-reporting.js";
|
|
2
2
|
import { getRemainingZombieRecoveryDelayMs } from "./zombie-recovery.js";
|
|
3
|
+
const WRITER = "run-recovery-service";
|
|
3
4
|
const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
|
|
4
5
|
export class RunRecoveryService {
|
|
5
6
|
db;
|
|
@@ -30,12 +31,16 @@ export class RunRecoveryService {
|
|
|
30
31
|
if (params.isRequestedChangesRunType(runType)) {
|
|
31
32
|
const updated = this.withHeldLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
32
33
|
this.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
|
|
33
|
-
this.db.issueSessions.
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
34
|
+
this.db.issueSessions.commitIssueState({
|
|
35
|
+
writer: WRITER,
|
|
36
|
+
lease,
|
|
37
|
+
update: {
|
|
38
|
+
projectId: fresh.projectId,
|
|
39
|
+
linearIssueId: fresh.linearIssueId,
|
|
40
|
+
pendingRunType: null,
|
|
41
|
+
pendingRunContextJson: null,
|
|
42
|
+
factoryState: "escalated",
|
|
43
|
+
},
|
|
39
44
|
});
|
|
40
45
|
return true;
|
|
41
46
|
});
|
|
@@ -59,12 +64,16 @@ export class RunRecoveryService {
|
|
|
59
64
|
}
|
|
60
65
|
if (fresh.prState === "merged") {
|
|
61
66
|
const updated = this.withHeldLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
62
|
-
this.db.issueSessions.
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
this.db.issueSessions.commitIssueState({
|
|
68
|
+
writer: WRITER,
|
|
69
|
+
lease,
|
|
70
|
+
update: {
|
|
71
|
+
projectId: fresh.projectId,
|
|
72
|
+
linearIssueId: fresh.linearIssueId,
|
|
73
|
+
factoryState: "done",
|
|
74
|
+
zombieRecoveryAttempts: 0,
|
|
75
|
+
lastZombieRecoveryAt: null,
|
|
76
|
+
},
|
|
68
77
|
});
|
|
69
78
|
return true;
|
|
70
79
|
});
|
|
@@ -80,10 +89,14 @@ export class RunRecoveryService {
|
|
|
80
89
|
const attempts = fresh.zombieRecoveryAttempts + 1;
|
|
81
90
|
if (attempts > DEFAULT_ZOMBIE_RECOVERY_BUDGET) {
|
|
82
91
|
const updated = this.withHeldLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
83
|
-
this.db.issueSessions.
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
92
|
+
this.db.issueSessions.commitIssueState({
|
|
93
|
+
writer: WRITER,
|
|
94
|
+
lease,
|
|
95
|
+
update: {
|
|
96
|
+
projectId: fresh.projectId,
|
|
97
|
+
linearIssueId: fresh.linearIssueId,
|
|
98
|
+
factoryState: "escalated",
|
|
99
|
+
},
|
|
87
100
|
});
|
|
88
101
|
return true;
|
|
89
102
|
});
|
|
@@ -116,14 +129,23 @@ export class RunRecoveryService {
|
|
|
116
129
|
}
|
|
117
130
|
}
|
|
118
131
|
const requeued = this.withHeldLease(fresh.projectId, fresh.linearIssueId, (lease) => {
|
|
119
|
-
|
|
132
|
+
// `attempts` is read-modify-write against the fresh row read above; on
|
|
133
|
+
// conflict recompute the counter from the current row.
|
|
134
|
+
const buildRequeueUpdate = (record) => ({
|
|
120
135
|
projectId: fresh.projectId,
|
|
121
136
|
linearIssueId: fresh.linearIssueId,
|
|
122
137
|
pendingRunType: null,
|
|
123
138
|
pendingRunContextJson: null,
|
|
124
|
-
zombieRecoveryAttempts:
|
|
139
|
+
zombieRecoveryAttempts: record.zombieRecoveryAttempts + 1,
|
|
125
140
|
lastZombieRecoveryAt: new Date().toISOString(),
|
|
126
141
|
});
|
|
142
|
+
this.db.issueSessions.commitIssueState({
|
|
143
|
+
writer: WRITER,
|
|
144
|
+
lease,
|
|
145
|
+
expectedVersion: fresh.version,
|
|
146
|
+
update: buildRequeueUpdate(fresh),
|
|
147
|
+
onConflict: (current) => buildRequeueUpdate(current),
|
|
148
|
+
});
|
|
127
149
|
return this.appendWakeEventWithLease(lease, fresh, runType, undefined, `recovery:${attempts}`);
|
|
128
150
|
});
|
|
129
151
|
if (!requeued) {
|
|
@@ -138,18 +160,27 @@ export class RunRecoveryService {
|
|
|
138
160
|
const { issue, runType, reason } = params;
|
|
139
161
|
this.logger.warn({ issueKey: issue.issueKey, runType, reason }, "Escalating to human");
|
|
140
162
|
const escalated = this.withHeldLease(issue.projectId, issue.linearIssueId, (lease) => {
|
|
163
|
+
// Escalation is an operator-facing decision: the issue write and the
|
|
164
|
+
// run release ride in the held-lease transaction, with the run gated
|
|
165
|
+
// on the issue commit.
|
|
166
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
167
|
+
writer: WRITER,
|
|
168
|
+
lease,
|
|
169
|
+
update: {
|
|
170
|
+
projectId: issue.projectId,
|
|
171
|
+
linearIssueId: issue.linearIssueId,
|
|
172
|
+
pendingRunType: null,
|
|
173
|
+
pendingRunContextJson: null,
|
|
174
|
+
activeRunId: null,
|
|
175
|
+
factoryState: "escalated",
|
|
176
|
+
},
|
|
177
|
+
});
|
|
178
|
+
if (commit.outcome !== "applied")
|
|
179
|
+
return false;
|
|
141
180
|
if (issue.activeRunId) {
|
|
142
|
-
this.db.
|
|
181
|
+
this.db.runs.finishRun(issue.activeRunId, { status: "released" });
|
|
143
182
|
}
|
|
144
183
|
this.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
|
|
145
|
-
this.db.issueSessions.upsertIssueWithLease(lease, {
|
|
146
|
-
projectId: issue.projectId,
|
|
147
|
-
linearIssueId: issue.linearIssueId,
|
|
148
|
-
pendingRunType: null,
|
|
149
|
-
pendingRunContextJson: null,
|
|
150
|
-
activeRunId: null,
|
|
151
|
-
factoryState: "escalated",
|
|
152
|
-
});
|
|
153
184
|
return true;
|
|
154
185
|
});
|
|
155
186
|
if (!escalated) {
|
|
@@ -177,16 +208,22 @@ export class RunRecoveryService {
|
|
|
177
208
|
failRunAndClear(params) {
|
|
178
209
|
const { run, message, nextState } = params;
|
|
179
210
|
const updated = this.withHeldLease(run.projectId, run.linearIssueId, (lease) => {
|
|
211
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
212
|
+
writer: WRITER,
|
|
213
|
+
update: {
|
|
214
|
+
projectId: run.projectId,
|
|
215
|
+
linearIssueId: run.linearIssueId,
|
|
216
|
+
activeRunId: null,
|
|
217
|
+
factoryState: nextState,
|
|
218
|
+
},
|
|
219
|
+
});
|
|
220
|
+
if (commit.outcome !== "applied") {
|
|
221
|
+
return false;
|
|
222
|
+
}
|
|
180
223
|
this.db.runs.finishRun(run.id, { status: "failed", failureReason: message });
|
|
181
224
|
if (nextState === "failed" || nextState === "escalated" || nextState === "awaiting_input" || nextState === "done") {
|
|
182
225
|
this.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
|
|
183
226
|
}
|
|
184
|
-
this.db.issues.upsertIssue({
|
|
185
|
-
projectId: run.projectId,
|
|
186
|
-
linearIssueId: run.linearIssueId,
|
|
187
|
-
activeRunId: null,
|
|
188
|
-
factoryState: nextState,
|
|
189
|
-
});
|
|
190
227
|
return true;
|
|
191
228
|
});
|
|
192
229
|
if (!updated) {
|
package/dist/run-wake-planner.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { getCiRepairBudget, getQueueRepairBudget, getReviewFixBudget, } from "./run-budgets.js";
|
|
2
2
|
import { buildRequestedChangesWakeIdentity } from "./reactive-wake-keys.js";
|
|
3
|
+
const WRITER = "run-wake-planner";
|
|
3
4
|
export class RunWakePlanner {
|
|
4
5
|
db;
|
|
5
6
|
constructor(db) {
|
|
@@ -62,15 +63,19 @@ export class RunWakePlanner {
|
|
|
62
63
|
? JSON.parse(issue.pendingRunContextJson)
|
|
63
64
|
: undefined;
|
|
64
65
|
this.appendWakeEventWithLease(lease, issue, issue.pendingRunType, context, "legacy_pending");
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
67
|
+
writer: WRITER,
|
|
68
|
+
lease,
|
|
69
|
+
update: {
|
|
70
|
+
projectId: issue.projectId,
|
|
71
|
+
linearIssueId: issue.linearIssueId,
|
|
72
|
+
pendingRunType: null,
|
|
73
|
+
pendingRunContextJson: null,
|
|
74
|
+
},
|
|
70
75
|
});
|
|
71
|
-
if (
|
|
76
|
+
if (commit.outcome !== "applied")
|
|
72
77
|
return issue;
|
|
73
|
-
return
|
|
78
|
+
return commit.issue;
|
|
74
79
|
}
|
|
75
80
|
budgetExceeded(issue, project, runType, isRequestedChangesRunType) {
|
|
76
81
|
const ciRepairBudget = getCiRepairBudget(project);
|
|
@@ -88,27 +93,32 @@ export class RunWakePlanner {
|
|
|
88
93
|
return undefined;
|
|
89
94
|
}
|
|
90
95
|
incrementAttemptCounters(issue, lease, runType, isRequestedChangesRunType) {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
projectId: issue.projectId,
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
96
|
+
// The increments are read-modify-write against the issue row (which may
|
|
97
|
+
// be stale by the time the launch path gets here); on conflict, recompute
|
|
98
|
+
// from the fresh row instead of writing a counter derived from the stale
|
|
99
|
+
// read.
|
|
100
|
+
const buildIncrement = (record) => {
|
|
101
|
+
if (runType === "ci_repair") {
|
|
102
|
+
return { projectId: issue.projectId, linearIssueId: issue.linearIssueId, ciRepairAttempts: record.ciRepairAttempts + 1 };
|
|
103
|
+
}
|
|
104
|
+
if (runType === "queue_repair") {
|
|
105
|
+
return { projectId: issue.projectId, linearIssueId: issue.linearIssueId, queueRepairAttempts: record.queueRepairAttempts + 1 };
|
|
106
|
+
}
|
|
107
|
+
if (isRequestedChangesRunType(runType)) {
|
|
108
|
+
return { projectId: issue.projectId, linearIssueId: issue.linearIssueId, reviewFixAttempts: record.reviewFixAttempts + 1 };
|
|
109
|
+
}
|
|
110
|
+
return undefined;
|
|
111
|
+
};
|
|
112
|
+
const update = buildIncrement(issue);
|
|
113
|
+
if (!update)
|
|
114
|
+
return true;
|
|
115
|
+
const commit = this.db.issueSessions.commitIssueState({
|
|
116
|
+
writer: WRITER,
|
|
117
|
+
lease,
|
|
118
|
+
expectedVersion: issue.version,
|
|
119
|
+
update,
|
|
120
|
+
onConflict: (current) => buildIncrement(current),
|
|
121
|
+
});
|
|
122
|
+
return commit.outcome === "applied";
|
|
113
123
|
}
|
|
114
124
|
}
|