patchrelay 0.36.7 → 0.36.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/cluster-health.js +1 -1
- package/dist/cli/data.js +11 -11
- package/dist/db/issue-session-store.js +292 -0
- package/dist/db/run-store.js +127 -0
- package/dist/db/webhook-event-store.js +71 -0
- package/dist/db.js +22 -520
- package/dist/github-webhook-handler.js +25 -25
- package/dist/idle-reconciliation.js +5 -5
- package/dist/issue-query-service.js +9 -9
- package/dist/issue-session-lease-service.js +143 -0
- package/dist/linear-session-sync.js +4 -4
- package/dist/queue-health-monitor.js +2 -2
- package/dist/run-finalizer.js +161 -0
- package/dist/run-launcher.js +193 -0
- package/dist/run-orchestrator.js +148 -856
- package/dist/run-recovery-service.js +203 -0
- package/dist/run-wake-planner.js +101 -0
- package/dist/service.js +24 -24
- package/dist/tracked-issue-projector.js +69 -0
- package/dist/webhook-handler.js +59 -688
- package/dist/webhooks/agent-session-handler.js +212 -0
- package/dist/webhooks/comment-policy.js +41 -0
- package/dist/webhooks/comment-wake-handler.js +133 -0
- package/dist/webhooks/decision-helpers.js +74 -0
- package/dist/webhooks/desired-stage-recorder.js +177 -0
- package/dist/webhooks/issue-removal-handler.js +68 -0
- package/package.json +1 -1
|
@@ -60,11 +60,11 @@ export class GitHubWebhookHandler {
|
|
|
60
60
|
}
|
|
61
61
|
async acceptGitHubWebhook(params) {
|
|
62
62
|
// Deduplicate
|
|
63
|
-
if (this.db.isWebhookDuplicate(params.deliveryId)) {
|
|
63
|
+
if (this.db.webhookEvents.isWebhookDuplicate(params.deliveryId)) {
|
|
64
64
|
return { status: 200, body: { ok: true, duplicate: true } };
|
|
65
65
|
}
|
|
66
66
|
// Store the event
|
|
67
|
-
const stored = this.db.insertWebhookEvent(params.deliveryId, new Date().toISOString());
|
|
67
|
+
const stored = this.db.webhookEvents.insertWebhookEvent(params.deliveryId, new Date().toISOString());
|
|
68
68
|
// Parse payload
|
|
69
69
|
const payload = safeJsonParse(params.rawBody.toString("utf8"));
|
|
70
70
|
if (!payload) {
|
|
@@ -163,7 +163,7 @@ export class GitHubWebhookHandler {
|
|
|
163
163
|
// Only transition and notify when the state actually changes.
|
|
164
164
|
// Multiple check_suite events can arrive for the same outcome.
|
|
165
165
|
if (newState && newState !== afterMetadata.factoryState) {
|
|
166
|
-
this.db.upsertIssueRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
166
|
+
this.db.issueSessions.upsertIssueRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
167
167
|
projectId: issue.projectId,
|
|
168
168
|
linearIssueId: issue.linearIssueId,
|
|
169
169
|
factoryState: newState,
|
|
@@ -179,7 +179,7 @@ export class GitHubWebhookHandler {
|
|
|
179
179
|
// Reset repair counters on new push — but only when no repair run is active,
|
|
180
180
|
// since Codex pushes during repair and resetting mid-run would bypass budgets.
|
|
181
181
|
if (event.triggerEvent === "pr_synchronize" && !freshIssue.activeRunId) {
|
|
182
|
-
this.db.upsertIssueRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
182
|
+
this.db.issueSessions.upsertIssueRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
183
183
|
projectId: issue.projectId,
|
|
184
184
|
linearIssueId: issue.linearIssueId,
|
|
185
185
|
ciRepairAttempts: 0,
|
|
@@ -340,7 +340,7 @@ export class GitHubWebhookHandler {
|
|
|
340
340
|
if (this.hasDuplicatePendingReactiveRun(issue, "queue_repair", failureContext)) {
|
|
341
341
|
return;
|
|
342
342
|
}
|
|
343
|
-
const hadPendingWake = this.db.hasPendingIssueSessionEvents(issue.projectId, issue.linearIssueId);
|
|
343
|
+
const hadPendingWake = this.db.issueSessions.hasPendingIssueSessionEvents(issue.projectId, issue.linearIssueId);
|
|
344
344
|
this.db.upsertIssue({
|
|
345
345
|
projectId: issue.projectId,
|
|
346
346
|
linearIssueId: issue.linearIssueId,
|
|
@@ -354,7 +354,7 @@ export class GitHubWebhookHandler {
|
|
|
354
354
|
lastQueueSignalAt: new Date().toISOString(),
|
|
355
355
|
lastQueueIncidentJson: JSON.stringify(queueRepairContext),
|
|
356
356
|
});
|
|
357
|
-
this.db.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
357
|
+
this.db.issueSessions.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
358
358
|
projectId: issue.projectId,
|
|
359
359
|
linearIssueId: issue.linearIssueId,
|
|
360
360
|
eventType: "merge_steward_incident",
|
|
@@ -364,7 +364,7 @@ export class GitHubWebhookHandler {
|
|
|
364
364
|
}),
|
|
365
365
|
dedupeKey: failureContext.failureSignature,
|
|
366
366
|
});
|
|
367
|
-
this.db.setBranchOwnerRespectingActiveLease(issue.projectId, issue.linearIssueId, "patchrelay");
|
|
367
|
+
this.db.issueSessions.setBranchOwnerRespectingActiveLease(issue.projectId, issue.linearIssueId, "patchrelay");
|
|
368
368
|
const queuedRunType = hadPendingWake
|
|
369
369
|
? this.peekPendingSessionWakeRunType(issue.projectId, issue.linearIssueId)
|
|
370
370
|
: this.enqueuePendingSessionWake(issue.projectId, issue.linearIssueId);
|
|
@@ -397,7 +397,7 @@ export class GitHubWebhookHandler {
|
|
|
397
397
|
if (this.hasDuplicatePendingReactiveRun(issue, "ci_repair", failureContext)) {
|
|
398
398
|
return;
|
|
399
399
|
}
|
|
400
|
-
const hadPendingWake = this.db.hasPendingIssueSessionEvents(issue.projectId, issue.linearIssueId);
|
|
400
|
+
const hadPendingWake = this.db.issueSessions.hasPendingIssueSessionEvents(issue.projectId, issue.linearIssueId);
|
|
401
401
|
const snapshot = this.getRelevantCiSnapshot(issue, event);
|
|
402
402
|
this.db.upsertIssue({
|
|
403
403
|
projectId: issue.projectId,
|
|
@@ -411,7 +411,7 @@ export class GitHubWebhookHandler {
|
|
|
411
411
|
lastGitHubFailureAt: new Date().toISOString(),
|
|
412
412
|
lastQueueIncidentJson: null,
|
|
413
413
|
});
|
|
414
|
-
this.db.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
414
|
+
this.db.issueSessions.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
415
415
|
projectId: issue.projectId,
|
|
416
416
|
linearIssueId: issue.linearIssueId,
|
|
417
417
|
eventType: "settled_red_ci",
|
|
@@ -422,7 +422,7 @@ export class GitHubWebhookHandler {
|
|
|
422
422
|
}),
|
|
423
423
|
dedupeKey: failureContext.failureSignature,
|
|
424
424
|
});
|
|
425
|
-
this.db.setBranchOwnerRespectingActiveLease(issue.projectId, issue.linearIssueId, "patchrelay");
|
|
425
|
+
this.db.issueSessions.setBranchOwnerRespectingActiveLease(issue.projectId, issue.linearIssueId, "patchrelay");
|
|
426
426
|
const queuedRunType = hadPendingWake
|
|
427
427
|
? this.peekPendingSessionWakeRunType(issue.projectId, issue.linearIssueId)
|
|
428
428
|
: this.enqueuePendingSessionWake(issue.projectId, issue.linearIssueId);
|
|
@@ -440,7 +440,7 @@ export class GitHubWebhookHandler {
|
|
|
440
440
|
}
|
|
441
441
|
}
|
|
442
442
|
if (event.triggerEvent === "review_changes_requested") {
|
|
443
|
-
const hadPendingWake = this.db.hasPendingIssueSessionEvents(issue.projectId, issue.linearIssueId);
|
|
443
|
+
const hadPendingWake = this.db.issueSessions.hasPendingIssueSessionEvents(issue.projectId, issue.linearIssueId);
|
|
444
444
|
const reviewComments = await this.fetchReviewCommentsForEvent(event).catch((error) => {
|
|
445
445
|
this.logger.warn({
|
|
446
446
|
issueKey: issue.issueKey,
|
|
@@ -450,7 +450,7 @@ export class GitHubWebhookHandler {
|
|
|
450
450
|
}, "Failed to fetch inline review comments for requested-changes event");
|
|
451
451
|
return undefined;
|
|
452
452
|
});
|
|
453
|
-
this.db.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
453
|
+
this.db.issueSessions.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
454
454
|
projectId: issue.projectId,
|
|
455
455
|
linearIssueId: issue.linearIssueId,
|
|
456
456
|
eventType: "review_changes_requested",
|
|
@@ -468,7 +468,7 @@ export class GitHubWebhookHandler {
|
|
|
468
468
|
event.reviewerName ?? "unknown-reviewer",
|
|
469
469
|
].join("::"),
|
|
470
470
|
});
|
|
471
|
-
this.db.setBranchOwnerRespectingActiveLease(issue.projectId, issue.linearIssueId, "patchrelay");
|
|
471
|
+
this.db.issueSessions.setBranchOwnerRespectingActiveLease(issue.projectId, issue.linearIssueId, "patchrelay");
|
|
472
472
|
const queuedRunType = hadPendingWake
|
|
473
473
|
? this.peekPendingSessionWakeRunType(issue.projectId, issue.linearIssueId)
|
|
474
474
|
: this.enqueuePendingSessionWake(issue.projectId, issue.linearIssueId);
|
|
@@ -489,14 +489,14 @@ export class GitHubWebhookHandler {
|
|
|
489
489
|
}
|
|
490
490
|
async handleTerminalPrEvent(issue, event) {
|
|
491
491
|
const eventType = event.triggerEvent === "pr_merged" ? "pr_merged" : "pr_closed";
|
|
492
|
-
this.db.appendIssueSessionEvent({
|
|
492
|
+
this.db.issueSessions.appendIssueSessionEvent({
|
|
493
493
|
projectId: issue.projectId,
|
|
494
494
|
linearIssueId: issue.linearIssueId,
|
|
495
495
|
eventType,
|
|
496
496
|
dedupeKey: [eventType, issue.prNumber ?? event.prNumber ?? "unknown-pr", issue.prHeadSha ?? event.headSha ?? "unknown-sha"].join("::"),
|
|
497
497
|
});
|
|
498
|
-
this.db.clearPendingIssueSessionEventsRespectingActiveLease(issue.projectId, issue.linearIssueId);
|
|
499
|
-
const run = issue.activeRunId ? this.db.
|
|
498
|
+
this.db.issueSessions.clearPendingIssueSessionEventsRespectingActiveLease(issue.projectId, issue.linearIssueId);
|
|
499
|
+
const run = issue.activeRunId ? this.db.runs.getRunById(issue.activeRunId) : undefined;
|
|
500
500
|
if (run?.threadId && run.turnId) {
|
|
501
501
|
try {
|
|
502
502
|
await this.codex.steerTurn({
|
|
@@ -513,7 +513,7 @@ export class GitHubWebhookHandler {
|
|
|
513
513
|
}
|
|
514
514
|
const commitTerminalUpdate = () => {
|
|
515
515
|
if (run) {
|
|
516
|
-
this.db.finishRun(run.id, {
|
|
516
|
+
this.db.runs.finishRun(run.id, {
|
|
517
517
|
status: "released",
|
|
518
518
|
failureReason: event.triggerEvent === "pr_merged"
|
|
519
519
|
? "Pull request merged during active run"
|
|
@@ -527,14 +527,14 @@ export class GitHubWebhookHandler {
|
|
|
527
527
|
factoryState: event.triggerEvent === "pr_merged" ? "done" : "failed",
|
|
528
528
|
});
|
|
529
529
|
};
|
|
530
|
-
const activeLease = this.db.getActiveIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
530
|
+
const activeLease = this.db.issueSessions.getActiveIssueSessionLease(issue.projectId, issue.linearIssueId);
|
|
531
531
|
if (activeLease) {
|
|
532
|
-
this.db.withIssueSessionLease(issue.projectId, issue.linearIssueId, activeLease.leaseId, commitTerminalUpdate);
|
|
532
|
+
this.db.issueSessions.withIssueSessionLease(issue.projectId, issue.linearIssueId, activeLease.leaseId, commitTerminalUpdate);
|
|
533
533
|
}
|
|
534
534
|
else {
|
|
535
535
|
this.db.transaction(commitTerminalUpdate);
|
|
536
536
|
}
|
|
537
|
-
this.db.releaseIssueSessionLeaseRespectingActiveLease(issue.projectId, issue.linearIssueId);
|
|
537
|
+
this.db.issueSessions.releaseIssueSessionLeaseRespectingActiveLease(issue.projectId, issue.linearIssueId);
|
|
538
538
|
const updatedIssue = this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
|
|
539
539
|
if (event.triggerEvent === "pr_merged") {
|
|
540
540
|
await this.completeLinearIssueAfterMerge(updatedIssue);
|
|
@@ -685,7 +685,7 @@ export class GitHubWebhookHandler {
|
|
|
685
685
|
: typeof failureContext.headSha === "string" ? failureContext.headSha : undefined;
|
|
686
686
|
if (!signature)
|
|
687
687
|
return false;
|
|
688
|
-
const pendingWake = this.db.peekIssueSessionWake(issue.projectId, issue.linearIssueId);
|
|
688
|
+
const pendingWake = this.db.issueSessions.peekIssueSessionWake(issue.projectId, issue.linearIssueId);
|
|
689
689
|
if (pendingWake?.runType === runType) {
|
|
690
690
|
const existing = pendingWake.context;
|
|
691
691
|
if (existing?.failureSignature === signature
|
|
@@ -920,7 +920,7 @@ export class GitHubWebhookHandler {
|
|
|
920
920
|
detail: body.slice(0, 200),
|
|
921
921
|
});
|
|
922
922
|
if (issue.activeRunId) {
|
|
923
|
-
const run = this.db.
|
|
923
|
+
const run = this.db.runs.getRunById(issue.activeRunId);
|
|
924
924
|
if (run?.threadId && run.turnId) {
|
|
925
925
|
try {
|
|
926
926
|
await this.codex.steerTurn({
|
|
@@ -937,7 +937,7 @@ export class GitHubWebhookHandler {
|
|
|
937
937
|
}
|
|
938
938
|
}
|
|
939
939
|
}
|
|
940
|
-
this.db.appendIssueSessionEvent({
|
|
940
|
+
this.db.issueSessions.appendIssueSessionEvent({
|
|
941
941
|
projectId: issue.projectId,
|
|
942
942
|
linearIssueId: issue.linearIssueId,
|
|
943
943
|
eventType: "followup_comment",
|
|
@@ -961,10 +961,10 @@ export class GitHubWebhookHandler {
|
|
|
961
961
|
return response.statusText || `GitHub API responded with ${response.status}`;
|
|
962
962
|
}
|
|
963
963
|
peekPendingSessionWakeRunType(projectId, issueId) {
|
|
964
|
-
return this.db.peekIssueSessionWake(projectId, issueId)?.runType;
|
|
964
|
+
return this.db.issueSessions.peekIssueSessionWake(projectId, issueId)?.runType;
|
|
965
965
|
}
|
|
966
966
|
enqueuePendingSessionWake(projectId, issueId) {
|
|
967
|
-
const wake = this.db.peekIssueSessionWake(projectId, issueId);
|
|
967
|
+
const wake = this.db.issueSessions.peekIssueSessionWake(projectId, issueId);
|
|
968
968
|
if (!wake) {
|
|
969
969
|
return undefined;
|
|
970
970
|
}
|
|
@@ -153,13 +153,13 @@ export class IdleIssueReconciler {
|
|
|
153
153
|
for (const issue of this.db.listBlockedDelegatedIssues()) {
|
|
154
154
|
const unresolved = this.db.countUnresolvedBlockers(issue.projectId, issue.linearIssueId);
|
|
155
155
|
if (unresolved === 0) {
|
|
156
|
-
this.db.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
156
|
+
this.db.issueSessions.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
157
157
|
projectId: issue.projectId,
|
|
158
158
|
linearIssueId: issue.linearIssueId,
|
|
159
159
|
eventType: "delegated",
|
|
160
160
|
dedupeKey: `delegated:${issue.linearIssueId}`,
|
|
161
161
|
});
|
|
162
|
-
if (this.db.peekIssueSessionWake(issue.projectId, issue.linearIssueId)) {
|
|
162
|
+
if (this.db.issueSessions.peekIssueSessionWake(issue.projectId, issue.linearIssueId)) {
|
|
163
163
|
this.deps.enqueueIssue(issue.projectId, issue.linearIssueId);
|
|
164
164
|
}
|
|
165
165
|
}
|
|
@@ -220,7 +220,7 @@ export class IdleIssueReconciler {
|
|
|
220
220
|
status: "reconciled",
|
|
221
221
|
summary: `Reconciliation: ${issue.factoryState} \u2192 ${newState}`,
|
|
222
222
|
});
|
|
223
|
-
if (options?.pendingRunType && this.db.peekIssueSessionWake(issue.projectId, issue.linearIssueId)) {
|
|
223
|
+
if (options?.pendingRunType && this.db.issueSessions.peekIssueSessionWake(issue.projectId, issue.linearIssueId)) {
|
|
224
224
|
this.deps.enqueueIssue(issue.projectId, issue.linearIssueId);
|
|
225
225
|
}
|
|
226
226
|
}
|
|
@@ -243,7 +243,7 @@ export class IdleIssueReconciler {
|
|
|
243
243
|
eventType = "delegated";
|
|
244
244
|
dedupeKey = `${dedupeScope}:implementation:${issue.linearIssueId}`;
|
|
245
245
|
}
|
|
246
|
-
this.db.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
246
|
+
this.db.issueSessions.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
247
247
|
projectId: issue.projectId,
|
|
248
248
|
linearIssueId: issue.linearIssueId,
|
|
249
249
|
eventType,
|
|
@@ -254,7 +254,7 @@ export class IdleIssueReconciler {
|
|
|
254
254
|
async routeFailedIssue(issue) {
|
|
255
255
|
issue = await this.refreshMissingFailureProvenance(issue);
|
|
256
256
|
issue = await this.reclassifyStaleBranchFailure(issue);
|
|
257
|
-
const latestRun = this.db.getLatestRunForIssue(issue.projectId, issue.linearIssueId);
|
|
257
|
+
const latestRun = this.db.runs.getLatestRunForIssue(issue.projectId, issue.linearIssueId);
|
|
258
258
|
const ignoreDuplicateAttempt = latestRun?.status === "failed"
|
|
259
259
|
&& latestRun.failureReason === "Codex turn was interrupted";
|
|
260
260
|
const reactiveIntent = deriveIssueSessionReactiveIntent({
|
|
@@ -29,7 +29,7 @@ export class IssueQueryService {
|
|
|
29
29
|
return await this.codex.readThread(run.threadId, true).catch(() => undefined);
|
|
30
30
|
}
|
|
31
31
|
buildRuns(projectId, linearIssueId) {
|
|
32
|
-
return this.db.listRunsForIssue(projectId, linearIssueId).map((run) => ({
|
|
32
|
+
return this.db.runs.listRunsForIssue(projectId, linearIssueId).map((run) => ({
|
|
33
33
|
id: run.id,
|
|
34
34
|
runType: run.runType,
|
|
35
35
|
status: run.status,
|
|
@@ -41,7 +41,7 @@ export class IssueQueryService {
|
|
|
41
41
|
return report ? { report } : {};
|
|
42
42
|
})(),
|
|
43
43
|
...(() => {
|
|
44
|
-
const events = this.db.listThreadEvents(run.id).flatMap((event) => {
|
|
44
|
+
const events = this.db.runs.listThreadEvents(run.id).flatMap((event) => {
|
|
45
45
|
try {
|
|
46
46
|
const parsed = JSON.parse(event.eventJson);
|
|
47
47
|
return [{
|
|
@@ -66,7 +66,7 @@ export class IssueQueryService {
|
|
|
66
66
|
}));
|
|
67
67
|
}
|
|
68
68
|
async getIssueOverview(issueKey) {
|
|
69
|
-
const session = this.db.getIssueSessionByKey(issueKey);
|
|
69
|
+
const session = this.db.issueSessions.getIssueSessionByKey(issueKey);
|
|
70
70
|
if (!session) {
|
|
71
71
|
const legacy = this.db.getIssueOverview(issueKey);
|
|
72
72
|
if (!legacy)
|
|
@@ -74,8 +74,8 @@ export class IssueQueryService {
|
|
|
74
74
|
const issueRecord = this.db.getIssueByKey(issueKey);
|
|
75
75
|
const activeStatus = await this.runStatusProvider.getActiveRunStatus(issueKey);
|
|
76
76
|
const activeRun = activeStatus?.run ?? legacy.activeRun;
|
|
77
|
-
const latestRun = this.db.getLatestRunForIssue(legacy.issue.projectId, legacy.issue.linearIssueId);
|
|
78
|
-
const latestEvent = this.db.listIssueSessionEvents(legacy.issue.projectId, legacy.issue.linearIssueId, { limit: 1 }).at(-1);
|
|
77
|
+
const latestRun = this.db.runs.getLatestRunForIssue(legacy.issue.projectId, legacy.issue.linearIssueId);
|
|
78
|
+
const latestEvent = this.db.issueSessions.listIssueSessionEvents(legacy.issue.projectId, legacy.issue.linearIssueId, { limit: 1 }).at(-1);
|
|
79
79
|
const runs = this.buildRuns(legacy.issue.projectId, legacy.issue.linearIssueId);
|
|
80
80
|
const runCount = runs.length;
|
|
81
81
|
const liveThread = await this.readLiveThread(activeRun);
|
|
@@ -130,9 +130,9 @@ export class IssueQueryService {
|
|
|
130
130
|
const blockedByKeys = unresolvedBlockedBy.map((entry) => entry.blockerIssueKey ?? entry.blockerLinearIssueId);
|
|
131
131
|
const activeStatus = await this.runStatusProvider.getActiveRunStatus(issueKey);
|
|
132
132
|
const activeRun = activeStatus?.run
|
|
133
|
-
?? (session.activeRunId !== undefined ? this.db.
|
|
134
|
-
const latestRun = this.db.getLatestRunForIssue(session.projectId, session.linearIssueId);
|
|
135
|
-
const latestEvent = this.db.listIssueSessionEvents(session.projectId, session.linearIssueId, { limit: 1 }).at(-1);
|
|
133
|
+
?? (session.activeRunId !== undefined ? this.db.runs.getRunById(session.activeRunId) : undefined);
|
|
134
|
+
const latestRun = this.db.runs.getLatestRunForIssue(session.projectId, session.linearIssueId);
|
|
135
|
+
const latestEvent = this.db.issueSessions.listIssueSessionEvents(session.projectId, session.linearIssueId, { limit: 1 }).at(-1);
|
|
136
136
|
const runs = this.buildRuns(session.projectId, session.linearIssueId);
|
|
137
137
|
const runCount = runs.length;
|
|
138
138
|
const liveThread = await this.readLiveThread(activeRun);
|
|
@@ -166,7 +166,7 @@ export class IssueQueryService {
|
|
|
166
166
|
factoryState: issueRecord?.factoryState ?? "delegated",
|
|
167
167
|
...(activeRun ? { activeRunId: activeRun.id } : {}),
|
|
168
168
|
blockedByCount: unresolvedBlockedBy.length,
|
|
169
|
-
hasPendingWake: this.db.peekIssueSessionWake(session.projectId, session.linearIssueId) !== undefined,
|
|
169
|
+
hasPendingWake: this.db.issueSessions.peekIssueSessionWake(session.projectId, session.linearIssueId) !== undefined,
|
|
170
170
|
hasLegacyPendingRun: issueRecord?.pendingRunType !== undefined,
|
|
171
171
|
...(session.prNumber !== undefined ? { prNumber: session.prNumber } : {}),
|
|
172
172
|
...(issueRecord?.prState ? { prState: issueRecord.prState } : {}),
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { getThreadTurns } from "./codex-thread-utils.js";
|
|
3
|
+
const ISSUE_SESSION_LEASE_MS = 10 * 60_000;
|
|
4
|
+
export class IssueSessionLeaseService {
|
|
5
|
+
db;
|
|
6
|
+
logger;
|
|
7
|
+
workerId;
|
|
8
|
+
readThreadWithRetry;
|
|
9
|
+
activeSessionLeases = new Map();
|
|
10
|
+
constructor(db, logger, workerId, readThreadWithRetry) {
|
|
11
|
+
this.db = db;
|
|
12
|
+
this.logger = logger;
|
|
13
|
+
this.workerId = workerId;
|
|
14
|
+
this.readThreadWithRetry = readThreadWithRetry;
|
|
15
|
+
}
|
|
16
|
+
hasLocalLease(projectId, linearIssueId) {
|
|
17
|
+
return this.activeSessionLeases.has(this.issueSessionLeaseKey(projectId, linearIssueId));
|
|
18
|
+
}
|
|
19
|
+
getHeldLease(projectId, linearIssueId) {
|
|
20
|
+
const leaseId = this.activeSessionLeases.get(this.issueSessionLeaseKey(projectId, linearIssueId));
|
|
21
|
+
if (!leaseId)
|
|
22
|
+
return undefined;
|
|
23
|
+
return { projectId, linearIssueId, leaseId };
|
|
24
|
+
}
|
|
25
|
+
withHeldLease(projectId, linearIssueId, fn) {
|
|
26
|
+
const lease = this.getHeldLease(projectId, linearIssueId);
|
|
27
|
+
if (!lease)
|
|
28
|
+
return undefined;
|
|
29
|
+
return this.db.issueSessions.withIssueSessionLease(projectId, linearIssueId, lease.leaseId, () => fn(lease));
|
|
30
|
+
}
|
|
31
|
+
acquire(projectId, linearIssueId) {
|
|
32
|
+
const leaseId = randomUUID();
|
|
33
|
+
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
34
|
+
const acquired = this.db.issueSessions.acquireIssueSessionLease({
|
|
35
|
+
projectId,
|
|
36
|
+
linearIssueId,
|
|
37
|
+
leaseId,
|
|
38
|
+
workerId: this.workerId,
|
|
39
|
+
leasedUntil,
|
|
40
|
+
});
|
|
41
|
+
if (!acquired)
|
|
42
|
+
return undefined;
|
|
43
|
+
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
44
|
+
return leaseId;
|
|
45
|
+
}
|
|
46
|
+
forceAcquire(projectId, linearIssueId) {
|
|
47
|
+
const leaseId = randomUUID();
|
|
48
|
+
const leasedUntil = new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString();
|
|
49
|
+
const acquired = this.db.issueSessions.forceAcquireIssueSessionLease({
|
|
50
|
+
projectId,
|
|
51
|
+
linearIssueId,
|
|
52
|
+
leaseId,
|
|
53
|
+
workerId: this.workerId,
|
|
54
|
+
leasedUntil,
|
|
55
|
+
});
|
|
56
|
+
if (!acquired)
|
|
57
|
+
return undefined;
|
|
58
|
+
this.activeSessionLeases.set(this.issueSessionLeaseKey(projectId, linearIssueId), leaseId);
|
|
59
|
+
return leaseId;
|
|
60
|
+
}
|
|
61
|
+
claimForReconciliation(projectId, linearIssueId) {
|
|
62
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
63
|
+
if (this.activeSessionLeases.has(key)) {
|
|
64
|
+
return "owned";
|
|
65
|
+
}
|
|
66
|
+
const session = this.db.issueSessions.getIssueSession(projectId, linearIssueId);
|
|
67
|
+
if (!session)
|
|
68
|
+
return "skip";
|
|
69
|
+
const leasedUntilMs = session.leasedUntil ? Date.parse(session.leasedUntil) : undefined;
|
|
70
|
+
if (leasedUntilMs !== undefined && Number.isFinite(leasedUntilMs) && leasedUntilMs > Date.now()) {
|
|
71
|
+
return "skip";
|
|
72
|
+
}
|
|
73
|
+
return this.acquire(projectId, linearIssueId) ? true : "skip";
|
|
74
|
+
}
|
|
75
|
+
async reclaimForeignRecoveryLeaseIfSafe(run, issue) {
|
|
76
|
+
const key = this.issueSessionLeaseKey(run.projectId, run.linearIssueId);
|
|
77
|
+
if (this.activeSessionLeases.has(key)) {
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
const session = this.db.issueSessions.getIssueSession(run.projectId, run.linearIssueId);
|
|
81
|
+
if (!session?.leaseId || !session.workerId || session.workerId === this.workerId) {
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
if (issue.activeRunId !== run.id) {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
let safeToReclaim = !run.threadId;
|
|
88
|
+
if (!safeToReclaim && run.threadId) {
|
|
89
|
+
try {
|
|
90
|
+
const thread = await this.readThreadWithRetry(run.threadId, 1);
|
|
91
|
+
const latestTurn = getThreadTurns(thread).at(-1);
|
|
92
|
+
safeToReclaim = thread.status === "notLoaded"
|
|
93
|
+
|| latestTurn?.status === "interrupted"
|
|
94
|
+
|| latestTurn?.status === "completed";
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
safeToReclaim = true;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
if (!safeToReclaim) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
const leaseId = this.forceAcquire(run.projectId, run.linearIssueId);
|
|
104
|
+
if (!leaseId) {
|
|
105
|
+
return false;
|
|
106
|
+
}
|
|
107
|
+
this.logger.info({
|
|
108
|
+
issueKey: issue.issueKey,
|
|
109
|
+
runId: run.id,
|
|
110
|
+
previousWorkerId: session.workerId,
|
|
111
|
+
previousLeaseId: session.leaseId,
|
|
112
|
+
reclaimedLeaseId: leaseId,
|
|
113
|
+
}, "Reclaimed foreign issue-session lease for active-run recovery");
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
heartbeat(projectId, linearIssueId) {
|
|
117
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
118
|
+
const leaseId = this.activeSessionLeases.get(key) ?? this.db.issueSessions.getIssueSession(projectId, linearIssueId)?.leaseId;
|
|
119
|
+
if (!leaseId)
|
|
120
|
+
return false;
|
|
121
|
+
const renewed = this.db.issueSessions.renewIssueSessionLease({
|
|
122
|
+
projectId,
|
|
123
|
+
linearIssueId,
|
|
124
|
+
leaseId,
|
|
125
|
+
leasedUntil: new Date(Date.now() + ISSUE_SESSION_LEASE_MS).toISOString(),
|
|
126
|
+
});
|
|
127
|
+
if (renewed) {
|
|
128
|
+
this.activeSessionLeases.set(key, leaseId);
|
|
129
|
+
return true;
|
|
130
|
+
}
|
|
131
|
+
this.activeSessionLeases.delete(key);
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
release(projectId, linearIssueId) {
|
|
135
|
+
const key = this.issueSessionLeaseKey(projectId, linearIssueId);
|
|
136
|
+
const leaseId = this.activeSessionLeases.get(key);
|
|
137
|
+
this.db.issueSessions.releaseIssueSessionLease(projectId, linearIssueId, leaseId);
|
|
138
|
+
this.activeSessionLeases.delete(key);
|
|
139
|
+
}
|
|
140
|
+
issueSessionLeaseKey(projectId, linearIssueId) {
|
|
141
|
+
return `${projectId}:${linearIssueId}`;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
@@ -22,7 +22,7 @@ export class LinearSessionSync {
|
|
|
22
22
|
if (issue.agentSessionId) {
|
|
23
23
|
return issue;
|
|
24
24
|
}
|
|
25
|
-
const recoveredAgentSessionId = this.db.findLatestAgentSessionIdForIssue(issue.linearIssueId);
|
|
25
|
+
const recoveredAgentSessionId = this.db.webhookEvents.findLatestAgentSessionIdForIssue(issue.linearIssueId);
|
|
26
26
|
if (!recoveredAgentSessionId)
|
|
27
27
|
return issue;
|
|
28
28
|
this.logger.info({ issueKey: issue.issueKey, agentSessionId: recoveredAgentSessionId }, "Recovered missing Linear agent session id from webhook history");
|
|
@@ -229,9 +229,9 @@ function resolveProgressActivity(notification) {
|
|
|
229
229
|
return undefined;
|
|
230
230
|
}
|
|
231
231
|
function renderStatusComment(db, issue, trackedIssue, options) {
|
|
232
|
-
const activeRun = issue.activeRunId ? db.
|
|
233
|
-
const latestRun = db.getLatestRunForIssue(issue.projectId, issue.linearIssueId);
|
|
234
|
-
const latestEvent = db.listIssueSessionEvents(issue.projectId, issue.linearIssueId, { limit: 1 }).at(-1);
|
|
232
|
+
const activeRun = issue.activeRunId ? db.runs.getRunById(issue.activeRunId) : undefined;
|
|
233
|
+
const latestRun = db.runs.getLatestRunForIssue(issue.projectId, issue.linearIssueId);
|
|
234
|
+
const latestEvent = db.issueSessions.listIssueSessionEvents(issue.projectId, issue.linearIssueId, { limit: 1 }).at(-1);
|
|
235
235
|
const activeRunType = issue.activeRunId !== undefined
|
|
236
236
|
? (options?.activeRunType ?? activeRun?.runType)
|
|
237
237
|
: undefined;
|
|
@@ -107,7 +107,7 @@ export class QueueHealthMonitor {
|
|
|
107
107
|
lastAttemptedFailureHeadSha: headRefOid,
|
|
108
108
|
lastAttemptedFailureSignature: signature,
|
|
109
109
|
});
|
|
110
|
-
this.db.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
110
|
+
this.db.issueSessions.appendIssueSessionEventRespectingActiveLease(issue.projectId, issue.linearIssueId, {
|
|
111
111
|
projectId: issue.projectId,
|
|
112
112
|
linearIssueId: issue.linearIssueId,
|
|
113
113
|
eventType: "merge_steward_incident",
|
|
@@ -115,7 +115,7 @@ export class QueueHealthMonitor {
|
|
|
115
115
|
dedupeKey: `queue_health:queue_repair:${issue.linearIssueId}:${signature}`,
|
|
116
116
|
});
|
|
117
117
|
this.advancer.advanceIdleIssue(issue, "repairing_queue");
|
|
118
|
-
if (this.db.peekIssueSessionWake(issue.projectId, issue.linearIssueId)) {
|
|
118
|
+
if (this.db.issueSessions.peekIssueSessionWake(issue.projectId, issue.linearIssueId)) {
|
|
119
119
|
this.advancer.enqueueIssue(issue.projectId, issue.linearIssueId);
|
|
120
120
|
}
|
|
121
121
|
this.logger.info({ issueKey: issue.issueKey, prNumber: issue.prNumber, headRefOid, reason }, "Queue health: queue issue detected, dispatching repair");
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { buildStageReport, countEventMethods } from "./run-reporting.js";
|
|
2
|
+
import { buildRunCompletedActivity, buildRunFailureActivity } from "./linear-session-reporting.js";
|
|
3
|
+
export class RunFinalizer {
|
|
4
|
+
db;
|
|
5
|
+
logger;
|
|
6
|
+
linearSync;
|
|
7
|
+
enqueueIssue;
|
|
8
|
+
feed;
|
|
9
|
+
constructor(db, logger, linearSync, enqueueIssue, feed) {
|
|
10
|
+
this.db = db;
|
|
11
|
+
this.logger = logger;
|
|
12
|
+
this.linearSync = linearSync;
|
|
13
|
+
this.enqueueIssue = enqueueIssue;
|
|
14
|
+
this.feed = feed;
|
|
15
|
+
}
|
|
16
|
+
async finalizeCompletedRun(params) {
|
|
17
|
+
const { run, issue, thread, threadId } = params;
|
|
18
|
+
const trackedIssue = this.db.issueToTrackedIssue(issue);
|
|
19
|
+
const report = buildStageReport({ ...run, status: "completed" }, trackedIssue, thread, countEventMethods(this.db.runs.listThreadEvents(run.id)));
|
|
20
|
+
const freshIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
21
|
+
const verifiedRepairError = await params.verifyReactiveRunAdvancedBranch(run, freshIssue);
|
|
22
|
+
if (verifiedRepairError) {
|
|
23
|
+
const holdState = params.resolveRecoverableRunState(freshIssue) ?? "failed";
|
|
24
|
+
params.failRunAndClear(run, verifiedRepairError, holdState);
|
|
25
|
+
const heldIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
26
|
+
this.feed?.publish({
|
|
27
|
+
level: "warn",
|
|
28
|
+
kind: "turn",
|
|
29
|
+
issueKey: freshIssue.issueKey,
|
|
30
|
+
projectId: run.projectId,
|
|
31
|
+
stage: run.runType,
|
|
32
|
+
status: "branch_not_advanced",
|
|
33
|
+
summary: verifiedRepairError,
|
|
34
|
+
});
|
|
35
|
+
void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
|
|
36
|
+
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
37
|
+
this.linearSync.clearProgress(run.id);
|
|
38
|
+
params.releaseLease(run.projectId, run.linearIssueId);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
const missingReviewFixHeadError = await params.verifyReviewFixAdvancedHead(run, freshIssue);
|
|
42
|
+
if (missingReviewFixHeadError) {
|
|
43
|
+
params.failRunAndClear(run, missingReviewFixHeadError, "escalated");
|
|
44
|
+
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
45
|
+
this.feed?.publish({
|
|
46
|
+
level: "error",
|
|
47
|
+
kind: "turn",
|
|
48
|
+
issueKey: freshIssue.issueKey,
|
|
49
|
+
projectId: run.projectId,
|
|
50
|
+
stage: run.runType,
|
|
51
|
+
status: "same_head_review_handoff_blocked",
|
|
52
|
+
summary: missingReviewFixHeadError,
|
|
53
|
+
});
|
|
54
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, missingReviewFixHeadError));
|
|
55
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
56
|
+
this.linearSync.clearProgress(run.id);
|
|
57
|
+
params.releaseLease(run.projectId, run.linearIssueId);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const publishedOutcomeError = await params.verifyPublishedRunOutcome(run, freshIssue);
|
|
61
|
+
if (publishedOutcomeError) {
|
|
62
|
+
params.failRunAndClear(run, publishedOutcomeError, "failed");
|
|
63
|
+
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? freshIssue;
|
|
64
|
+
this.feed?.publish({
|
|
65
|
+
level: "warn",
|
|
66
|
+
kind: "turn",
|
|
67
|
+
issueKey: freshIssue.issueKey,
|
|
68
|
+
projectId: run.projectId,
|
|
69
|
+
stage: run.runType,
|
|
70
|
+
status: "publish_incomplete",
|
|
71
|
+
summary: publishedOutcomeError,
|
|
72
|
+
});
|
|
73
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, publishedOutcomeError));
|
|
74
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
75
|
+
this.linearSync.clearProgress(run.id);
|
|
76
|
+
if (params.source === "notification") {
|
|
77
|
+
params.releaseLease(run.projectId, run.linearIssueId);
|
|
78
|
+
}
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
const refreshedIssue = await params.refreshIssueAfterReactivePublish(run, freshIssue);
|
|
82
|
+
const postRunFollowUp = await params.resolvePostRunFollowUp(run, refreshedIssue);
|
|
83
|
+
const postRunState = postRunFollowUp?.factoryState ?? params.resolveCompletedRunState(refreshedIssue, run);
|
|
84
|
+
const completed = params.withHeldLease(run.projectId, run.linearIssueId, (lease) => {
|
|
85
|
+
this.db.runs.finishRun(run.id, {
|
|
86
|
+
status: "completed",
|
|
87
|
+
threadId,
|
|
88
|
+
...(params.completedTurnId ? { turnId: params.completedTurnId } : {}),
|
|
89
|
+
summaryJson: JSON.stringify({ latestAssistantMessage: report.assistantMessages.at(-1) ?? null }),
|
|
90
|
+
reportJson: JSON.stringify(report),
|
|
91
|
+
});
|
|
92
|
+
this.db.upsertIssue({
|
|
93
|
+
projectId: run.projectId,
|
|
94
|
+
linearIssueId: run.linearIssueId,
|
|
95
|
+
activeRunId: null,
|
|
96
|
+
...(postRunState ? { factoryState: postRunState } : {}),
|
|
97
|
+
pendingRunType: null,
|
|
98
|
+
pendingRunContextJson: null,
|
|
99
|
+
...(postRunFollowUp ? {} : (postRunState === "awaiting_queue" || postRunState === "done"
|
|
100
|
+
? {
|
|
101
|
+
lastGitHubFailureSource: null,
|
|
102
|
+
lastGitHubFailureHeadSha: null,
|
|
103
|
+
lastGitHubFailureSignature: null,
|
|
104
|
+
lastGitHubFailureCheckName: null,
|
|
105
|
+
lastGitHubFailureCheckUrl: null,
|
|
106
|
+
lastGitHubFailureContextJson: null,
|
|
107
|
+
lastGitHubFailureAt: null,
|
|
108
|
+
lastQueueIncidentJson: null,
|
|
109
|
+
lastAttemptedFailureHeadSha: null,
|
|
110
|
+
lastAttemptedFailureSignature: null,
|
|
111
|
+
}
|
|
112
|
+
: {})),
|
|
113
|
+
});
|
|
114
|
+
if (postRunFollowUp) {
|
|
115
|
+
return params.appendWakeEventWithLease(lease, issue, postRunFollowUp.pendingRunType, postRunFollowUp.context, "post_run");
|
|
116
|
+
}
|
|
117
|
+
return true;
|
|
118
|
+
});
|
|
119
|
+
if (!completed) {
|
|
120
|
+
this.logger.warn({ runId: run.id, issueId: run.linearIssueId }, "Skipping completion writes after losing issue-session lease");
|
|
121
|
+
this.linearSync.clearProgress(run.id);
|
|
122
|
+
params.releaseLease(run.projectId, run.linearIssueId);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
if (postRunFollowUp) {
|
|
126
|
+
this.feed?.publish({
|
|
127
|
+
level: "info",
|
|
128
|
+
kind: "stage",
|
|
129
|
+
issueKey: issue.issueKey,
|
|
130
|
+
projectId: run.projectId,
|
|
131
|
+
stage: postRunFollowUp.factoryState,
|
|
132
|
+
status: "follow_up_queued",
|
|
133
|
+
summary: postRunFollowUp.summary,
|
|
134
|
+
});
|
|
135
|
+
this.enqueueIssue(run.projectId, run.linearIssueId);
|
|
136
|
+
}
|
|
137
|
+
this.feed?.publish({
|
|
138
|
+
level: "info",
|
|
139
|
+
kind: "turn",
|
|
140
|
+
issueKey: issue.issueKey,
|
|
141
|
+
projectId: run.projectId,
|
|
142
|
+
stage: run.runType,
|
|
143
|
+
status: "completed",
|
|
144
|
+
summary: params.source === "notification"
|
|
145
|
+
? `Turn completed for ${run.runType}`
|
|
146
|
+
: `Reconciliation: ${run.runType} completed${postRunState ? ` -> ${postRunState}` : ""}`,
|
|
147
|
+
...(report.assistantMessages.at(-1) ? { detail: report.assistantMessages.at(-1) } : {}),
|
|
148
|
+
});
|
|
149
|
+
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? refreshedIssue;
|
|
150
|
+
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
151
|
+
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
152
|
+
runType: run.runType,
|
|
153
|
+
completionSummary,
|
|
154
|
+
postRunState: updatedIssue.factoryState,
|
|
155
|
+
...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
|
|
156
|
+
}));
|
|
157
|
+
void this.linearSync.syncSession(updatedIssue);
|
|
158
|
+
this.linearSync.clearProgress(run.id);
|
|
159
|
+
params.releaseLease(run.projectId, run.linearIssueId);
|
|
160
|
+
}
|
|
161
|
+
}
|