patchrelay 0.35.6 → 0.35.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/idle-reconciliation.js +262 -0
- package/dist/linear-session-sync.js +143 -0
- package/dist/queue-health-monitor.js +131 -0
- package/dist/run-orchestrator.js +40 -497
- package/dist/webhook-handler.js +92 -57
- package/package.json +1 -1
package/dist/run-orchestrator.js
CHANGED
|
@@ -1,14 +1,10 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import { ACTIVE_RUN_STATES, TERMINAL_STATES } from "./factory-state.js";
|
|
4
|
-
import { parseGitHubFailureContext } from "./github-failure-context.js";
|
|
5
4
|
import { buildHookEnv, runProjectHook } from "./hook-runner.js";
|
|
6
|
-
import { buildAgentSessionPlanForIssue, } from "./agent-session-plan.js";
|
|
7
5
|
import { buildStageReport, countEventMethods, extractTurnId, resolveRunCompletionStatus, summarizeCurrentThread, } from "./run-reporting.js";
|
|
8
6
|
import { buildRunCompletedActivity, buildRunFailureActivity, buildRunStartedActivity, } from "./linear-session-reporting.js";
|
|
9
7
|
import { requestMergeQueueAdmission, resolveMergeQueueProtocol, } from "./merge-queue-protocol.js";
|
|
10
|
-
import { parseStoredQueueRepairContext } from "./merge-queue-incident.js";
|
|
11
|
-
import { buildAgentSessionExternalUrls } from "./agent-session-presentation.js";
|
|
12
8
|
import { WorktreeManager } from "./worktree-manager.js";
|
|
13
9
|
import { resolveAuthoritativeLinearStopState } from "./linear-workflow.js";
|
|
14
10
|
import { execCommand } from "./utils.js";
|
|
@@ -17,12 +13,9 @@ const DEFAULT_QUEUE_REPAIR_BUDGET = 3;
|
|
|
17
13
|
const DEFAULT_REVIEW_FIX_BUDGET = 3;
|
|
18
14
|
const DEFAULT_ZOMBIE_RECOVERY_BUDGET = 5;
|
|
19
15
|
const ZOMBIE_RECOVERY_BASE_DELAY_MS = 15_000; // 15s, 30s, 60s, 120s, 240s
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const QUEUE_HEALTH_GRACE_MS = 120_000;
|
|
24
|
-
// Suppress repeated probe-failure feed events — at most one per issue per window.
|
|
25
|
-
const QUEUE_HEALTH_PROBE_FAILURE_COOLDOWN_MS = 300_000; // 5 minutes
|
|
16
|
+
import { QueueHealthMonitor } from "./queue-health-monitor.js";
|
|
17
|
+
import { IdleIssueReconciler, resolveBranchOwnerForStateTransition } from "./idle-reconciliation.js";
|
|
18
|
+
import { LinearSessionSync } from "./linear-session-sync.js";
|
|
26
19
|
function slugify(value) {
|
|
27
20
|
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 60);
|
|
28
21
|
}
|
|
@@ -93,7 +86,6 @@ function buildRunPrompt(issue, runType, repoPath, context) {
|
|
|
93
86
|
}
|
|
94
87
|
return lines.join("\n");
|
|
95
88
|
}
|
|
96
|
-
const PROGRESS_THROTTLE_MS = 10_000;
|
|
97
89
|
export class RunOrchestrator {
|
|
98
90
|
config;
|
|
99
91
|
db;
|
|
@@ -103,9 +95,10 @@ export class RunOrchestrator {
|
|
|
103
95
|
logger;
|
|
104
96
|
feed;
|
|
105
97
|
worktreeManager;
|
|
106
|
-
progressThrottle = new Map();
|
|
107
98
|
/** Tracks last probe-failure feed event per issue to avoid spamming the operator feed. */
|
|
108
|
-
|
|
99
|
+
queueHealthMonitor;
|
|
100
|
+
idleReconciler;
|
|
101
|
+
linearSync;
|
|
109
102
|
activeThreadId;
|
|
110
103
|
botIdentity;
|
|
111
104
|
constructor(config, db, codex, linearProvider, enqueueIssue, logger, feed) {
|
|
@@ -117,6 +110,14 @@ export class RunOrchestrator {
|
|
|
117
110
|
this.logger = logger;
|
|
118
111
|
this.feed = feed;
|
|
119
112
|
this.worktreeManager = new WorktreeManager(config);
|
|
113
|
+
this.linearSync = new LinearSessionSync(config, db, linearProvider, logger, feed);
|
|
114
|
+
this.idleReconciler = new IdleIssueReconciler(db, config, {
|
|
115
|
+
requestMergeQueueAdmission: (issue, projectId) => this.requestMergeQueueAdmission(issue, projectId),
|
|
116
|
+
enqueueIssue: (projectId, issueId) => this.enqueueIssue(projectId, issueId),
|
|
117
|
+
}, logger, feed);
|
|
118
|
+
this.queueHealthMonitor = new QueueHealthMonitor(db, config, {
|
|
119
|
+
advanceIdleIssue: (issue, newState, options) => this.idleReconciler.advanceIdleIssue(issue, newState, options),
|
|
120
|
+
}, logger, feed);
|
|
120
121
|
}
|
|
121
122
|
// ─── Run ────────────────────────────────────────────────────────
|
|
122
123
|
async run(item) {
|
|
@@ -279,8 +280,8 @@ export class RunOrchestrator {
|
|
|
279
280
|
});
|
|
280
281
|
this.logger.error({ issueKey: issue.issueKey, runType, error: message }, `Failed to launch ${runType} run`);
|
|
281
282
|
const failedIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
282
|
-
void this.
|
|
283
|
-
void this.
|
|
283
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(runType, `Failed to start ${lowerCaseFirst(message)}`));
|
|
284
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: runType });
|
|
284
285
|
throw error;
|
|
285
286
|
}
|
|
286
287
|
this.db.updateRunThread(run.id, { threadId, turnId });
|
|
@@ -296,8 +297,8 @@ export class RunOrchestrator {
|
|
|
296
297
|
this.logger.info({ issueKey: issue.issueKey, runType, threadId, turnId }, `Started ${runType} run`);
|
|
297
298
|
// Emit Linear activity + plan
|
|
298
299
|
const freshIssue = this.db.getIssue(item.projectId, item.issueId) ?? issue;
|
|
299
|
-
void this.
|
|
300
|
-
void this.
|
|
300
|
+
void this.linearSync.emitActivity(freshIssue, buildRunStartedActivity(runType));
|
|
301
|
+
void this.linearSync.syncSession(freshIssue, { activeRunType: runType });
|
|
301
302
|
}
|
|
302
303
|
// ─── Pre-run branch freshening ────────────────────────────────────
|
|
303
304
|
/**
|
|
@@ -378,12 +379,12 @@ export class RunOrchestrator {
|
|
|
378
379
|
});
|
|
379
380
|
}
|
|
380
381
|
// Emit ephemeral progress activity to Linear for notable in-flight events
|
|
381
|
-
this.
|
|
382
|
+
this.linearSync.maybeEmitProgress(notification, run);
|
|
382
383
|
// Sync codex plan to Linear session when it updates
|
|
383
384
|
if (notification.method === "turn/plan/updated") {
|
|
384
385
|
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
385
386
|
if (issue) {
|
|
386
|
-
void this.
|
|
387
|
+
void this.linearSync.syncCodexPlan(issue, notification.params);
|
|
387
388
|
}
|
|
388
389
|
}
|
|
389
390
|
if (notification.method !== "turn/completed")
|
|
@@ -417,9 +418,9 @@ export class RunOrchestrator {
|
|
|
417
418
|
summary: `Turn failed for ${run.runType}`,
|
|
418
419
|
});
|
|
419
420
|
const failedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
420
|
-
void this.
|
|
421
|
-
void this.
|
|
422
|
-
this.
|
|
421
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType));
|
|
422
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
423
|
+
this.linearSync.clearProgress(run.id);
|
|
423
424
|
this.activeThreadId = undefined;
|
|
424
425
|
return;
|
|
425
426
|
}
|
|
@@ -442,9 +443,9 @@ export class RunOrchestrator {
|
|
|
442
443
|
status: "branch_not_advanced",
|
|
443
444
|
summary: verifiedRepairError,
|
|
444
445
|
});
|
|
445
|
-
void this.
|
|
446
|
-
void this.
|
|
447
|
-
this.
|
|
446
|
+
void this.linearSync.emitActivity(heldIssue, buildRunFailureActivity(run.runType, verifiedRepairError));
|
|
447
|
+
void this.linearSync.syncSession(heldIssue, { activeRunType: run.runType });
|
|
448
|
+
this.linearSync.clearProgress(run.id);
|
|
448
449
|
this.activeThreadId = undefined;
|
|
449
450
|
return;
|
|
450
451
|
}
|
|
@@ -461,6 +462,7 @@ export class RunOrchestrator {
|
|
|
461
462
|
projectId: run.projectId,
|
|
462
463
|
linearIssueId: run.linearIssueId,
|
|
463
464
|
activeRunId: null,
|
|
465
|
+
...(postRunState === "awaiting_queue" ? { queueLabelApplied: false } : {}),
|
|
464
466
|
...(postRunState ? { factoryState: postRunState } : {}),
|
|
465
467
|
...(postRunState === "awaiting_queue" || postRunState === "done"
|
|
466
468
|
? {
|
|
@@ -498,54 +500,16 @@ export class RunOrchestrator {
|
|
|
498
500
|
// Emit Linear completion activity + plan
|
|
499
501
|
const updatedIssue = this.db.getIssue(run.projectId, run.linearIssueId) ?? issue;
|
|
500
502
|
const completionSummary = report.assistantMessages.at(-1)?.slice(0, 300) ?? `${run.runType} completed.`;
|
|
501
|
-
void this.
|
|
503
|
+
void this.linearSync.emitActivity(updatedIssue, buildRunCompletedActivity({
|
|
502
504
|
runType: run.runType,
|
|
503
505
|
completionSummary,
|
|
504
506
|
postRunState: updatedIssue.factoryState,
|
|
505
507
|
...(updatedIssue.prNumber !== undefined ? { prNumber: updatedIssue.prNumber } : {}),
|
|
506
508
|
}));
|
|
507
|
-
void this.
|
|
508
|
-
this.
|
|
509
|
+
void this.linearSync.syncSession(updatedIssue);
|
|
510
|
+
this.linearSync.clearProgress(run.id);
|
|
509
511
|
this.activeThreadId = undefined;
|
|
510
512
|
}
|
|
511
|
-
// ─── In-flight progress ──────────────────────────────────────────
|
|
512
|
-
maybeEmitProgressActivity(notification, run) {
|
|
513
|
-
const activity = this.resolveProgressActivity(notification);
|
|
514
|
-
if (!activity)
|
|
515
|
-
return;
|
|
516
|
-
const now = Date.now();
|
|
517
|
-
const lastEmit = this.progressThrottle.get(run.id) ?? 0;
|
|
518
|
-
if (now - lastEmit < PROGRESS_THROTTLE_MS)
|
|
519
|
-
return;
|
|
520
|
-
this.progressThrottle.set(run.id, now);
|
|
521
|
-
const issue = this.db.getIssue(run.projectId, run.linearIssueId);
|
|
522
|
-
if (issue) {
|
|
523
|
-
void this.emitLinearActivity(issue, activity, { ephemeral: true });
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
resolveProgressActivity(notification) {
|
|
527
|
-
if (notification.method === "item/started") {
|
|
528
|
-
const item = notification.params.item;
|
|
529
|
-
if (!item)
|
|
530
|
-
return undefined;
|
|
531
|
-
const type = typeof item.type === "string" ? item.type : undefined;
|
|
532
|
-
if (type === "commandExecution") {
|
|
533
|
-
const cmd = item.command;
|
|
534
|
-
const cmdStr = Array.isArray(cmd) ? cmd.join(" ") : typeof cmd === "string" ? cmd : undefined;
|
|
535
|
-
return { type: "action", action: "Running", parameter: cmdStr?.slice(0, 120) ?? "command" };
|
|
536
|
-
}
|
|
537
|
-
if (type === "mcpToolCall") {
|
|
538
|
-
const server = typeof item.server === "string" ? item.server : "";
|
|
539
|
-
const tool = typeof item.tool === "string" ? item.tool : "";
|
|
540
|
-
return { type: "action", action: "Using", parameter: `${server}/${tool}` };
|
|
541
|
-
}
|
|
542
|
-
if (type === "dynamicToolCall") {
|
|
543
|
-
const tool = typeof item.tool === "string" ? item.tool : "tool";
|
|
544
|
-
return { type: "action", action: "Using", parameter: tool };
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
return undefined;
|
|
548
|
-
}
|
|
549
513
|
// ─── Active status for query ──────────────────────────────────────
|
|
550
514
|
async getActiveRunStatus(issueKey) {
|
|
551
515
|
const issue = this.db.getIssueByKey(issueKey);
|
|
@@ -569,309 +533,14 @@ export class RunOrchestrator {
|
|
|
569
533
|
}
|
|
570
534
|
// Preemptively detect stuck merge-queue PRs (conflicts visible on
|
|
571
535
|
// GitHub) and dispatch queue_repair before the Steward evicts.
|
|
572
|
-
await this.
|
|
536
|
+
await this.queueHealthMonitor.reconcile();
|
|
573
537
|
// Advance issues stuck in pr_open whose stored PR metadata already
|
|
574
538
|
// shows they should transition (e.g. approved PR, missed webhook).
|
|
575
|
-
await this.
|
|
576
|
-
}
|
|
577
|
-
// ─── Queue Health Monitor ──────────────────────────────────────────
|
|
578
|
-
async reconcileQueueHealth() {
|
|
579
|
-
for (const issue of this.db.listAwaitingQueueIssues()) {
|
|
580
|
-
await this.probeQueuedIssue(issue);
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
|
-
async probeQueuedIssue(issue) {
|
|
584
|
-
if (!issue.prNumber)
|
|
585
|
-
return;
|
|
586
|
-
const project = this.config.projects.find((p) => p.id === issue.projectId);
|
|
587
|
-
if (!project?.github?.repoFullName)
|
|
588
|
-
return;
|
|
589
|
-
// Grace period — don't probe PRs that just entered the queue.
|
|
590
|
-
const age = Date.now() - Date.parse(issue.updatedAt);
|
|
591
|
-
if (age < QUEUE_HEALTH_GRACE_MS)
|
|
592
|
-
return;
|
|
593
|
-
const protocol = resolveMergeQueueProtocol(project);
|
|
594
|
-
let pr;
|
|
595
|
-
try {
|
|
596
|
-
const { stdout } = await execCommand("gh", [
|
|
597
|
-
"pr", "view", String(issue.prNumber),
|
|
598
|
-
"--repo", project.github.repoFullName,
|
|
599
|
-
"--json", "state,mergeable,mergeStateStatus,headRefOid,labels",
|
|
600
|
-
], { timeoutMs: 10_000 });
|
|
601
|
-
pr = JSON.parse(stdout);
|
|
602
|
-
}
|
|
603
|
-
catch (error) {
|
|
604
|
-
this.logger.debug({ issueKey: issue.issueKey, prNumber: issue.prNumber, error: error instanceof Error ? error.message : String(error) }, "Queue health: failed to probe GitHub PR state");
|
|
605
|
-
// Throttle feed events — at most one per issue per cooldown window.
|
|
606
|
-
const issueKey = `${issue.projectId}::${issue.linearIssueId}`;
|
|
607
|
-
const lastFeedAt = this.probeFailureFeedTimes.get(issueKey) ?? 0;
|
|
608
|
-
if (Date.now() - lastFeedAt >= QUEUE_HEALTH_PROBE_FAILURE_COOLDOWN_MS) {
|
|
609
|
-
this.probeFailureFeedTimes.set(issueKey, Date.now());
|
|
610
|
-
this.feed?.publish({
|
|
611
|
-
level: "info",
|
|
612
|
-
kind: "github",
|
|
613
|
-
issueKey: issue.issueKey,
|
|
614
|
-
projectId: issue.projectId,
|
|
615
|
-
stage: "awaiting_queue",
|
|
616
|
-
status: "queue_health_probe_failed",
|
|
617
|
-
summary: `Queue health: failed to probe PR #${issue.prNumber}`,
|
|
618
|
-
});
|
|
619
|
-
}
|
|
620
|
-
return;
|
|
621
|
-
}
|
|
622
|
-
// Successful probe — clear any probe-failure throttle for this issue.
|
|
623
|
-
this.probeFailureFeedTimes.delete(`${issue.projectId}::${issue.linearIssueId}`);
|
|
624
|
-
// Missed merge webhook — advance to done.
|
|
625
|
-
if (pr.state === "MERGED") {
|
|
626
|
-
this.db.upsertIssue({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, prState: "merged" });
|
|
627
|
-
this.advanceIdleIssue(issue, "done", { clearFailureProvenance: true });
|
|
628
|
-
return;
|
|
629
|
-
}
|
|
630
|
-
// Non-open PRs (closed, draft) — don't enter repair logic.
|
|
631
|
-
if (pr.state !== "OPEN")
|
|
632
|
-
return;
|
|
633
|
-
// Verify admission label is still present — if the Steward removed it
|
|
634
|
-
// (eviction, dequeue) but PatchRelay missed the webhook, we should not
|
|
635
|
-
// treat a DIRTY PR as a queue-health problem.
|
|
636
|
-
const hasQueueLabel = pr.labels?.some((l) => l.name === protocol.admissionLabel) ?? false;
|
|
637
|
-
if (!hasQueueLabel)
|
|
638
|
-
return;
|
|
639
|
-
// Detect queue issues: either GitHub reports DIRTY, or the steward
|
|
640
|
-
// eviction check run failed (webhook may have been missed).
|
|
641
|
-
const isDirty = pr.mergeStateStatus === "DIRTY" || pr.mergeable === "CONFLICTING";
|
|
642
|
-
let hasEvictionCheckRun = false;
|
|
643
|
-
if (!isDirty) {
|
|
644
|
-
// Check for missed eviction webhook by looking for the steward's
|
|
645
|
-
// check run on the PR head.
|
|
646
|
-
try {
|
|
647
|
-
const { stdout: checksOut } = await execCommand("gh", [
|
|
648
|
-
"api", `repos/${project.github.repoFullName}/commits/${pr.headRefOid}/check-runs`,
|
|
649
|
-
"--jq", `.check_runs[] | select(.name == "${protocol.evictionCheckName}" and .conclusion == "failure") | .name`,
|
|
650
|
-
], { timeoutMs: 10_000 });
|
|
651
|
-
hasEvictionCheckRun = checksOut.trim().length > 0;
|
|
652
|
-
}
|
|
653
|
-
catch {
|
|
654
|
-
// Best-effort check.
|
|
655
|
-
}
|
|
656
|
-
}
|
|
657
|
-
if (isDirty || hasEvictionCheckRun) {
|
|
658
|
-
const headRefOid = pr.headRefOid ?? "unknown";
|
|
659
|
-
const reason = hasEvictionCheckRun ? "queue_eviction_missed" : "preemptive_conflict";
|
|
660
|
-
const signature = `preemptive_queue_conflict:${headRefOid}`;
|
|
661
|
-
const pendingRunContext = {
|
|
662
|
-
source: "queue_health_monitor",
|
|
663
|
-
failureReason: reason,
|
|
664
|
-
failureHeadSha: headRefOid,
|
|
665
|
-
failureSignature: signature,
|
|
666
|
-
};
|
|
667
|
-
if (isDuplicateRepairAttempt(issue, pendingRunContext)) {
|
|
668
|
-
return;
|
|
669
|
-
}
|
|
670
|
-
this.db.upsertIssue({
|
|
671
|
-
projectId: issue.projectId,
|
|
672
|
-
linearIssueId: issue.linearIssueId,
|
|
673
|
-
lastAttemptedFailureHeadSha: headRefOid,
|
|
674
|
-
lastAttemptedFailureSignature: signature,
|
|
675
|
-
});
|
|
676
|
-
this.advanceIdleIssue(issue, "repairing_queue", {
|
|
677
|
-
pendingRunType: "queue_repair",
|
|
678
|
-
pendingRunContext,
|
|
679
|
-
});
|
|
680
|
-
this.logger.info({ issueKey: issue.issueKey, prNumber: issue.prNumber, headRefOid, reason }, "Queue health: queue issue detected, dispatching repair");
|
|
681
|
-
this.feed?.publish({
|
|
682
|
-
level: "warn",
|
|
683
|
-
kind: "github",
|
|
684
|
-
issueKey: issue.issueKey,
|
|
685
|
-
projectId: issue.projectId,
|
|
686
|
-
stage: "repairing_queue",
|
|
687
|
-
status: hasEvictionCheckRun ? "queue_health_eviction_detected" : "queue_health_conflict_detected",
|
|
688
|
-
summary: hasEvictionCheckRun
|
|
689
|
-
? `Queue health: missed eviction detected on PR #${issue.prNumber}, dispatching repair`
|
|
690
|
-
: `Queue health: merge conflict detected on PR #${issue.prNumber}, dispatching preemptive repair`,
|
|
691
|
-
});
|
|
692
|
-
}
|
|
693
|
-
}
|
|
694
|
-
async reconcileIdleIssues() {
|
|
695
|
-
for (const issue of this.db.listIdleNonTerminalIssues()) {
|
|
696
|
-
// PR already merged — advance to done regardless of current state
|
|
697
|
-
if (issue.prState === "merged") {
|
|
698
|
-
this.advanceIdleIssue(issue, "done", { clearFailureProvenance: true });
|
|
699
|
-
continue;
|
|
700
|
-
}
|
|
701
|
-
// Review approved + checks not failed — advance to awaiting_queue
|
|
702
|
-
if (issue.prReviewState === "approved" && issue.prCheckStatus !== "failed") {
|
|
703
|
-
if (issue.factoryState !== "awaiting_queue" || issue.branchOwner !== "merge_steward") {
|
|
704
|
-
this.advanceIdleIssue(issue, "awaiting_queue", { clearFailureProvenance: true });
|
|
705
|
-
}
|
|
706
|
-
else if (!issue.queueLabelApplied) {
|
|
707
|
-
// Retry failed label application
|
|
708
|
-
await this.requestMergeQueueAdmission(issue, issue.projectId);
|
|
709
|
-
}
|
|
710
|
-
continue;
|
|
711
|
-
}
|
|
712
|
-
// Checks failed + idle — route based on durable GitHub failure provenance.
|
|
713
|
-
if (issue.prCheckStatus === "failed") {
|
|
714
|
-
if (issue.lastGitHubFailureSource === "queue_eviction") {
|
|
715
|
-
const pendingRunContext = buildFailureContext(issue);
|
|
716
|
-
if (isDuplicateRepairAttempt(issue, pendingRunContext)) {
|
|
717
|
-
this.advanceIdleIssue(issue, "repairing_queue");
|
|
718
|
-
}
|
|
719
|
-
else {
|
|
720
|
-
this.advanceIdleIssue(issue, "repairing_queue", {
|
|
721
|
-
pendingRunType: "queue_repair",
|
|
722
|
-
...(pendingRunContext ? { pendingRunContext } : {}),
|
|
723
|
-
});
|
|
724
|
-
}
|
|
725
|
-
continue;
|
|
726
|
-
}
|
|
727
|
-
if (issue.lastGitHubFailureSource === "branch_ci") {
|
|
728
|
-
const pendingRunContext = buildFailureContext(issue);
|
|
729
|
-
if (isDuplicateRepairAttempt(issue, pendingRunContext)) {
|
|
730
|
-
this.advanceIdleIssue(issue, "repairing_ci");
|
|
731
|
-
}
|
|
732
|
-
else {
|
|
733
|
-
this.advanceIdleIssue(issue, "repairing_ci", {
|
|
734
|
-
pendingRunType: "ci_repair",
|
|
735
|
-
...(pendingRunContext ? { pendingRunContext } : {}),
|
|
736
|
-
});
|
|
737
|
-
}
|
|
738
|
-
continue;
|
|
739
|
-
}
|
|
740
|
-
if (issue.factoryState === "awaiting_queue") {
|
|
741
|
-
// Infer provenance: check if steward eviction check run exists on the PR
|
|
742
|
-
const inferProject = this.config.projects.find((p) => p.id === issue.projectId);
|
|
743
|
-
const inferProtocol = resolveMergeQueueProtocol(inferProject);
|
|
744
|
-
let inferred = "branch_ci";
|
|
745
|
-
const probeSha = issue.lastGitHubFailureHeadSha ?? issue.lastGitHubCiSnapshotHeadSha;
|
|
746
|
-
if (inferProject?.github?.repoFullName && issue.prNumber && probeSha) {
|
|
747
|
-
try {
|
|
748
|
-
const { stdout } = await execCommand("gh", [
|
|
749
|
-
"api",
|
|
750
|
-
`repos/${inferProject.github.repoFullName}/commits/${probeSha}/check-runs`,
|
|
751
|
-
"--jq", `.check_runs[] | select(.name == "${inferProtocol.evictionCheckName}" and .conclusion == "failure") | .name`,
|
|
752
|
-
], { timeoutMs: 10_000 });
|
|
753
|
-
if (stdout.trim().length > 0)
|
|
754
|
-
inferred = "queue_eviction";
|
|
755
|
-
}
|
|
756
|
-
catch { /* best effort */ }
|
|
757
|
-
}
|
|
758
|
-
const inferRunType = inferred === "queue_eviction" ? "queue_repair" : "ci_repair";
|
|
759
|
-
const inferState = inferred === "queue_eviction" ? "repairing_queue" : "repairing_ci";
|
|
760
|
-
this.logger.info({ issueKey: issue.issueKey, prNumber: issue.prNumber, inferred }, "Inferred failure provenance for awaiting_queue issue");
|
|
761
|
-
const pendingRunContext = buildFailureContext(issue);
|
|
762
|
-
this.advanceIdleIssue(issue, inferState, {
|
|
763
|
-
pendingRunType: inferRunType,
|
|
764
|
-
...(pendingRunContext ? { pendingRunContext } : {}),
|
|
765
|
-
});
|
|
766
|
-
continue;
|
|
767
|
-
}
|
|
768
|
-
const pendingRunContext = buildFailureContext(issue);
|
|
769
|
-
if (isDuplicateRepairAttempt(issue, pendingRunContext)) {
|
|
770
|
-
this.advanceIdleIssue(issue, "repairing_ci");
|
|
771
|
-
}
|
|
772
|
-
else {
|
|
773
|
-
this.advanceIdleIssue(issue, "repairing_ci", {
|
|
774
|
-
pendingRunType: "ci_repair",
|
|
775
|
-
...(pendingRunContext ? { pendingRunContext } : {}),
|
|
776
|
-
});
|
|
777
|
-
}
|
|
778
|
-
continue;
|
|
779
|
-
}
|
|
780
|
-
// For pr_open issues with no review decision, check GitHub for stale metadata
|
|
781
|
-
if (issue.factoryState === "pr_open" && !issue.prReviewState) {
|
|
782
|
-
await this.reconcileFromGitHub(issue);
|
|
783
|
-
}
|
|
784
|
-
}
|
|
785
|
-
// Unblock delegated issues whose blockers have been resolved.
|
|
786
|
-
for (const issue of this.db.listBlockedDelegatedIssues()) {
|
|
787
|
-
const unresolved = this.db.countUnresolvedBlockers(issue.projectId, issue.linearIssueId);
|
|
788
|
-
if (unresolved === 0) {
|
|
789
|
-
this.db.upsertIssue({
|
|
790
|
-
projectId: issue.projectId,
|
|
791
|
-
linearIssueId: issue.linearIssueId,
|
|
792
|
-
pendingRunType: "implementation",
|
|
793
|
-
});
|
|
794
|
-
this.enqueueIssue(issue.projectId, issue.linearIssueId);
|
|
795
|
-
}
|
|
796
|
-
}
|
|
797
|
-
}
|
|
798
|
-
async reconcileFromGitHub(issue) {
|
|
799
|
-
const project = this.config.projects.find((p) => p.id === issue.projectId);
|
|
800
|
-
if (!project?.github?.repoFullName || !issue.prNumber)
|
|
801
|
-
return;
|
|
802
|
-
try {
|
|
803
|
-
const { stdout } = await execCommand("gh", [
|
|
804
|
-
"pr", "view", String(issue.prNumber),
|
|
805
|
-
"--repo", project.github.repoFullName,
|
|
806
|
-
"--json", "state,reviewDecision",
|
|
807
|
-
], { timeoutMs: 10_000 });
|
|
808
|
-
const pr = JSON.parse(stdout);
|
|
809
|
-
if (pr.state === "MERGED") {
|
|
810
|
-
this.db.upsertIssue({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, prState: "merged" });
|
|
811
|
-
this.advanceIdleIssue(issue, "done", { clearFailureProvenance: true });
|
|
812
|
-
}
|
|
813
|
-
else if (pr.reviewDecision === "APPROVED") {
|
|
814
|
-
this.db.upsertIssue({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, prReviewState: "approved" });
|
|
815
|
-
this.advanceIdleIssue(issue, "awaiting_queue", { clearFailureProvenance: true });
|
|
816
|
-
}
|
|
817
|
-
}
|
|
818
|
-
catch (error) {
|
|
819
|
-
this.logger.debug({ issueKey: issue.issueKey, error: error instanceof Error ? error.message : String(error) }, "Failed to query GitHub PR state during reconciliation");
|
|
820
|
-
}
|
|
539
|
+
await this.idleReconciler.reconcile();
|
|
821
540
|
}
|
|
541
|
+
// advanceIdleIssue is now on IdleIssueReconciler — delegate for internal callers
|
|
822
542
|
advanceIdleIssue(issue, newState, options) {
|
|
823
|
-
|
|
824
|
-
return;
|
|
825
|
-
}
|
|
826
|
-
this.logger.info({ issueKey: issue.issueKey, from: issue.factoryState, to: newState, pendingRunType: options?.pendingRunType }, "Reconciliation: advancing idle issue");
|
|
827
|
-
// Reset queueLabelApplied when entering or leaving awaiting_queue so
|
|
828
|
-
// the retry loop re-applies the label on each queue cycle.
|
|
829
|
-
const resetQueueLabel = newState === "awaiting_queue" || issue.factoryState === "awaiting_queue";
|
|
830
|
-
this.db.upsertIssue({
|
|
831
|
-
projectId: issue.projectId,
|
|
832
|
-
linearIssueId: issue.linearIssueId,
|
|
833
|
-
factoryState: newState,
|
|
834
|
-
...(options?.pendingRunType ? { pendingRunType: options.pendingRunType } : {}),
|
|
835
|
-
...(options?.pendingRunType
|
|
836
|
-
? {
|
|
837
|
-
pendingRunContextJson: options.pendingRunContext ? JSON.stringify(options.pendingRunContext) : null,
|
|
838
|
-
}
|
|
839
|
-
: {}),
|
|
840
|
-
...(resetQueueLabel ? { queueLabelApplied: false } : {}),
|
|
841
|
-
...(options?.clearFailureProvenance
|
|
842
|
-
? {
|
|
843
|
-
lastGitHubFailureSource: null,
|
|
844
|
-
lastGitHubFailureHeadSha: null,
|
|
845
|
-
lastGitHubFailureSignature: null,
|
|
846
|
-
lastGitHubFailureCheckName: null,
|
|
847
|
-
lastGitHubFailureCheckUrl: null,
|
|
848
|
-
lastGitHubFailureContextJson: null,
|
|
849
|
-
lastGitHubFailureAt: null,
|
|
850
|
-
lastQueueIncidentJson: null,
|
|
851
|
-
lastAttemptedFailureHeadSha: null,
|
|
852
|
-
lastAttemptedFailureSignature: null,
|
|
853
|
-
}
|
|
854
|
-
: {}),
|
|
855
|
-
});
|
|
856
|
-
const branchOwner = this.resolveBranchOwnerForStateTransition(newState, options?.pendingRunType);
|
|
857
|
-
if (branchOwner) {
|
|
858
|
-
this.db.setBranchOwner(issue.projectId, issue.linearIssueId, branchOwner);
|
|
859
|
-
}
|
|
860
|
-
this.feed?.publish({
|
|
861
|
-
level: "info",
|
|
862
|
-
kind: "stage",
|
|
863
|
-
issueKey: issue.issueKey,
|
|
864
|
-
projectId: issue.projectId,
|
|
865
|
-
stage: newState,
|
|
866
|
-
status: "reconciled",
|
|
867
|
-
summary: `Reconciliation: ${issue.factoryState} \u2192 ${newState}`,
|
|
868
|
-
});
|
|
869
|
-
if (newState === "awaiting_queue" && issue.factoryState !== "awaiting_queue") {
|
|
870
|
-
this.requestMergeQueueAdmission(issue, issue.projectId);
|
|
871
|
-
}
|
|
872
|
-
if (options?.pendingRunType) {
|
|
873
|
-
this.enqueueIssue(issue.projectId, issue.linearIssueId);
|
|
874
|
-
}
|
|
543
|
+
this.idleReconciler.advanceIdleIssue(issue, newState, options);
|
|
875
544
|
}
|
|
876
545
|
/**
|
|
877
546
|
* After a zombie/stale run is cleared, decide whether to re-enqueue
|
|
@@ -1035,9 +704,9 @@ export class RunOrchestrator {
|
|
|
1035
704
|
});
|
|
1036
705
|
}
|
|
1037
706
|
else {
|
|
1038
|
-
void this.
|
|
707
|
+
void this.linearSync.emitActivity(failedIssue, buildRunFailureActivity(run.runType, "The Codex turn was interrupted."));
|
|
1039
708
|
}
|
|
1040
|
-
void this.
|
|
709
|
+
void this.linearSync.syncSession(failedIssue, { activeRunType: run.runType });
|
|
1041
710
|
return;
|
|
1042
711
|
}
|
|
1043
712
|
// Handle completed turn discovered during reconciliation
|
|
@@ -1073,6 +742,7 @@ export class RunOrchestrator {
|
|
|
1073
742
|
projectId: run.projectId,
|
|
1074
743
|
linearIssueId: run.linearIssueId,
|
|
1075
744
|
activeRunId: null,
|
|
745
|
+
...(postRunState === "awaiting_queue" ? { queueLabelApplied: false } : {}),
|
|
1076
746
|
...(postRunState ? { factoryState: postRunState } : {}),
|
|
1077
747
|
...(postRunState === "awaiting_queue" || postRunState === "done"
|
|
1078
748
|
? {
|
|
@@ -1133,11 +803,11 @@ export class RunOrchestrator {
|
|
|
1133
803
|
summary: `Escalated: ${reason}`,
|
|
1134
804
|
});
|
|
1135
805
|
const escalatedIssue = this.db.getIssue(issue.projectId, issue.linearIssueId) ?? issue;
|
|
1136
|
-
void this.
|
|
806
|
+
void this.linearSync.emitActivity(escalatedIssue, {
|
|
1137
807
|
type: "error",
|
|
1138
808
|
body: `PatchRelay needs human help to continue.\n\n${reason}`,
|
|
1139
809
|
});
|
|
1140
|
-
void this.
|
|
810
|
+
void this.linearSync.syncSession(escalatedIssue);
|
|
1141
811
|
}
|
|
1142
812
|
/** Add the merge queue admission label for external-queue projects (best-effort). */
|
|
1143
813
|
async requestMergeQueueAdmission(issue, projectId) {
|
|
@@ -1169,13 +839,7 @@ export class RunOrchestrator {
|
|
|
1169
839
|
});
|
|
1170
840
|
}
|
|
1171
841
|
resolveBranchOwnerForStateTransition(newState, pendingRunType) {
|
|
1172
|
-
|
|
1173
|
-
return "patchrelay";
|
|
1174
|
-
if (newState === "awaiting_queue")
|
|
1175
|
-
return "merge_steward";
|
|
1176
|
-
if (newState === "repairing_ci" || newState === "repairing_queue")
|
|
1177
|
-
return "patchrelay";
|
|
1178
|
-
return undefined;
|
|
842
|
+
return resolveBranchOwnerForStateTransition(newState, pendingRunType);
|
|
1179
843
|
}
|
|
1180
844
|
async verifyReactiveRunAdvancedBranch(run, issue) {
|
|
1181
845
|
if (run.runType !== "ci_repair" && run.runType !== "queue_repair") {
|
|
@@ -1212,93 +876,6 @@ export class RunOrchestrator {
|
|
|
1212
876
|
return undefined;
|
|
1213
877
|
}
|
|
1214
878
|
}
|
|
1215
|
-
async emitLinearActivity(issue, content, options) {
|
|
1216
|
-
if (!issue.agentSessionId)
|
|
1217
|
-
return;
|
|
1218
|
-
try {
|
|
1219
|
-
const linear = await this.linearProvider.forProject(issue.projectId);
|
|
1220
|
-
if (!linear)
|
|
1221
|
-
return;
|
|
1222
|
-
const allowEphemeral = content.type === "thought" || content.type === "action";
|
|
1223
|
-
await linear.createAgentActivity({
|
|
1224
|
-
agentSessionId: issue.agentSessionId,
|
|
1225
|
-
content,
|
|
1226
|
-
...(options?.ephemeral && allowEphemeral ? { ephemeral: true } : {}),
|
|
1227
|
-
});
|
|
1228
|
-
}
|
|
1229
|
-
catch (error) {
|
|
1230
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
1231
|
-
this.logger.warn({ issueKey: issue.issueKey, type: content.type, error: msg }, "Failed to emit Linear activity");
|
|
1232
|
-
this.feed?.publish({
|
|
1233
|
-
level: "warn",
|
|
1234
|
-
kind: "linear",
|
|
1235
|
-
issueKey: issue.issueKey,
|
|
1236
|
-
projectId: issue.projectId,
|
|
1237
|
-
status: "linear_error",
|
|
1238
|
-
summary: `Linear activity failed: ${msg}`,
|
|
1239
|
-
});
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
async syncLinearSession(issue, options) {
|
|
1243
|
-
if (!issue.agentSessionId)
|
|
1244
|
-
return;
|
|
1245
|
-
try {
|
|
1246
|
-
const linear = await this.linearProvider.forProject(issue.projectId);
|
|
1247
|
-
if (!linear?.updateAgentSession)
|
|
1248
|
-
return;
|
|
1249
|
-
const externalUrls = buildAgentSessionExternalUrls(this.config, {
|
|
1250
|
-
...(issue.issueKey ? { issueKey: issue.issueKey } : {}),
|
|
1251
|
-
...(issue.prUrl ? { prUrl: issue.prUrl } : {}),
|
|
1252
|
-
});
|
|
1253
|
-
await linear.updateAgentSession({
|
|
1254
|
-
agentSessionId: issue.agentSessionId,
|
|
1255
|
-
plan: buildAgentSessionPlanForIssue(issue, options),
|
|
1256
|
-
...(externalUrls ? { externalUrls } : {}),
|
|
1257
|
-
});
|
|
1258
|
-
}
|
|
1259
|
-
catch (error) {
|
|
1260
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
1261
|
-
this.logger.warn({ issueKey: issue.issueKey, error: msg }, "Failed to update Linear plan");
|
|
1262
|
-
}
|
|
1263
|
-
}
|
|
1264
|
-
async syncLinearSessionWithCodexPlan(issue, params) {
|
|
1265
|
-
if (!issue.agentSessionId)
|
|
1266
|
-
return;
|
|
1267
|
-
const plan = params.plan;
|
|
1268
|
-
if (!Array.isArray(plan))
|
|
1269
|
-
return;
|
|
1270
|
-
const STATUS_MAP = {
|
|
1271
|
-
pending: "pending",
|
|
1272
|
-
inProgress: "inProgress",
|
|
1273
|
-
completed: "completed",
|
|
1274
|
-
};
|
|
1275
|
-
const steps = plan.map((entry) => {
|
|
1276
|
-
const e = entry;
|
|
1277
|
-
const step = typeof e.step === "string" ? e.step : String(e.step ?? "");
|
|
1278
|
-
const status = typeof e.status === "string" ? (STATUS_MAP[e.status] ?? "pending") : "pending";
|
|
1279
|
-
return { content: step, status };
|
|
1280
|
-
});
|
|
1281
|
-
// Prepend a "Prepare workspace" completed step and append a "Merge" pending step
|
|
1282
|
-
// to frame the codex plan within the PatchRelay lifecycle
|
|
1283
|
-
const fullPlan = [
|
|
1284
|
-
{ content: "Prepare workspace", status: "completed" },
|
|
1285
|
-
...steps,
|
|
1286
|
-
{ content: "Merge", status: "pending" },
|
|
1287
|
-
];
|
|
1288
|
-
try {
|
|
1289
|
-
const linear = await this.linearProvider.forProject(issue.projectId);
|
|
1290
|
-
if (!linear?.updateAgentSession)
|
|
1291
|
-
return;
|
|
1292
|
-
await linear.updateAgentSession({
|
|
1293
|
-
agentSessionId: issue.agentSessionId,
|
|
1294
|
-
plan: fullPlan,
|
|
1295
|
-
});
|
|
1296
|
-
}
|
|
1297
|
-
catch (error) {
|
|
1298
|
-
const msg = error instanceof Error ? error.message : String(error);
|
|
1299
|
-
this.logger.warn({ issueKey: issue.issueKey, error: msg }, "Failed to sync codex plan to Linear");
|
|
1300
|
-
}
|
|
1301
|
-
}
|
|
1302
879
|
async readThreadWithRetry(threadId, maxRetries = 3) {
|
|
1303
880
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
1304
881
|
try {
|
|
@@ -1348,40 +925,6 @@ function resolveRecoverablePostRunState(issue) {
|
|
|
1348
925
|
}
|
|
1349
926
|
return resolvePostRunState(issue);
|
|
1350
927
|
}
|
|
1351
|
-
function buildFailureContext(issue) {
|
|
1352
|
-
const storedFailureContext = parseGitHubFailureContext(issue.lastGitHubFailureContextJson);
|
|
1353
|
-
const queueRepairContext = issue.lastQueueIncidentJson
|
|
1354
|
-
? parseStoredQueueRepairContext(issue.lastQueueIncidentJson)
|
|
1355
|
-
: undefined;
|
|
1356
|
-
if (!queueRepairContext
|
|
1357
|
-
&& !issue.lastGitHubFailureSource
|
|
1358
|
-
&& !issue.lastGitHubFailureHeadSha
|
|
1359
|
-
&& !issue.lastGitHubFailureSignature
|
|
1360
|
-
&& !issue.lastGitHubFailureCheckName
|
|
1361
|
-
&& !issue.lastGitHubFailureCheckUrl
|
|
1362
|
-
&& !storedFailureContext) {
|
|
1363
|
-
return undefined;
|
|
1364
|
-
}
|
|
1365
|
-
return {
|
|
1366
|
-
...(issue.lastGitHubFailureSource ? { failureReason: issue.lastGitHubFailureSource } : {}),
|
|
1367
|
-
...(issue.lastGitHubFailureHeadSha ? { failureHeadSha: issue.lastGitHubFailureHeadSha } : {}),
|
|
1368
|
-
...(issue.lastGitHubFailureSignature ? { failureSignature: issue.lastGitHubFailureSignature } : {}),
|
|
1369
|
-
...(issue.lastGitHubFailureCheckName ? { checkName: issue.lastGitHubFailureCheckName } : {}),
|
|
1370
|
-
...(issue.lastGitHubFailureCheckUrl ? { checkUrl: issue.lastGitHubFailureCheckUrl } : {}),
|
|
1371
|
-
...(storedFailureContext ? storedFailureContext : {}),
|
|
1372
|
-
...(queueRepairContext ? queueRepairContext : {}),
|
|
1373
|
-
};
|
|
1374
|
-
}
|
|
1375
|
-
function isDuplicateRepairAttempt(issue, context) {
|
|
1376
|
-
const signature = typeof context?.failureSignature === "string" ? context.failureSignature : undefined;
|
|
1377
|
-
const headSha = typeof context?.failureHeadSha === "string"
|
|
1378
|
-
? context.failureHeadSha
|
|
1379
|
-
: typeof context?.headSha === "string" ? context.headSha : undefined;
|
|
1380
|
-
if (!signature)
|
|
1381
|
-
return false;
|
|
1382
|
-
return issue.lastAttemptedFailureSignature === signature
|
|
1383
|
-
&& (headSha === undefined || issue.lastAttemptedFailureHeadSha === headSha);
|
|
1384
|
-
}
|
|
1385
928
|
function appendQueueRepairContext(lines, context) {
|
|
1386
929
|
const incidentTitle = typeof context?.incidentTitle === "string" ? context.incidentTitle.trim() : "";
|
|
1387
930
|
const incidentSummary = typeof context?.incidentSummary === "string" ? context.incidentSummary.trim() : "";
|