patchrelay 0.75.3 → 0.77.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/agent-input-service.js +40 -26
  2. package/dist/build-info.json +3 -3
  3. package/dist/cli/data.js +3 -1
  4. package/dist/db/issue-session-store.js +44 -9
  5. package/dist/db/issue-store.js +11 -2
  6. package/dist/db/migrations.js +3 -0
  7. package/dist/factory-state.js +23 -0
  8. package/dist/github-webhook-reactive-run.js +15 -11
  9. package/dist/github-webhook-stack-coordination.js +8 -4
  10. package/dist/github-webhook-state-projector.js +204 -139
  11. package/dist/github-webhook-terminal-handler.js +37 -27
  12. package/dist/idle-reconciliation.js +122 -66
  13. package/dist/implementation-outcome-policy.js +5 -1
  14. package/dist/issue-session-projection-invalidator.js +9 -0
  15. package/dist/linear-agent-session-client.js +16 -8
  16. package/dist/linear-issue-projection.js +15 -11
  17. package/dist/linear-status-comment-sync.js +8 -4
  18. package/dist/linear-workflow-state-sync.js +9 -5
  19. package/dist/merged-linear-completion-reconciler.js +39 -17
  20. package/dist/no-pr-completion-check.js +51 -29
  21. package/dist/orchestration-parent-wake.js +15 -8
  22. package/dist/queue-health-monitor.js +17 -8
  23. package/dist/reactive-run-policy.js +5 -1
  24. package/dist/run-budgets.js +40 -6
  25. package/dist/run-completion-policy.js +50 -9
  26. package/dist/run-failure-policy.js +463 -0
  27. package/dist/run-finalizer.js +68 -35
  28. package/dist/run-launcher.js +63 -12
  29. package/dist/run-notification-handler.js +19 -9
  30. package/dist/run-orchestrator.js +70 -78
  31. package/dist/run-reconciler.js +137 -64
  32. package/dist/run-settlement.js +57 -0
  33. package/dist/run-wake-planner.js +39 -29
  34. package/dist/service-issue-actions.js +45 -28
  35. package/dist/service-startup-recovery.js +61 -35
  36. package/dist/telemetry.js +9 -0
  37. package/dist/terminal-wake-reconciler.js +20 -3
  38. package/dist/webhooks/agent-session-handler.js +22 -12
  39. package/dist/webhooks/dependency-readiness-handler.js +17 -10
  40. package/dist/webhooks/desired-stage-recorder.js +32 -13
  41. package/dist/webhooks/issue-removal-handler.js +24 -13
  42. package/package.json +1 -1
  43. package/dist/interrupted-run-recovery.js +0 -227
  44. package/dist/run-recovery-service.js +0 -202
  45. package/dist/zombie-recovery.js +0 -13
@@ -1,6 +1,18 @@
1
1
  import { CLEARED_FAILURE_PROVENANCE } from "./failure-provenance.js";
2
2
  import { buildCompletionCheckActivity } from "./linear-session-reporting.js";
3
3
  import { wakeOrchestrationParentsForChildEvent } from "./orchestration-parent-wake.js";
4
+ const WRITER = "no-pr-completion-check";
5
+ // Post-completion-check decision writes all clear the run slot; on a version
6
+ // conflict, apply only if the slot still belongs to this run on the fresh row.
7
+ function commitRunSlotUpdate(db, run, issue, update) {
8
+ const commit = db.issueSessions.commitIssueState({
9
+ writer: WRITER,
10
+ expectedVersion: issue.version,
11
+ update,
12
+ onConflict: (current) => (current.activeRunId === run.id ? update : undefined),
13
+ });
14
+ return commit.outcome === "applied";
15
+ }
4
16
  function shouldContinueForUnpublishedLocalChanges(message) {
5
17
  const normalized = message.trim().toLowerCase();
6
18
  if (!normalized)
@@ -56,16 +68,18 @@ export async function handleNoPrCompletionCheck(params) {
56
68
  }
57
69
  if (completionCheck.outcome === "continue") {
58
70
  const continued = params.withHeldLease(params.run.projectId, params.run.linearIssueId, (lease) => {
59
- params.db.runs.finishRun(params.run.id, runUpdate);
60
- params.db.runs.saveCompletionCheck(params.run.id, completionCheck);
61
- params.db.issues.upsertIssue({
71
+ if (!commitRunSlotUpdate(params.db, params.run, params.issue, {
62
72
  projectId: params.run.projectId,
63
73
  linearIssueId: params.run.linearIssueId,
64
74
  activeRunId: null,
65
75
  factoryState: "delegated",
66
76
  pendingRunType: null,
67
77
  pendingRunContextJson: null,
68
- });
78
+ })) {
79
+ return false;
80
+ }
81
+ params.db.runs.finishRun(params.run.id, runUpdate);
82
+ params.db.runs.saveCompletionCheck(params.run.id, completionCheck);
69
83
  return Boolean(params.db.issueSessions.appendIssueSessionEventWithLease(lease, {
70
84
  projectId: params.run.projectId,
71
85
  linearIssueId: params.run.linearIssueId,
@@ -95,17 +109,19 @@ export async function handleNoPrCompletionCheck(params) {
95
109
  }
96
110
  if (completionCheck.outcome === "needs_input") {
97
111
  const completed = params.withHeldLease(params.run.projectId, params.run.linearIssueId, (lease) => {
98
- params.db.runs.finishRun(params.run.id, runUpdate);
99
- params.db.runs.saveCompletionCheck(params.run.id, completionCheck);
100
- params.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
101
- params.db.issues.upsertIssue({
112
+ if (!commitRunSlotUpdate(params.db, params.run, params.issue, {
102
113
  projectId: params.run.projectId,
103
114
  linearIssueId: params.run.linearIssueId,
104
115
  activeRunId: null,
105
116
  factoryState: "awaiting_input",
106
117
  pendingRunType: null,
107
118
  pendingRunContextJson: null,
108
- });
119
+ })) {
120
+ return false;
121
+ }
122
+ params.db.runs.finishRun(params.run.id, runUpdate);
123
+ params.db.runs.saveCompletionCheck(params.run.id, completionCheck);
124
+ params.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
109
125
  return true;
110
126
  });
111
127
  if (!completed) {
@@ -127,20 +143,22 @@ export async function handleNoPrCompletionCheck(params) {
127
143
  if (completionCheck.outcome === "done") {
128
144
  if (shouldContinueForUnpublishedLocalChanges(params.publishedOutcomeError)) {
129
145
  const continued = params.withHeldLease(params.run.projectId, params.run.linearIssueId, (lease) => {
130
- params.db.runs.finishRun(params.run.id, runUpdate);
131
- params.db.runs.saveCompletionCheck(params.run.id, {
132
- ...completionCheck,
133
- outcome: "continue",
134
- summary: "PatchRelay changed files locally but has not published them yet; continuing automatically to finish publication.",
135
- why: params.publishedOutcomeError,
136
- });
137
- params.db.issues.upsertIssue({
146
+ if (!commitRunSlotUpdate(params.db, params.run, params.issue, {
138
147
  projectId: params.run.projectId,
139
148
  linearIssueId: params.run.linearIssueId,
140
149
  activeRunId: null,
141
150
  factoryState: "delegated",
142
151
  pendingRunType: null,
143
152
  pendingRunContextJson: null,
153
+ })) {
154
+ return false;
155
+ }
156
+ params.db.runs.finishRun(params.run.id, runUpdate);
157
+ params.db.runs.saveCompletionCheck(params.run.id, {
158
+ ...completionCheck,
159
+ outcome: "continue",
160
+ summary: "PatchRelay changed files locally but has not published them yet; continuing automatically to finish publication.",
161
+ why: params.publishedOutcomeError,
144
162
  });
145
163
  return Boolean(params.db.issueSessions.appendIssueSessionEventWithLease(lease, {
146
164
  projectId: params.run.projectId,
@@ -173,10 +191,7 @@ export async function handleNoPrCompletionCheck(params) {
173
191
  ? params.db.issues.countOpenChildIssues(params.run.projectId, params.run.linearIssueId)
174
192
  : 0;
175
193
  const completed = params.withHeldLease(params.run.projectId, params.run.linearIssueId, (lease) => {
176
- params.db.runs.finishRun(params.run.id, runUpdate);
177
- params.db.runs.saveCompletionCheck(params.run.id, completionCheck);
178
- params.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
179
- params.db.issues.upsertIssue({
194
+ if (!commitRunSlotUpdate(params.db, params.run, params.issue, {
180
195
  projectId: params.run.projectId,
181
196
  linearIssueId: params.run.linearIssueId,
182
197
  activeRunId: null,
@@ -185,7 +200,12 @@ export async function handleNoPrCompletionCheck(params) {
185
200
  pendingRunContextJson: null,
186
201
  orchestrationSettleUntil: null,
187
202
  ...CLEARED_FAILURE_PROVENANCE,
188
- });
203
+ })) {
204
+ return false;
205
+ }
206
+ params.db.runs.finishRun(params.run.id, runUpdate);
207
+ params.db.runs.saveCompletionCheck(params.run.id, completionCheck);
208
+ params.db.issueSessions.clearPendingIssueSessionEventsWithLease(lease);
189
209
  return true;
190
210
  });
191
211
  if (!completed) {
@@ -217,20 +237,22 @@ export async function handleNoPrCompletionCheck(params) {
217
237
  }
218
238
  const failureReason = `No PR observed and the completion check failed this run: ${completionCheck.summary}`;
219
239
  const failed = params.withHeldLease(params.run.projectId, params.run.linearIssueId, () => {
220
- params.db.runs.finishRun(params.run.id, {
221
- ...runUpdate,
222
- status: "failed",
223
- failureReason,
224
- });
225
- params.db.runs.saveCompletionCheck(params.run.id, completionCheck);
226
- params.db.issues.upsertIssue({
240
+ if (!commitRunSlotUpdate(params.db, params.run, params.issue, {
227
241
  projectId: params.run.projectId,
228
242
  linearIssueId: params.run.linearIssueId,
229
243
  activeRunId: null,
230
244
  factoryState: "failed",
231
245
  pendingRunType: null,
232
246
  pendingRunContextJson: null,
247
+ })) {
248
+ return false;
249
+ }
250
+ params.db.runs.finishRun(params.run.id, {
251
+ ...runUpdate,
252
+ status: "failed",
253
+ failureReason,
233
254
  });
255
+ params.db.runs.saveCompletionCheck(params.run.id, completionCheck);
234
256
  return true;
235
257
  });
236
258
  if (!failed) {
@@ -1,4 +1,5 @@
1
1
  import { classifyIssue } from "./issue-class.js";
2
+ const WRITER = "orchestration-parent-wake";
2
3
  export const ORCHESTRATION_SETTLE_WINDOW_MS = 10_000;
3
4
  export function computeOrchestrationSettleUntil(now = Date.now()) {
4
5
  return new Date(now + ORCHESTRATION_SETTLE_WINDOW_MS).toISOString();
@@ -24,18 +25,24 @@ function resolveParentIssueIds(db, child) {
24
25
  }
25
26
  export function startOrchestrationSettleWindow(db, issue, now = Date.now()) {
26
27
  const settleUntil = computeOrchestrationSettleUntil(now);
27
- db.issues.upsertIssue({
28
- projectId: issue.projectId,
29
- linearIssueId: issue.linearIssueId,
30
- orchestrationSettleUntil: settleUntil,
28
+ db.issueSessions.commitIssueState({
29
+ writer: WRITER,
30
+ update: {
31
+ projectId: issue.projectId,
32
+ linearIssueId: issue.linearIssueId,
33
+ orchestrationSettleUntil: settleUntil,
34
+ },
31
35
  });
32
36
  return settleUntil;
33
37
  }
34
38
  export function queueSettledOrchestrationIssue(params) {
35
- params.db.issues.upsertIssue({
36
- projectId: params.issue.projectId,
37
- linearIssueId: params.issue.linearIssueId,
38
- orchestrationSettleUntil: null,
39
+ params.db.issueSessions.commitIssueState({
40
+ writer: WRITER,
41
+ update: {
42
+ projectId: params.issue.projectId,
43
+ linearIssueId: params.issue.linearIssueId,
44
+ orchestrationSettleUntil: null,
45
+ },
39
46
  });
40
47
  const dispatched = params.wakeDispatcher.recordEventAndDispatch(params.issue.projectId, params.issue.linearIssueId, {
41
48
  eventType: "delegated",
@@ -1,6 +1,7 @@
1
1
  import { resolveMergeQueueProtocol } from "./merge-queue-protocol.js";
2
2
  import { buildRepairWakeDedupeKey } from "./reactive-wake-keys.js";
3
3
  import { execCommand } from "./utils.js";
4
+ const WRITER = "queue-health-monitor";
4
5
  const QUEUE_HEALTH_GRACE_MS = 120_000;
5
6
  const QUEUE_HEALTH_PROBE_FAILURE_COOLDOWN_MS = 300_000;
6
7
  // Plan §6.2: an approved PR with red branch CI for >= this long is
@@ -113,8 +114,12 @@ export class QueueHealthMonitor {
113
114
  }
114
115
  this.probeFailureFeedTimes.delete(`${issue.projectId}::${issue.linearIssueId}`);
115
116
  if (pr.state === "MERGED") {
116
- this.db.issues.upsertIssue({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, prState: "merged" });
117
- this.advancer.advanceIdleIssue(issue, "done", { clearFailureProvenance: true });
117
+ const mergedCommit = this.db.issueSessions.commitIssueState({
118
+ writer: WRITER,
119
+ update: { projectId: issue.projectId, linearIssueId: issue.linearIssueId, prState: "merged" },
120
+ });
121
+ const merged = mergedCommit.outcome === "applied" ? mergedCommit.issue : issue;
122
+ this.advancer.advanceIdleIssue(merged, "done", { clearFailureProvenance: true });
118
123
  return;
119
124
  }
120
125
  if (pr.state !== "OPEN")
@@ -159,12 +164,16 @@ export class QueueHealthMonitor {
159
164
  if (isDuplicateProbe(issue, pendingRunContext)) {
160
165
  return;
161
166
  }
162
- this.db.issues.upsertIssue({
163
- projectId: issue.projectId,
164
- linearIssueId: issue.linearIssueId,
165
- lastAttemptedFailureHeadSha: headRefOid,
166
- lastAttemptedFailureSignature: signature,
167
+ const probedCommit = this.db.issueSessions.commitIssueState({
168
+ writer: WRITER,
169
+ update: {
170
+ projectId: issue.projectId,
171
+ linearIssueId: issue.linearIssueId,
172
+ lastAttemptedFailureHeadSha: headRefOid,
173
+ lastAttemptedFailureSignature: signature,
174
+ },
167
175
  });
176
+ const probed = probedCommit.outcome === "applied" ? probedCommit.issue : issue;
168
177
  this.advancer.wakeDispatcher.recordEventAndDispatch(issue.projectId, issue.linearIssueId, {
169
178
  eventType: "merge_steward_incident",
170
179
  eventJson: JSON.stringify(pendingRunContext),
@@ -175,7 +184,7 @@ export class QueueHealthMonitor {
175
184
  signature,
176
185
  }),
177
186
  });
178
- this.advancer.advanceIdleIssue(issue, "repairing_queue");
187
+ this.advancer.advanceIdleIssue(probed, "repairing_queue");
179
188
  this.logger.info({ issueKey: issue.issueKey, prNumber: issue.prNumber, headRefOid, reason }, "Queue health: queue issue detected, dispatching repair");
180
189
  this.feed?.publish({
181
190
  level: "warn",
@@ -2,6 +2,7 @@ import { CLEARED_FAILURE_PROVENANCE } from "./failure-provenance.js";
2
2
  import { buildReviewFixBranchUpkeepContext, isDirtyMergeStateStatus, isRequestedChangesRunType, readReactivePrSnapshot, } from "./reactive-pr-state.js";
3
3
  import { readReactivePublishDelta } from "./reactive-publish-delta.js";
4
4
  import { readLatestRequestedChangesReviewContext } from "./remote-pr-review.js";
5
+ const WRITER = "reactive-run-policy";
5
6
  const REACTIVE_SCOPE_RISK_PREFIXES = [
6
7
  ".github/workflows/",
7
8
  "scripts/bootstrap-worktree.",
@@ -273,7 +274,10 @@ export class ReactiveRunPolicy {
273
274
  }
274
275
  }
275
276
  upsertIssueIfLeaseHeld(projectId, linearIssueId, params, context) {
276
- const updated = this.withHeldLease(projectId, linearIssueId, (lease) => this.db.issueSessions.upsertIssueWithLease(lease, params));
277
+ const updated = this.withHeldLease(projectId, linearIssueId, (lease) => {
278
+ const commit = this.db.issueSessions.commitIssueState({ writer: WRITER, lease, update: params });
279
+ return commit.outcome === "applied" ? commit.issue : undefined;
280
+ });
277
281
  if (updated === undefined) {
278
282
  this.logger.warn({ projectId, linearIssueId, context }, "Skipping issue write after losing issue-session lease");
279
283
  }
@@ -1,12 +1,46 @@
1
- export const DEFAULT_CI_REPAIR_BUDGET = 10;
2
- export const DEFAULT_QUEUE_REPAIR_BUDGET = 10;
3
- export const DEFAULT_REVIEW_FIX_BUDGET = 10;
1
+ export const DEFAULT_RUN_BUDGETS = {
2
+ ciRepair: 10,
3
+ queueRepair: 10,
4
+ reviewFix: 10,
5
+ zombieRecovery: 5,
6
+ };
7
+ export function resolveRunBudgets(project) {
8
+ return {
9
+ ciRepair: project?.repairBudgets?.ciRepair ?? DEFAULT_RUN_BUDGETS.ciRepair,
10
+ queueRepair: project?.repairBudgets?.queueRepair ?? DEFAULT_RUN_BUDGETS.queueRepair,
11
+ reviewFix: project?.repairBudgets?.reviewFix ?? DEFAULT_RUN_BUDGETS.reviewFix,
12
+ // No per-project override exists for zombie recovery yet; add one to
13
+ // ProjectConfig.repairBudgets if a project ever needs it.
14
+ zombieRecovery: DEFAULT_RUN_BUDGETS.zombieRecovery,
15
+ };
16
+ }
4
17
  export function getCiRepairBudget(project) {
5
- return project?.repairBudgets?.ciRepair ?? DEFAULT_CI_REPAIR_BUDGET;
18
+ return resolveRunBudgets(project).ciRepair;
6
19
  }
7
20
  export function getQueueRepairBudget(project) {
8
- return project?.repairBudgets?.queueRepair ?? DEFAULT_QUEUE_REPAIR_BUDGET;
21
+ return resolveRunBudgets(project).queueRepair;
9
22
  }
10
23
  export function getReviewFixBudget(project) {
11
- return project?.repairBudgets?.reviewFix ?? DEFAULT_REVIEW_FIX_BUDGET;
24
+ return resolveRunBudgets(project).reviewFix;
25
+ }
26
+ export function getZombieRecoveryBudget(project) {
27
+ return resolveRunBudgets(project).zombieRecovery;
28
+ }
29
+ // ─── Zombie-recovery backoff schedule (formerly zombie-recovery.ts) ──
30
+ //
31
+ // Exponential backoff between retries of a run that died without doing
32
+ // its work. Owned here with the budgets so the whole retry discipline
33
+ // (how many attempts, how far apart) reads in one place.
34
+ const ZOMBIE_RECOVERY_BASE_DELAY_MS = 15_000;
35
+ export function getZombieRecoveryDelayMs(recoveryAttempts) {
36
+ return ZOMBIE_RECOVERY_BASE_DELAY_MS * Math.pow(2, recoveryAttempts);
37
+ }
38
+ export function getRemainingZombieRecoveryDelayMs(lastRecoveryAt, recoveryAttempts, now = Date.now()) {
39
+ if (!lastRecoveryAt)
40
+ return 0;
41
+ const recoveredAtMs = Date.parse(lastRecoveryAt);
42
+ if (!Number.isFinite(recoveredAtMs))
43
+ return 0;
44
+ const delay = getZombieRecoveryDelayMs(recoveryAttempts);
45
+ return Math.max(0, recoveredAtMs + delay - now);
12
46
  }
@@ -1,18 +1,59 @@
1
1
  import { ACTIVE_RUN_STATES } from "./factory-state.js";
2
2
  import { ImplementationOutcomePolicy } from "./implementation-outcome-policy.js";
3
3
  import { ReactiveRunPolicy } from "./reactive-run-policy.js";
4
- function resolvePostRunState(issue) {
5
- if (ACTIVE_RUN_STATES.has(issue.factoryState) && issue.prNumber) {
4
+ import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
5
+ // Plan §B3: the one post-run factory-state resolver. Unifies the former
6
+ // `resolveCompletedRunState` (run-completion-policy) and
7
+ // `resolveRecoverablePostRunState` (interrupted-run-recovery).
8
+ //
9
+ // Shared rule (both old functions agreed):
10
+ // - no PR on the issue → undefined (nothing to resolve from PR truth);
11
+ // - approved open/closed PR → awaiting_queue; otherwise pr_open;
12
+ // - merged PR (while the issue is in an active-run state) → done.
13
+ //
14
+ // The two old functions genuinely disagreed in two places, and the
15
+ // disagreement is semantic, so it survives as the `outcome` option rather
16
+ // than being averaged away:
17
+ // - outcome "completed" (the run did its work, default): gate every write
18
+ // on ACTIVE_RUN_STATES so a state advanced concurrently by webhooks
19
+ // (e.g. deploying, awaiting_queue) is never clobbered, and never
20
+ // re-derive a reactive repair state — the stale GitHub verdict
21
+ // (changes_requested / red CI) refers to the head the run just
22
+ // replaced, and routing it again would loop the fix forever.
23
+ // - outcome "recovered" (the run died without doing its work): GitHub
24
+ // truth is authoritative regardless of the local factory state —
25
+ // merged → done unconditionally, and an open PR re-derives the
26
+ // reactive intent (repairing_ci / repairing_queue / changes_requested)
27
+ // so the original problem is routed again.
28
+ export function resolvePostRunFactoryState(issue, _run, options) {
29
+ if (!issue.prNumber)
30
+ return undefined;
31
+ if (options?.outcome === "recovered") {
6
32
  if (issue.prState === "merged")
7
33
  return "done";
8
- if (issue.prReviewState === "approved")
9
- return "awaiting_queue";
10
- return "pr_open";
34
+ if (issue.prState === "open") {
35
+ const reactiveIntent = deriveIssueSessionReactiveIntent({
36
+ prNumber: issue.prNumber,
37
+ prState: issue.prState,
38
+ prReviewState: issue.prReviewState,
39
+ prCheckStatus: issue.prCheckStatus,
40
+ latestFailureSource: issue.lastGitHubFailureSource,
41
+ });
42
+ if (reactiveIntent)
43
+ return reactiveIntent.compatibilityFactoryState;
44
+ if (issue.prReviewState === "approved")
45
+ return "awaiting_queue";
46
+ return "pr_open";
47
+ }
48
+ // Closed (or unknown) PR: fall through to the factory-state-gated rule.
11
49
  }
12
- return undefined;
13
- }
14
- export function resolveCompletedRunState(issue, _run) {
15
- return resolvePostRunState(issue);
50
+ if (!ACTIVE_RUN_STATES.has(issue.factoryState))
51
+ return undefined;
52
+ if (issue.prState === "merged")
53
+ return "done";
54
+ if (issue.prReviewState === "approved")
55
+ return "awaiting_queue";
56
+ return "pr_open";
16
57
  }
17
58
  export class RunCompletionPolicy {
18
59
  reactive;