patchrelay 0.74.8 → 0.75.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/cluster-health/index.js +21 -9
- package/dist/cli/cluster-health/local-issue-health.js +15 -6
- package/dist/cli/output.js +1 -1
- package/dist/cli/watch/use-detail-stream.js +64 -6
- package/dist/db.js +1 -0
- package/dist/http.js +13 -0
- package/dist/idle-reconciliation.js +26 -1
- package/dist/linear-client.js +39 -0
- package/dist/linear-issue-projection.js +79 -0
- package/dist/merged-linear-completion-reconciler.js +2 -11
- package/dist/queue-failure-policy.js +11 -0
- package/dist/run-admission-controller.js +23 -0
- package/dist/run-orchestrator.js +20 -5
- package/dist/service-queue.js +40 -8
- package/dist/service-runtime.js +28 -1
- package/dist/service-startup-recovery.js +94 -13
- package/dist/service.js +21 -2
- package/dist/telemetry.js +16 -0
- package/dist/terminal-wake-reconciler.js +28 -0
- package/dist/webhooks/issue-dependency-sync.js +2 -11
- package/package.json +1 -1
package/dist/build-info.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { hasOpenPr } from "../../pr-state.js";
|
|
2
2
|
import { collectActiveOverlapFindings } from "./active-overlap.js";
|
|
3
|
-
import { evaluateLocalIssueHealth, isResolvedDependency, needsReviewAutomation, } from "./local-issue-health.js";
|
|
3
|
+
import { evaluateLocalIssueHealth, evaluateTerminalIssueHealth, isActiveWorkflowIssue, isTerminalFailureIssue, isResolvedDependency, needsReviewAutomation, } from "./local-issue-health.js";
|
|
4
4
|
import { evaluateGitHubIssueHealth } from "./github-issue-health.js";
|
|
5
5
|
import { collectReviewQuillAttemptOwners, } from "./review-quill-probe.js";
|
|
6
6
|
import { probeOptionalService, probePatchRelayService, } from "./service-probe.js";
|
|
@@ -10,7 +10,8 @@ export async function collectClusterHealth(config, db, runCommand) {
|
|
|
10
10
|
const ciEntries = [];
|
|
11
11
|
const now = Date.now();
|
|
12
12
|
const issues = db.listIssues();
|
|
13
|
-
const
|
|
13
|
+
const activeWorkflowIssues = issues.filter((issue) => isActiveWorkflowIssue(issue));
|
|
14
|
+
const historicalTerminalIssues = issues.filter((issue) => isTerminalFailureIssue(issue));
|
|
14
15
|
const trackedByKey = new Map(issues
|
|
15
16
|
.filter((issue) => issue.issueKey)
|
|
16
17
|
.map((issue) => [issue.issueKey, issue]));
|
|
@@ -21,7 +22,7 @@ export async function collectClusterHealth(config, db, runCommand) {
|
|
|
21
22
|
scope: "service:patchrelay",
|
|
22
23
|
message: patchRelayProbe.message,
|
|
23
24
|
});
|
|
24
|
-
const snapshots =
|
|
25
|
+
const snapshots = activeWorkflowIssues.map((issue) => {
|
|
25
26
|
const tracked = db.getTrackedIssue(issue.projectId, issue.linearIssueId);
|
|
26
27
|
const deps = db.issues.listIssueDependencies(issue.projectId, issue.linearIssueId);
|
|
27
28
|
const blockedBy = deps.filter((dep) => !isResolvedDependency(dep));
|
|
@@ -99,6 +100,17 @@ export async function collectClusterHealth(config, db, runCommand) {
|
|
|
99
100
|
});
|
|
100
101
|
}
|
|
101
102
|
}
|
|
103
|
+
for (const issue of historicalTerminalIssues) {
|
|
104
|
+
const finding = evaluateTerminalIssueHealth(issue);
|
|
105
|
+
if (finding) {
|
|
106
|
+
checks.push({
|
|
107
|
+
...finding,
|
|
108
|
+
...(issue.issueKey ? { issueKey: issue.issueKey } : {}),
|
|
109
|
+
projectId: issue.projectId,
|
|
110
|
+
...(issue.prNumber !== undefined ? { prNumber: issue.prNumber } : {}),
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
102
114
|
checks.push(...await collectActiveOverlapFindings(snapshots, runCommand));
|
|
103
115
|
for (const snapshot of snapshots) {
|
|
104
116
|
if (!hasOpenPr(snapshot.issue.prNumber, snapshot.issue.prState)) {
|
|
@@ -118,18 +130,18 @@ export async function collectClusterHealth(config, db, runCommand) {
|
|
|
118
130
|
}
|
|
119
131
|
}
|
|
120
132
|
const workflowFailures = checks.filter((check) => check.scope.startsWith("issue:") || check.scope.startsWith("github:"));
|
|
121
|
-
if (workflowFailures.every((check) => check.status === "pass" || check.status === "warn") &&
|
|
133
|
+
if (workflowFailures.every((check) => check.status === "pass" || check.status === "warn") && activeWorkflowIssues.length > 0) {
|
|
122
134
|
checks.push({
|
|
123
135
|
status: "pass",
|
|
124
136
|
scope: "workflow",
|
|
125
|
-
message: `All ${
|
|
137
|
+
message: `All ${activeWorkflowIssues.length} active workflow issues currently have active work, a tracked blocker, or a downstream owner`,
|
|
126
138
|
});
|
|
127
139
|
}
|
|
128
|
-
if (
|
|
140
|
+
if (activeWorkflowIssues.length === 0) {
|
|
129
141
|
checks.push({
|
|
130
142
|
status: "pass",
|
|
131
143
|
scope: "workflow",
|
|
132
|
-
message: "No
|
|
144
|
+
message: "No active workflow issues are currently tracked",
|
|
133
145
|
});
|
|
134
146
|
}
|
|
135
147
|
if (ciEntries.length > 0) {
|
|
@@ -144,8 +156,8 @@ export async function collectClusterHealth(config, db, runCommand) {
|
|
|
144
156
|
}
|
|
145
157
|
const summary = {
|
|
146
158
|
trackedIssues: issues.length,
|
|
147
|
-
openIssues:
|
|
148
|
-
activeRuns:
|
|
159
|
+
openIssues: activeWorkflowIssues.length,
|
|
160
|
+
activeRuns: activeWorkflowIssues.filter((issue) => issue.activeRunId !== undefined).length,
|
|
149
161
|
blockedIssues: snapshots.filter((snapshot) => snapshot.blockedBy.length > 0).length,
|
|
150
162
|
readyIssues: snapshots.filter((snapshot) => snapshot.readyForExecution).length,
|
|
151
163
|
ciTrackedPrs: ciEntries.length,
|
|
@@ -13,21 +13,30 @@ export function isResolvedDependency(dep) {
|
|
|
13
13
|
|| state === "cancelled";
|
|
14
14
|
}
|
|
15
15
|
export function needsReviewAutomation(issue) {
|
|
16
|
-
if (issue.factoryState === "awaiting_queue" || issue
|
|
16
|
+
if (issue.factoryState === "awaiting_queue" || !isActiveWorkflowIssue(issue)) {
|
|
17
17
|
return false;
|
|
18
18
|
}
|
|
19
19
|
return hasOpenPr(issue.prNumber, issue.prState);
|
|
20
20
|
}
|
|
21
|
-
export function
|
|
22
|
-
|
|
23
|
-
|
|
21
|
+
export function isActiveWorkflowIssue(issue) {
|
|
22
|
+
return issue.factoryState !== "done" && !isTerminalFailureIssue(issue);
|
|
23
|
+
}
|
|
24
|
+
export function isTerminalFailureIssue(issue) {
|
|
25
|
+
return issue.factoryState === "failed" || issue.factoryState === "escalated";
|
|
26
|
+
}
|
|
27
|
+
export function evaluateTerminalIssueHealth(issue) {
|
|
24
28
|
if (issue.factoryState === "failed" || issue.factoryState === "escalated") {
|
|
25
29
|
return {
|
|
26
|
-
status: "
|
|
30
|
+
status: "warn",
|
|
27
31
|
scope: "issue:terminal",
|
|
28
|
-
message: `
|
|
32
|
+
message: `Historical terminal issue is in failure state ${issue.factoryState}`,
|
|
29
33
|
};
|
|
30
34
|
}
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
export function evaluateLocalIssueHealth(snapshot) {
|
|
38
|
+
const { issue, session, missingTrackedBlockers, blockedBy, ageMs, readyForExecution } = snapshot;
|
|
39
|
+
const pausedNoPrWork = isUndelegatedPausedNoPrWork(issue);
|
|
31
40
|
if (missingTrackedBlockers.length > 0) {
|
|
32
41
|
return {
|
|
33
42
|
status: "fail",
|
package/dist/cli/output.js
CHANGED
|
@@ -34,7 +34,7 @@ export function formatClusterHealth(report) {
|
|
|
34
34
|
lines.push(`${marker} [${detail}] ${check.message}`);
|
|
35
35
|
}
|
|
36
36
|
lines.push("");
|
|
37
|
-
lines.push(`Summary: tracked=${report.summary.trackedIssues}
|
|
37
|
+
lines.push(`Summary: tracked=${report.summary.trackedIssues} active=${report.summary.openIssues} active_runs=${report.summary.activeRuns} blocked=${report.summary.blockedIssues} ready=${report.summary.readyIssues}`);
|
|
38
38
|
if (report.summary.ciTrackedPrs > 0) {
|
|
39
39
|
lines.push(`CI summary: prs=${report.summary.ciTrackedPrs} pending=${report.summary.ciPending} success=${report.summary.ciSuccess} failure=${report.summary.ciFailure} unknown=${report.summary.ciUnknown} missing_owner=${report.summary.ciOrphaned}`);
|
|
40
40
|
for (const entry of report.ci) {
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { useEffect, useRef } from "react";
|
|
2
|
+
const DETAIL_REHYDRATE_INTERVAL_MS = 3000;
|
|
3
|
+
const FEED_REHYDRATE_LIMIT = 100;
|
|
4
|
+
const MAX_CACHED_FEED_EVENTS = 300;
|
|
2
5
|
export function useDetailStream(options) {
|
|
3
6
|
const optionsRef = useRef(options);
|
|
4
7
|
optionsRef.current = options;
|
|
@@ -14,10 +17,24 @@ export function useDetailStream(options) {
|
|
|
14
17
|
if (bearerToken) {
|
|
15
18
|
headers.authorization = `Bearer ${bearerToken}`;
|
|
16
19
|
}
|
|
17
|
-
|
|
20
|
+
const feedState = {
|
|
21
|
+
lastFeedEventId: undefined,
|
|
22
|
+
feedEvents: [],
|
|
23
|
+
};
|
|
24
|
+
let inFlight = false;
|
|
25
|
+
const runRehydrate = () => {
|
|
26
|
+
if (inFlight)
|
|
27
|
+
return;
|
|
28
|
+
inFlight = true;
|
|
29
|
+
void rehydrate(baseUrl, issueKey, headers, abortController.signal, dispatch, feedState)
|
|
30
|
+
.finally(() => {
|
|
31
|
+
inFlight = false;
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
runRehydrate();
|
|
18
35
|
const intervalId = setInterval(() => {
|
|
19
|
-
|
|
20
|
-
},
|
|
36
|
+
runRehydrate();
|
|
37
|
+
}, DETAIL_REHYDRATE_INTERVAL_MS);
|
|
21
38
|
return () => {
|
|
22
39
|
clearInterval(intervalId);
|
|
23
40
|
abortController.abort();
|
|
@@ -25,17 +42,22 @@ export function useDetailStream(options) {
|
|
|
25
42
|
}, [options.issueKey, options.active]);
|
|
26
43
|
}
|
|
27
44
|
// ─── Rehydration ──────────────────────────────────────────────────
|
|
28
|
-
async function rehydrate(baseUrl, issueKey, headers, signal, dispatch) {
|
|
45
|
+
async function rehydrate(baseUrl, issueKey, headers, signal, dispatch, feedState) {
|
|
29
46
|
try {
|
|
30
47
|
const url = new URL(`/api/issues/${encodeURIComponent(issueKey)}`, baseUrl);
|
|
31
|
-
const
|
|
48
|
+
const feedUrl = buildFeedUrl(baseUrl, issueKey, feedState.lastFeedEventId);
|
|
49
|
+
const [response, newFeedEvents] = await Promise.all([
|
|
50
|
+
fetch(url, { headers: { ...headers, accept: "application/json" }, signal }),
|
|
51
|
+
fetchFeedEvents(feedUrl, headers, signal),
|
|
52
|
+
]);
|
|
32
53
|
if (!response.ok)
|
|
33
54
|
return;
|
|
55
|
+
updateFeedState(feedState, newFeedEvents);
|
|
34
56
|
const data = await response.json();
|
|
35
57
|
dispatch({
|
|
36
58
|
type: "timeline-rehydrate",
|
|
37
59
|
runs: Array.isArray(data.runs) ? data.runs : [],
|
|
38
|
-
feedEvents:
|
|
60
|
+
feedEvents: feedState.feedEvents,
|
|
39
61
|
liveThread: data.liveThread ?? null,
|
|
40
62
|
activeRunId: data.activeRun?.id ?? null,
|
|
41
63
|
activeRunStartedAt: data.activeRun?.startedAt ?? null,
|
|
@@ -46,3 +68,39 @@ async function rehydrate(baseUrl, issueKey, headers, signal, dispatch) {
|
|
|
46
68
|
// Rehydration is best-effort
|
|
47
69
|
}
|
|
48
70
|
}
|
|
71
|
+
function buildFeedUrl(baseUrl, issueKey, afterId) {
|
|
72
|
+
const feedUrl = new URL(`/api/issues/${encodeURIComponent(issueKey)}/feed`, baseUrl);
|
|
73
|
+
feedUrl.searchParams.set("limit", String(FEED_REHYDRATE_LIMIT));
|
|
74
|
+
if (afterId !== undefined) {
|
|
75
|
+
feedUrl.searchParams.set("afterId", String(afterId));
|
|
76
|
+
}
|
|
77
|
+
return feedUrl;
|
|
78
|
+
}
|
|
79
|
+
async function fetchFeedEvents(url, headers, signal) {
|
|
80
|
+
try {
|
|
81
|
+
const response = await fetch(url, { headers: { ...headers, accept: "application/json" }, signal });
|
|
82
|
+
if (!response.ok)
|
|
83
|
+
return [];
|
|
84
|
+
const data = await response.json();
|
|
85
|
+
return Array.isArray(data.events) ? data.events : [];
|
|
86
|
+
}
|
|
87
|
+
catch {
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
function updateFeedState(feedState, newEvents) {
|
|
92
|
+
if (newEvents.length === 0)
|
|
93
|
+
return;
|
|
94
|
+
const byId = new Map();
|
|
95
|
+
for (const event of feedState.feedEvents) {
|
|
96
|
+
byId.set(event.id, event);
|
|
97
|
+
}
|
|
98
|
+
for (const event of newEvents) {
|
|
99
|
+
byId.set(event.id, event);
|
|
100
|
+
}
|
|
101
|
+
const feedEvents = [...byId.values()]
|
|
102
|
+
.sort((left, right) => left.id - right.id)
|
|
103
|
+
.slice(-MAX_CACHED_FEED_EVENTS);
|
|
104
|
+
feedState.feedEvents = feedEvents;
|
|
105
|
+
feedState.lastFeedEventId = feedEvents.at(-1)?.id;
|
|
106
|
+
}
|
package/dist/db.js
CHANGED
|
@@ -38,6 +38,7 @@ export class PatchRelayDatabase {
|
|
|
38
38
|
this.connection.pragma("foreign_keys = ON");
|
|
39
39
|
if (wal) {
|
|
40
40
|
this.connection.pragma("journal_mode = WAL");
|
|
41
|
+
this.connection.pragma("synchronous = NORMAL");
|
|
41
42
|
}
|
|
42
43
|
this.linearInstallations = new LinearInstallationStore(this.connection);
|
|
43
44
|
this.operatorFeed = new OperatorFeedStore(this.connection);
|
package/dist/http.js
CHANGED
|
@@ -264,6 +264,19 @@ export async function buildHttpServer(config, service, logger) {
|
|
|
264
264
|
}
|
|
265
265
|
return reply.send({ ok: true, ...result });
|
|
266
266
|
});
|
|
267
|
+
app.get("/api/issues/:issueKey/feed", async (request, reply) => {
|
|
268
|
+
const issueKey = request.params.issueKey;
|
|
269
|
+
const afterId = getPositiveIntegerQueryParam(request, "afterId");
|
|
270
|
+
const requestedLimit = getPositiveIntegerQueryParam(request, "limit");
|
|
271
|
+
const result = service.listIssueFeedEvents(issueKey, {
|
|
272
|
+
...(afterId !== undefined ? { afterId } : {}),
|
|
273
|
+
limit: requestedLimit ? Math.min(requestedLimit, 100) : 100,
|
|
274
|
+
});
|
|
275
|
+
if (!result) {
|
|
276
|
+
return reply.code(404).send({ ok: false, reason: "issue_not_found" });
|
|
277
|
+
}
|
|
278
|
+
return reply.send({ ok: true, events: result.events });
|
|
279
|
+
});
|
|
267
280
|
app.get("/api/issues/:issueKey/live", async (request, reply) => {
|
|
268
281
|
const issueKey = request.params.issueKey;
|
|
269
282
|
const result = await service.getActiveRunStatus(issueKey);
|
|
@@ -12,6 +12,10 @@ import { fetchPullRequestSnapshot } from "./reconcile-pr-fetch.js";
|
|
|
12
12
|
import { buildPrStateUpdates } from "./reconcile-pr-state-updates.js";
|
|
13
13
|
import { buildRepairWakeDedupeKey, buildRequestedChangesWakeIdentity, reactiveWakeEventType } from "./reactive-wake-keys.js";
|
|
14
14
|
import { execCommand } from "./utils.js";
|
|
15
|
+
import { LinearIssueProjectionService } from "./linear-issue-projection.js";
|
|
16
|
+
import { TerminalWakeReconciler } from "./terminal-wake-reconciler.js";
|
|
17
|
+
const BLOCKED_DEPENDENCY_REFRESH_SUCCESS_BACKOFF_MS = 60_000;
|
|
18
|
+
const BLOCKED_DEPENDENCY_REFRESH_FAILURE_BACKOFF_MS = 5 * 60_000;
|
|
15
19
|
export class IdleIssueReconciler {
|
|
16
20
|
db;
|
|
17
21
|
config;
|
|
@@ -20,9 +24,13 @@ export class IdleIssueReconciler {
|
|
|
20
24
|
feed;
|
|
21
25
|
deployEvaluator;
|
|
22
26
|
syncIssue;
|
|
27
|
+
linearProvider;
|
|
28
|
+
blockedDependencyRefreshAfter = new Map();
|
|
29
|
+
terminalWakeReconciler;
|
|
30
|
+
linearIssueProjection;
|
|
23
31
|
constructor(db, config, wakeDispatcher, logger, feed,
|
|
24
32
|
// Injectable for tests; production uses the real `gh`-backed watcher.
|
|
25
|
-
deployEvaluator = evaluateDeploy, syncIssue) {
|
|
33
|
+
deployEvaluator = evaluateDeploy, syncIssue, linearProvider) {
|
|
26
34
|
this.db = db;
|
|
27
35
|
this.config = config;
|
|
28
36
|
this.wakeDispatcher = wakeDispatcher;
|
|
@@ -30,6 +38,11 @@ export class IdleIssueReconciler {
|
|
|
30
38
|
this.feed = feed;
|
|
31
39
|
this.deployEvaluator = deployEvaluator;
|
|
32
40
|
this.syncIssue = syncIssue;
|
|
41
|
+
this.linearProvider = linearProvider;
|
|
42
|
+
this.terminalWakeReconciler = new TerminalWakeReconciler(db, logger);
|
|
43
|
+
this.linearIssueProjection = linearProvider
|
|
44
|
+
? new LinearIssueProjectionService(db, linearProvider, logger)
|
|
45
|
+
: undefined;
|
|
33
46
|
}
|
|
34
47
|
async reconcile() {
|
|
35
48
|
// Wrap the entire reconcile pass in a dispatcher tick. Every
|
|
@@ -82,9 +95,21 @@ export class IdleIssueReconciler {
|
|
|
82
95
|
continue;
|
|
83
96
|
await this.reconcileFromGitHub(issue);
|
|
84
97
|
}
|
|
98
|
+
this.terminalWakeReconciler.reconcile();
|
|
85
99
|
for (const issue of this.db.issues.listBlockedDelegatedIssues()) {
|
|
86
100
|
if (!issue.delegatedToPatchRelay)
|
|
87
101
|
continue;
|
|
102
|
+
const dependencyKey = `${issue.projectId}::${issue.linearIssueId}`;
|
|
103
|
+
const refreshAfter = this.blockedDependencyRefreshAfter.get(dependencyKey);
|
|
104
|
+
if (this.linearIssueProjection) {
|
|
105
|
+
if (refreshAfter === undefined || refreshAfter <= Date.now()) {
|
|
106
|
+
const refresh = await this.linearIssueProjection.refreshIssue(issue.projectId, issue.linearIssueId);
|
|
107
|
+
this.blockedDependencyRefreshAfter.set(dependencyKey, Date.now() + (refresh.refreshed ? BLOCKED_DEPENDENCY_REFRESH_SUCCESS_BACKOFF_MS : BLOCKED_DEPENDENCY_REFRESH_FAILURE_BACKOFF_MS));
|
|
108
|
+
if (!refresh.refreshed) {
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
88
113
|
const unresolved = this.db.issues.countUnresolvedBlockers(issue.projectId, issue.linearIssueId);
|
|
89
114
|
if (unresolved === 0) {
|
|
90
115
|
this.wakeDispatcher.recordEventAndDispatch(issue.projectId, issue.linearIssueId, {
|
package/dist/linear-client.js
CHANGED
|
@@ -113,6 +113,45 @@ export class LinearGraphqlClient {
|
|
|
113
113
|
}
|
|
114
114
|
return this.mapIssue(response.issue);
|
|
115
115
|
}
|
|
116
|
+
async listIssuesDelegatedTo(params) {
|
|
117
|
+
const teamIds = params.teamIds.filter((teamId) => teamId.trim().length > 0);
|
|
118
|
+
if (teamIds.length === 0) {
|
|
119
|
+
return [];
|
|
120
|
+
}
|
|
121
|
+
const first = Math.max(1, Math.min(params.first ?? 100, 100));
|
|
122
|
+
const issues = [];
|
|
123
|
+
let after;
|
|
124
|
+
do {
|
|
125
|
+
const response = await this.request(`
|
|
126
|
+
query PatchRelayDelegatedIssues($delegateId: ID!, $teamIds: [ID!], $first: Int!, $after: String) {
|
|
127
|
+
issues(
|
|
128
|
+
first: $first
|
|
129
|
+
after: $after
|
|
130
|
+
filter: {
|
|
131
|
+
delegate: { id: { eq: $delegateId } }
|
|
132
|
+
team: { id: { in: $teamIds } }
|
|
133
|
+
}
|
|
134
|
+
) {
|
|
135
|
+
nodes {
|
|
136
|
+
${LINEAR_ISSUE_SELECTION}
|
|
137
|
+
}
|
|
138
|
+
pageInfo {
|
|
139
|
+
hasNextPage
|
|
140
|
+
endCursor
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
`, {
|
|
145
|
+
delegateId: params.delegateId,
|
|
146
|
+
teamIds,
|
|
147
|
+
first,
|
|
148
|
+
after: after ?? null,
|
|
149
|
+
});
|
|
150
|
+
issues.push(...(response.issues.nodes ?? []).map((issue) => this.mapIssue(issue)));
|
|
151
|
+
after = response.issues.pageInfo.hasNextPage ? response.issues.pageInfo.endCursor ?? undefined : undefined;
|
|
152
|
+
} while (after);
|
|
153
|
+
return issues;
|
|
154
|
+
}
|
|
116
155
|
async createIssue(params) {
|
|
117
156
|
const response = await this.request(`
|
|
118
157
|
mutation PatchRelayCreateIssue($input: IssueCreateInput!) {
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
export class LinearIssueProjectionService {
|
|
2
|
+
db;
|
|
3
|
+
linearProvider;
|
|
4
|
+
logger;
|
|
5
|
+
constructor(db, linearProvider, logger) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
this.linearProvider = linearProvider;
|
|
8
|
+
this.logger = logger;
|
|
9
|
+
}
|
|
10
|
+
async refreshIssue(projectId, linearIssueId) {
|
|
11
|
+
return refreshIssueFromLinear({
|
|
12
|
+
db: this.db,
|
|
13
|
+
linearProvider: this.linearProvider,
|
|
14
|
+
projectId,
|
|
15
|
+
linearIssueId,
|
|
16
|
+
logger: this.logger,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export async function refreshIssueFromLinear(params) {
|
|
21
|
+
const linear = await params.linearProvider.forProject(params.projectId).catch((error) => {
|
|
22
|
+
params.logger?.warn({
|
|
23
|
+
projectId: params.projectId,
|
|
24
|
+
linearIssueId: params.linearIssueId,
|
|
25
|
+
error: error instanceof Error ? error.message : String(error),
|
|
26
|
+
}, "Failed to resolve Linear client while refreshing issue projection");
|
|
27
|
+
return undefined;
|
|
28
|
+
});
|
|
29
|
+
if (!linear) {
|
|
30
|
+
return { refreshed: false, error: "linear_client_unavailable" };
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
const liveIssue = await linear.getIssue(params.linearIssueId);
|
|
34
|
+
upsertLinearIssueProjection(params.db, params.projectId, liveIssue);
|
|
35
|
+
return { refreshed: true, liveIssue };
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
39
|
+
params.logger?.warn({
|
|
40
|
+
projectId: params.projectId,
|
|
41
|
+
linearIssueId: params.linearIssueId,
|
|
42
|
+
error: message,
|
|
43
|
+
}, "Failed to refresh issue projection from Linear");
|
|
44
|
+
return { refreshed: false, error: message };
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
export function upsertLinearIssueProjection(db, projectId, liveIssue) {
|
|
48
|
+
replaceIssueDependenciesFromLinearIssue(db, projectId, liveIssue);
|
|
49
|
+
db.issues.replaceIssueParentLink({
|
|
50
|
+
projectId,
|
|
51
|
+
childLinearIssueId: liveIssue.id,
|
|
52
|
+
parentLinearIssueId: liveIssue.parentId ?? null,
|
|
53
|
+
});
|
|
54
|
+
db.issues.upsertIssue({
|
|
55
|
+
projectId,
|
|
56
|
+
linearIssueId: liveIssue.id,
|
|
57
|
+
...(liveIssue.identifier ? { issueKey: liveIssue.identifier } : {}),
|
|
58
|
+
...(liveIssue.title ? { title: liveIssue.title } : {}),
|
|
59
|
+
...(liveIssue.description ? { description: liveIssue.description } : {}),
|
|
60
|
+
...(liveIssue.url ? { url: liveIssue.url } : {}),
|
|
61
|
+
...(liveIssue.priority != null ? { priority: liveIssue.priority } : {}),
|
|
62
|
+
...(liveIssue.estimate != null ? { estimate: liveIssue.estimate } : {}),
|
|
63
|
+
...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
|
|
64
|
+
...(liveIssue.stateType ? { currentLinearStateType: liveIssue.stateType } : {}),
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
export function replaceIssueDependenciesFromLinearIssue(db, projectId, liveIssue) {
|
|
68
|
+
db.issues.replaceIssueDependencies({
|
|
69
|
+
projectId,
|
|
70
|
+
linearIssueId: liveIssue.id,
|
|
71
|
+
blockers: liveIssue.blockedBy.map((blocker) => ({
|
|
72
|
+
blockerLinearIssueId: blocker.id,
|
|
73
|
+
...(blocker.identifier ? { blockerIssueKey: blocker.identifier } : {}),
|
|
74
|
+
...(blocker.title ? { blockerTitle: blocker.title } : {}),
|
|
75
|
+
...(blocker.stateName ? { blockerCurrentLinearState: blocker.stateName } : {}),
|
|
76
|
+
...(blocker.stateType ? { blockerCurrentLinearStateType: blocker.stateType } : {}),
|
|
77
|
+
})),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
@@ -2,6 +2,7 @@ import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
|
2
2
|
import { resolvePreferredCompletedLinearState } from "./linear-workflow.js";
|
|
3
3
|
import { isCompletedLinearState } from "./pr-state.js";
|
|
4
4
|
import { hasTrustedNoPrCompletion } from "./trusted-no-pr-completion.js";
|
|
5
|
+
import { replaceIssueDependenciesFromLinearIssue } from "./linear-issue-projection.js";
|
|
5
6
|
const COMPLETION_RECONCILE_WINDOW_MS = 60 * 60 * 1000;
|
|
6
7
|
const COMPLETION_RECONCILE_SUCCESS_BACKOFF_MS = 60 * 60 * 1000;
|
|
7
8
|
const COMPLETION_RECONCILE_FAILURE_BACKOFF_MS = 5 * 60 * 1000;
|
|
@@ -45,17 +46,7 @@ export class MergedLinearCompletionReconciler {
|
|
|
45
46
|
attemptedIssues += 1;
|
|
46
47
|
try {
|
|
47
48
|
const liveIssue = await linear.getIssue(issue.linearIssueId);
|
|
48
|
-
this.db.
|
|
49
|
-
projectId: issue.projectId,
|
|
50
|
-
linearIssueId: issue.linearIssueId,
|
|
51
|
-
blockers: liveIssue.blockedBy.map((blocker) => ({
|
|
52
|
-
blockerLinearIssueId: blocker.id,
|
|
53
|
-
...(blocker.identifier ? { blockerIssueKey: blocker.identifier } : {}),
|
|
54
|
-
...(blocker.title ? { blockerTitle: blocker.title } : {}),
|
|
55
|
-
...(blocker.stateName ? { blockerCurrentLinearState: blocker.stateName } : {}),
|
|
56
|
-
...(blocker.stateType ? { blockerCurrentLinearStateType: blocker.stateType } : {}),
|
|
57
|
-
})),
|
|
58
|
-
});
|
|
49
|
+
replaceIssueDependenciesFromLinearIssue(this.db, issue.projectId, liveIssue);
|
|
59
50
|
const latestRun = this.db.runs.getLatestRunForIssue(issue.projectId, issue.linearIssueId);
|
|
60
51
|
const trustedNoPrDone = hasTrustedNoPrCompletion(issue, latestRun);
|
|
61
52
|
if (issue.prState === "merged" || trustedNoPrDone) {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
const SQLITE_LOCK_RETRY_DELAYS_MS = [250, 1_000, 2_500, 5_000, 10_000];
|
|
2
|
+
export function retrySqliteLockedQueueFailure(error, attempt) {
|
|
3
|
+
if (!isSqliteDatabaseLockedError(error)) {
|
|
4
|
+
return undefined;
|
|
5
|
+
}
|
|
6
|
+
const delayMs = SQLITE_LOCK_RETRY_DELAYS_MS[attempt - 1];
|
|
7
|
+
return delayMs === undefined ? undefined : { delayMs };
|
|
8
|
+
}
|
|
9
|
+
export function isSqliteDatabaseLockedError(error) {
|
|
10
|
+
return /\bdatabase is locked\b/i.test(error.message);
|
|
11
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export class RunAdmissionController {
|
|
2
|
+
db;
|
|
3
|
+
linearIssueProjection;
|
|
4
|
+
constructor(db, linearIssueProjection) {
|
|
5
|
+
this.db = db;
|
|
6
|
+
this.linearIssueProjection = linearIssueProjection;
|
|
7
|
+
}
|
|
8
|
+
async check(params) {
|
|
9
|
+
if (params.runType !== "implementation") {
|
|
10
|
+
return { allowed: true };
|
|
11
|
+
}
|
|
12
|
+
const knownDependencyRows = this.db.issues.listIssueDependencies(params.projectId, params.linearIssueId).length;
|
|
13
|
+
const refresh = await this.linearIssueProjection.refreshIssue(params.projectId, params.linearIssueId);
|
|
14
|
+
if (!refresh.refreshed && knownDependencyRows > 0) {
|
|
15
|
+
return { allowed: false, reason: "dependency_refresh_failed", knownDependencyRows };
|
|
16
|
+
}
|
|
17
|
+
const blockerCount = this.db.issues.countUnresolvedBlockers(params.projectId, params.linearIssueId);
|
|
18
|
+
if (blockerCount > 0) {
|
|
19
|
+
return { allowed: false, reason: "blocked", blockerCount };
|
|
20
|
+
}
|
|
21
|
+
return { allowed: true };
|
|
22
|
+
}
|
|
23
|
+
}
|
package/dist/run-orchestrator.js
CHANGED
|
@@ -24,6 +24,8 @@ import { buildIssueTriageHash, IssueTriageService } from "./issue-triage.js";
|
|
|
24
24
|
import { loadConfig } from "./config.js";
|
|
25
25
|
import { CodexThreadMaterializingError, isThreadMaterializingError } from "./codex-thread-errors.js";
|
|
26
26
|
import { emitTelemetry, noopTelemetry } from "./telemetry.js";
|
|
27
|
+
import { LinearIssueProjectionService } from "./linear-issue-projection.js";
|
|
28
|
+
import { RunAdmissionController } from "./run-admission-controller.js";
|
|
27
29
|
function lowerCaseFirst(value) {
|
|
28
30
|
return value ? `${value.slice(0, 1).toLowerCase()}${value.slice(1)}` : value;
|
|
29
31
|
}
|
|
@@ -64,6 +66,8 @@ export class RunOrchestrator {
|
|
|
64
66
|
runNotificationHandler;
|
|
65
67
|
runReconciler;
|
|
66
68
|
mergedLinearCompletionReconciler;
|
|
69
|
+
linearIssueProjection;
|
|
70
|
+
runAdmission;
|
|
67
71
|
codexRuntimeConfig;
|
|
68
72
|
threadPorts = {
|
|
69
73
|
readThreadWithRetry: (threadId, maxRetries) => this.readThreadWithRetry(threadId, maxRetries),
|
|
@@ -133,7 +137,9 @@ export class RunOrchestrator {
|
|
|
133
137
|
this.interruptedRunRecovery = new InterruptedRunRecovery(db, logger, this.linearSync, this.leasePorts.withHeldLease, this.leasePorts.releaseLease, this.recoveryPorts.failRunAndClear, this.recoveryPorts.restoreIdleWorktree, this.runCompletionPolicy, (projectId, issueId) => this.enqueueIssue(projectId, issueId), feed);
|
|
134
138
|
this.runReconciler = new RunReconciler(db, logger, linearProvider, this.linearSync, this.interruptedRunRecovery, this.runFinalizer, this.leasePorts.withHeldLease, this.leasePorts.releaseLease, this.threadPorts.readThreadWithRetry, this.recoveryPorts.recoverOrEscalate, (projectId) => this.config.projects.find((project) => project.id === projectId)?.github?.repoFullName, feed);
|
|
135
139
|
this.runWakePlanner = new RunWakePlanner(db);
|
|
136
|
-
this.
|
|
140
|
+
this.linearIssueProjection = new LinearIssueProjectionService(db, linearProvider, logger);
|
|
141
|
+
this.runAdmission = new RunAdmissionController(db, this.linearIssueProjection);
|
|
142
|
+
this.idleReconciler = new IdleIssueReconciler(db, config, this.wakeDispatcher, logger, feed, undefined, (issue) => this.linearSync.syncSession(issue), linearProvider);
|
|
137
143
|
this.mergedLinearCompletionReconciler = new MergedLinearCompletionReconciler(db, linearProvider, logger);
|
|
138
144
|
this.queueHealthMonitor = new QueueHealthMonitor(db, config, {
|
|
139
145
|
advanceIdleIssue: (issue, newState, options) => this.idleReconciler.advanceIdleIssue(issue, newState, options),
|
|
@@ -312,12 +318,21 @@ export class RunOrchestrator {
|
|
|
312
318
|
return;
|
|
313
319
|
}
|
|
314
320
|
const { runType, context, resumeThread } = wake;
|
|
315
|
-
|
|
316
|
-
|
|
321
|
+
const admission = await this.runAdmission.check({
|
|
322
|
+
projectId: item.projectId,
|
|
323
|
+
linearIssueId: item.issueId,
|
|
324
|
+
runType,
|
|
325
|
+
});
|
|
326
|
+
if (!admission.allowed) {
|
|
317
327
|
this.db.issueSessions.clearPendingIssueSessionEventsRespectingActiveLease(item.projectId, item.issueId);
|
|
318
328
|
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
319
|
-
this.emitRunSkipped(item,
|
|
320
|
-
|
|
329
|
+
this.emitRunSkipped(item, admission.reason, issue, { runType, ...admission });
|
|
330
|
+
if (admission.reason === "dependency_refresh_failed") {
|
|
331
|
+
this.logger.info({ issueKey: issue.issueKey, projectId: item.projectId, knownDependencyRows: admission.knownDependencyRows }, "Skipped implementation launch because dependency refresh failed for an issue with known blockers");
|
|
332
|
+
}
|
|
333
|
+
else {
|
|
334
|
+
this.logger.info({ issueKey: issue.issueKey, blockerCount: admission.blockerCount }, "Skipped implementation launch because the issue is blocked");
|
|
335
|
+
}
|
|
321
336
|
return;
|
|
322
337
|
}
|
|
323
338
|
const remainingZombieDelayMs = shouldDelayZombieRecoveryLaunch(issue, issueSession, runType);
|
package/dist/service-queue.js
CHANGED
|
@@ -2,24 +2,33 @@ export class SerialWorkQueue {
|
|
|
2
2
|
onDequeue;
|
|
3
3
|
logger;
|
|
4
4
|
getKey;
|
|
5
|
+
options;
|
|
5
6
|
items = [];
|
|
6
7
|
queuedKeys = new Set();
|
|
7
8
|
pending = false;
|
|
8
|
-
constructor(onDequeue, logger, getKey) {
|
|
9
|
+
constructor(onDequeue, logger, getKey, options = {}) {
|
|
9
10
|
this.onDequeue = onDequeue;
|
|
10
11
|
this.logger = logger;
|
|
11
12
|
this.getKey = getKey;
|
|
13
|
+
this.options = options;
|
|
12
14
|
}
|
|
13
15
|
enqueue(item, options) {
|
|
16
|
+
this.enqueueEntry({ item, attempt: 0 }, options);
|
|
17
|
+
}
|
|
18
|
+
size() {
|
|
19
|
+
return this.items.length;
|
|
20
|
+
}
|
|
21
|
+
enqueueEntry(entry, options) {
|
|
22
|
+
const { item } = entry;
|
|
14
23
|
const key = this.getKey?.(item);
|
|
15
24
|
if (key && this.queuedKeys.has(key)) {
|
|
16
25
|
return;
|
|
17
26
|
}
|
|
18
27
|
if (options?.priority) {
|
|
19
|
-
this.items.unshift(
|
|
28
|
+
this.items.unshift(entry);
|
|
20
29
|
}
|
|
21
30
|
else {
|
|
22
|
-
this.items.push(
|
|
31
|
+
this.items.push(entry);
|
|
23
32
|
}
|
|
24
33
|
if (key) {
|
|
25
34
|
this.queuedKeys.add(key);
|
|
@@ -31,22 +40,45 @@ export class SerialWorkQueue {
|
|
|
31
40
|
});
|
|
32
41
|
}
|
|
33
42
|
}
|
|
43
|
+
scheduleRetry(entry, delayMs) {
|
|
44
|
+
const key = this.getKey?.(entry.item);
|
|
45
|
+
if (key && this.queuedKeys.has(key)) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if (key) {
|
|
49
|
+
this.queuedKeys.add(key);
|
|
50
|
+
}
|
|
51
|
+
const timer = setTimeout(() => {
|
|
52
|
+
if (key) {
|
|
53
|
+
this.queuedKeys.delete(key);
|
|
54
|
+
}
|
|
55
|
+
this.enqueueEntry(entry);
|
|
56
|
+
}, delayMs);
|
|
57
|
+
timer.unref?.();
|
|
58
|
+
}
|
|
34
59
|
async drain() {
|
|
35
60
|
while (this.items.length > 0) {
|
|
36
|
-
const
|
|
37
|
-
if (
|
|
61
|
+
const entry = this.items.shift();
|
|
62
|
+
if (entry === undefined) {
|
|
38
63
|
continue;
|
|
39
64
|
}
|
|
40
|
-
const key = this.getKey?.(
|
|
65
|
+
const key = this.getKey?.(entry.item);
|
|
41
66
|
if (key) {
|
|
42
67
|
this.queuedKeys.delete(key);
|
|
43
68
|
}
|
|
44
69
|
try {
|
|
45
|
-
await this.onDequeue(
|
|
70
|
+
await this.onDequeue(entry.item);
|
|
46
71
|
}
|
|
47
72
|
catch (error) {
|
|
48
73
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
49
|
-
|
|
74
|
+
const nextAttempt = entry.attempt + 1;
|
|
75
|
+
const retry = this.options.retryOnError?.(err, entry.item, nextAttempt);
|
|
76
|
+
if (retry) {
|
|
77
|
+
this.logger.warn({ item: entry.item, error: err.message, attempt: nextAttempt, retryDelayMs: retry.delayMs }, "Queue item processing failed; retrying");
|
|
78
|
+
this.scheduleRetry({ item: entry.item, attempt: nextAttempt }, retry.delayMs);
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
this.logger.error({ item: entry.item, error: err.message, stack: err.stack }, "Queue item processing failed");
|
|
50
82
|
}
|
|
51
83
|
}
|
|
52
84
|
this.pending = false;
|
package/dist/service-runtime.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { SerialWorkQueue } from "./service-queue.js";
|
|
2
|
+
import { retrySqliteLockedQueueFailure } from "./queue-failure-policy.js";
|
|
2
3
|
const ISSUE_KEY_DELIMITER = "::";
|
|
3
4
|
const DEFAULT_RECONCILE_INTERVAL_MS = 5_000;
|
|
4
5
|
const DEFAULT_RECONCILE_TIMEOUT_MS = 60_000;
|
|
6
|
+
const DEFAULT_MAX_ACTIVE_ISSUE_RUNS = 4;
|
|
5
7
|
function makeIssueQueueKey(item) {
|
|
6
8
|
return `${item.projectId}${ISSUE_KEY_DELIMITER}${item.issueId}`;
|
|
7
9
|
}
|
|
@@ -27,7 +29,9 @@ export class ServiceRuntime {
|
|
|
27
29
|
this.readyIssueSource = readyIssueSource;
|
|
28
30
|
this.options = options;
|
|
29
31
|
this.webhookQueue = new SerialWorkQueue((eventId) => webhookProcessor.processWebhookEvent(eventId), logger, (eventId) => String(eventId));
|
|
30
|
-
this.issueQueue = new SerialWorkQueue((item) => issueProcessor.processIssue(item), logger, makeIssueQueueKey
|
|
32
|
+
this.issueQueue = new SerialWorkQueue((item) => issueProcessor.processIssue(item), logger, makeIssueQueueKey, {
|
|
33
|
+
retryOnError: (error, _item, attempt) => retrySqliteLockedQueueFailure(error, attempt),
|
|
34
|
+
});
|
|
31
35
|
}
|
|
32
36
|
async start() {
|
|
33
37
|
try {
|
|
@@ -54,6 +58,16 @@ export class ServiceRuntime {
|
|
|
54
58
|
this.webhookQueue.enqueue(eventId, options);
|
|
55
59
|
}
|
|
56
60
|
enqueueIssue(projectId, issueId) {
|
|
61
|
+
if (!this.hasIssueRunCapacity()) {
|
|
62
|
+
this.logger.warn({
|
|
63
|
+
projectId,
|
|
64
|
+
issueId,
|
|
65
|
+
activeIssueRuns: this.getActiveIssueRunCount(),
|
|
66
|
+
queuedIssueRuns: this.issueQueue.size(),
|
|
67
|
+
maxActiveIssueRuns: this.getMaxActiveIssueRuns(),
|
|
68
|
+
}, "Skipped issue enqueue: active run capacity is full");
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
57
71
|
this.issueQueue.enqueue({ projectId, issueId });
|
|
58
72
|
}
|
|
59
73
|
setLinearConnected(connected) {
|
|
@@ -100,6 +114,9 @@ export class ServiceRuntime {
|
|
|
100
114
|
// Pick up issues that became ready outside the webhook path
|
|
101
115
|
// (e.g. CLI retry, manual DB edits) without requiring a restart.
|
|
102
116
|
for (const issue of this.readyIssueSource.listIssuesReadyForExecution()) {
|
|
117
|
+
if (!this.hasIssueRunCapacity()) {
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
103
120
|
this.enqueueIssue(issue.projectId, issue.linearIssueId);
|
|
104
121
|
}
|
|
105
122
|
}
|
|
@@ -113,6 +130,16 @@ export class ServiceRuntime {
|
|
|
113
130
|
}
|
|
114
131
|
}
|
|
115
132
|
}
|
|
133
|
+
getMaxActiveIssueRuns() {
|
|
134
|
+
const configured = this.options.maxActiveIssueRuns ?? DEFAULT_MAX_ACTIVE_ISSUE_RUNS;
|
|
135
|
+
return Math.max(1, Math.floor(configured));
|
|
136
|
+
}
|
|
137
|
+
getActiveIssueRunCount() {
|
|
138
|
+
return Math.max(0, this.readyIssueSource.countActiveIssueRuns?.() ?? 0);
|
|
139
|
+
}
|
|
140
|
+
hasIssueRunCapacity() {
|
|
141
|
+
return this.getActiveIssueRunCount() + this.issueQueue.size() < this.getMaxActiveIssueRuns();
|
|
142
|
+
}
|
|
116
143
|
}
|
|
117
144
|
function promiseWithTimeout(promise, timeoutMs, label) {
|
|
118
145
|
return new Promise((resolve, reject) => {
|
|
@@ -2,13 +2,16 @@ import { appendDelegationObservedEvent } from "./delegation-audit.js";
|
|
|
2
2
|
import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
3
3
|
import { isResumablePausedLocalWork } from "./paused-issue-state.js";
|
|
4
4
|
import { buildRepairWakeDedupeKey, buildRequestedChangesWakeIdentity, reactiveWakeEventType } from "./reactive-wake-keys.js";
|
|
5
|
+
import { upsertLinearIssueProjection } from "./linear-issue-projection.js";
|
|
5
6
|
export class ServiceStartupRecovery {
|
|
7
|
+
config;
|
|
6
8
|
db;
|
|
7
9
|
linearProvider;
|
|
8
10
|
linearSync;
|
|
9
11
|
enqueueIssue;
|
|
10
12
|
logger;
|
|
11
|
-
constructor(db, linearProvider, linearSync, enqueueIssue, logger) {
|
|
13
|
+
constructor(config, db, linearProvider, linearSync, enqueueIssue, logger) {
|
|
14
|
+
this.config = config;
|
|
12
15
|
this.db = db;
|
|
13
16
|
this.linearProvider = linearProvider;
|
|
14
17
|
this.linearSync = linearSync;
|
|
@@ -20,6 +23,9 @@ export class ServiceStartupRecovery {
|
|
|
20
23
|
if (issue.factoryState === "done") {
|
|
21
24
|
continue;
|
|
22
25
|
}
|
|
26
|
+
if (!issue.activeRunId) {
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
23
29
|
const syncedIssue = issue.agentSessionId
|
|
24
30
|
? issue
|
|
25
31
|
: (() => {
|
|
@@ -35,7 +41,11 @@ export class ServiceStartupRecovery {
|
|
|
35
41
|
if (!syncedIssue.agentSessionId) {
|
|
36
42
|
continue;
|
|
37
43
|
}
|
|
38
|
-
const
|
|
44
|
+
const activeRunId = syncedIssue.activeRunId;
|
|
45
|
+
if (!activeRunId) {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const activeRun = this.db.runs.getRunById(activeRunId);
|
|
39
49
|
if (!activeRun) {
|
|
40
50
|
continue;
|
|
41
51
|
}
|
|
@@ -43,6 +53,7 @@ export class ServiceStartupRecovery {
|
|
|
43
53
|
}
|
|
44
54
|
}
|
|
45
55
|
async recoverDelegatedIssueStateFromLinear() {
|
|
56
|
+
await this.discoverDelegatedIssuesFromLinear();
|
|
46
57
|
for (const issue of this.db.issues.listIssues()) {
|
|
47
58
|
if (issue.factoryState === "done" || issue.activeRunId !== undefined) {
|
|
48
59
|
continue;
|
|
@@ -59,17 +70,7 @@ export class ServiceStartupRecovery {
|
|
|
59
70
|
if (!liveIssue) {
|
|
60
71
|
continue;
|
|
61
72
|
}
|
|
62
|
-
this.db.
|
|
63
|
-
projectId: issue.projectId,
|
|
64
|
-
linearIssueId: issue.linearIssueId,
|
|
65
|
-
blockers: liveIssue.blockedBy.map((blocker) => ({
|
|
66
|
-
blockerLinearIssueId: blocker.id,
|
|
67
|
-
...(blocker.identifier ? { blockerIssueKey: blocker.identifier } : {}),
|
|
68
|
-
...(blocker.title ? { blockerTitle: blocker.title } : {}),
|
|
69
|
-
...(blocker.stateName ? { blockerCurrentLinearState: blocker.stateName } : {}),
|
|
70
|
-
...(blocker.stateType ? { blockerCurrentLinearStateType: blocker.stateType } : {}),
|
|
71
|
-
})),
|
|
72
|
-
});
|
|
73
|
+
upsertLinearIssueProjection(this.db, issue.projectId, liveIssue);
|
|
73
74
|
const delegated = liveIssue.delegateId === installation.actorId;
|
|
74
75
|
if (issue.delegatedToPatchRelay !== delegated) {
|
|
75
76
|
appendDelegationObservedEvent(this.db, {
|
|
@@ -169,6 +170,86 @@ export class ServiceStartupRecovery {
|
|
|
169
170
|
}
|
|
170
171
|
}
|
|
171
172
|
}
|
|
173
|
+
async discoverDelegatedIssuesFromLinear() {
|
|
174
|
+
for (const project of this.config.projects) {
|
|
175
|
+
const installation = this.db.linearInstallations.getLinearInstallationForProject(project.id);
|
|
176
|
+
if (!installation?.actorId) {
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
const linear = await this.linearProvider.forProject(project.id).catch(() => undefined);
|
|
180
|
+
if (!linear?.listIssuesDelegatedTo) {
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
const liveIssues = await linear.listIssuesDelegatedTo({
|
|
184
|
+
delegateId: installation.actorId,
|
|
185
|
+
teamIds: project.linearTeamIds,
|
|
186
|
+
}).catch((error) => {
|
|
187
|
+
this.logger.warn({
|
|
188
|
+
projectId: project.id,
|
|
189
|
+
error: error instanceof Error ? error.message : String(error),
|
|
190
|
+
}, "Failed to discover delegated Linear issues during startup recovery");
|
|
191
|
+
return [];
|
|
192
|
+
});
|
|
193
|
+
for (const liveIssue of liveIssues) {
|
|
194
|
+
if (!this.shouldRecoverDiscoveredIssue(project, liveIssue, installation.actorId)) {
|
|
195
|
+
continue;
|
|
196
|
+
}
|
|
197
|
+
const existing = this.db.issues.getIssue(project.id, liveIssue.id);
|
|
198
|
+
if (existing) {
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
this.upsertDiscoveredDelegatedIssue(project, liveIssue);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
shouldRecoverDiscoveredIssue(project, liveIssue, actorId) {
|
|
206
|
+
if (liveIssue.delegateId !== actorId)
|
|
207
|
+
return false;
|
|
208
|
+
if (liveIssue.stateType === "completed" || liveIssue.stateType === "canceled")
|
|
209
|
+
return false;
|
|
210
|
+
if (project.linearTeamIds.length > 0 && (!liveIssue.teamId || !project.linearTeamIds.includes(liveIssue.teamId))) {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
return true;
|
|
214
|
+
}
|
|
215
|
+
upsertDiscoveredDelegatedIssue(project, liveIssue) {
|
|
216
|
+
upsertLinearIssueProjection(this.db, project.id, liveIssue);
|
|
217
|
+
const existing = this.db.issues.getIssue(project.id, liveIssue.id);
|
|
218
|
+
const updated = this.db.issues.upsertIssue({
|
|
219
|
+
projectId: project.id,
|
|
220
|
+
linearIssueId: liveIssue.id,
|
|
221
|
+
delegatedToPatchRelay: true,
|
|
222
|
+
factoryState: existing?.factoryState ?? "delegated",
|
|
223
|
+
...(liveIssue.identifier ? { issueKey: liveIssue.identifier } : {}),
|
|
224
|
+
...(liveIssue.title ? { title: liveIssue.title } : {}),
|
|
225
|
+
...(liveIssue.description ? { description: liveIssue.description } : {}),
|
|
226
|
+
...(liveIssue.url ? { url: liveIssue.url } : {}),
|
|
227
|
+
...(liveIssue.priority != null ? { priority: liveIssue.priority } : {}),
|
|
228
|
+
...(liveIssue.estimate != null ? { estimate: liveIssue.estimate } : {}),
|
|
229
|
+
...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
|
|
230
|
+
...(liveIssue.stateType ? { currentLinearStateType: liveIssue.stateType } : {}),
|
|
231
|
+
});
|
|
232
|
+
const hasPendingWake = this.db.workflowWakes.peekIssueWake(project.id, liveIssue.id) !== undefined;
|
|
233
|
+
const unresolvedBlockers = this.db.issues.countUnresolvedBlockers(project.id, liveIssue.id);
|
|
234
|
+
if (!hasPendingWake && unresolvedBlockers === 0) {
|
|
235
|
+
this.db.issueSessions.appendIssueSessionEventRespectingActiveLease(project.id, liveIssue.id, {
|
|
236
|
+
projectId: project.id,
|
|
237
|
+
linearIssueId: liveIssue.id,
|
|
238
|
+
eventType: "delegated",
|
|
239
|
+
dedupeKey: `delegated:${liveIssue.id}`,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
if (this.db.workflowWakes.peekIssueWake(project.id, liveIssue.id)) {
|
|
243
|
+
this.enqueueIssue(project.id, liveIssue.id);
|
|
244
|
+
}
|
|
245
|
+
this.logger.info({
|
|
246
|
+
issueKey: updated.issueKey,
|
|
247
|
+
projectId: project.id,
|
|
248
|
+
unresolvedBlockers,
|
|
249
|
+
}, unresolvedBlockers === 0
|
|
250
|
+
? "Discovered delegated Linear issue during startup recovery and queued implementation"
|
|
251
|
+
: "Discovered delegated blocked Linear issue during startup recovery");
|
|
252
|
+
}
|
|
172
253
|
appendReactiveWakeEvent(projectId, linearIssueId, issue, runType) {
|
|
173
254
|
const eventType = reactiveWakeEventType(runType);
|
|
174
255
|
const dedupeKey = runType === "queue_repair" || runType === "ci_repair"
|
package/dist/service.js
CHANGED
|
@@ -67,7 +67,10 @@ export class PatchRelayService {
|
|
|
67
67
|
leaseRelease = (projectId, issueId) => this.orchestrator.leaseService.release(projectId, issueId);
|
|
68
68
|
this.webhookHandler = new WebhookHandler(config, db, this.linearProvider, codex, dispatcher, logger, this.feed, undefined, agentInput, telemetry);
|
|
69
69
|
this.githubWebhookHandler = new GitHubWebhookHandler(config, db, this.linearProvider, dispatcher, logger, codex, this.feed);
|
|
70
|
-
const runtime = new ServiceRuntime(codex, logger, this.orchestrator, {
|
|
70
|
+
const runtime = new ServiceRuntime(codex, logger, this.orchestrator, {
|
|
71
|
+
listIssuesReadyForExecution: () => db.listIssuesReadyForExecution(),
|
|
72
|
+
countActiveIssueRuns: () => db.runs.listActiveRuns().length,
|
|
73
|
+
}, this.webhookHandler, {
|
|
71
74
|
processIssue: async (item) => {
|
|
72
75
|
await this.orchestrator.run(item);
|
|
73
76
|
},
|
|
@@ -77,7 +80,7 @@ export class PatchRelayService {
|
|
|
77
80
|
this.queryService = new IssueQueryService(db, codex, this.orchestrator);
|
|
78
81
|
this.runtime = runtime;
|
|
79
82
|
this.issueActions = new ServiceIssueActions(config, db, agentInput, codex, runtime, this.feed, logger);
|
|
80
|
-
this.startupRecovery = new ServiceStartupRecovery(db, this.linearProvider, this.orchestrator.linearSync, (projectId, issueId) => runtime.enqueueIssue(projectId, issueId), logger);
|
|
83
|
+
this.startupRecovery = new ServiceStartupRecovery(config, db, this.linearProvider, this.orchestrator.linearSync, (projectId, issueId) => runtime.enqueueIssue(projectId, issueId), logger);
|
|
81
84
|
this.trackedIssueListQuery = new TrackedIssueListQuery(db);
|
|
82
85
|
// Optional GitHub App token management for bot identity
|
|
83
86
|
const ghAppCredentials = resolveGitHubAppCredentials();
|
|
@@ -338,6 +341,22 @@ export class PatchRelayService {
|
|
|
338
341
|
async getIssueOverview(issueKey) {
|
|
339
342
|
return await this.queryService.getIssueOverview(issueKey);
|
|
340
343
|
}
|
|
344
|
+
listIssueFeedEvents(issueKey, options) {
|
|
345
|
+
const session = this.db.issueSessions.getIssueSessionByKey(issueKey);
|
|
346
|
+
const issue = this.db.issues.getIssueByKey(issueKey);
|
|
347
|
+
const projectId = session?.projectId ?? issue?.projectId;
|
|
348
|
+
const resolvedIssueKey = session?.issueKey ?? issue?.issueKey ?? issueKey;
|
|
349
|
+
if (!projectId)
|
|
350
|
+
return undefined;
|
|
351
|
+
return {
|
|
352
|
+
events: this.db.operatorFeed.list({
|
|
353
|
+
issueKey: resolvedIssueKey,
|
|
354
|
+
projectId,
|
|
355
|
+
...(options?.afterId !== undefined ? { afterId: options.afterId } : {}),
|
|
356
|
+
limit: Math.min(options?.limit ?? 100, 100),
|
|
357
|
+
}),
|
|
358
|
+
};
|
|
359
|
+
}
|
|
341
360
|
async getActiveRunStatus(issueKey) {
|
|
342
361
|
return await this.orchestrator.getActiveRunStatus(issueKey);
|
|
343
362
|
}
|
package/dist/telemetry.js
CHANGED
|
@@ -78,8 +78,24 @@ export class OperatorFeedTelemetrySink {
|
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
80
|
return undefined;
|
|
81
|
+
case "health.invariant":
|
|
82
|
+
return {
|
|
83
|
+
level: event.status === "observed" ? "warn" : "info",
|
|
84
|
+
kind: "workflow",
|
|
85
|
+
...(event.issueKey ? { issueKey: event.issueKey } : {}),
|
|
86
|
+
...(event.projectId ? { projectId: event.projectId } : {}),
|
|
87
|
+
...(event.runType ? { stage: event.runType } : {}),
|
|
88
|
+
status: `health_${event.status}`,
|
|
89
|
+
summary: event.status === "observed"
|
|
90
|
+
? `Health warning: ${formatInvariant(event.invariant)}`
|
|
91
|
+
: `Health repaired: ${formatInvariant(event.invariant)}`,
|
|
92
|
+
...(event.detail ? { detail: event.detail } : {}),
|
|
93
|
+
};
|
|
81
94
|
default:
|
|
82
95
|
return undefined;
|
|
83
96
|
}
|
|
84
97
|
}
|
|
85
98
|
}
|
|
99
|
+
function formatInvariant(invariant) {
|
|
100
|
+
return invariant.replaceAll("_", " ");
|
|
101
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { TERMINAL_STATES } from "./factory-state.js";
|
|
2
|
+
export class TerminalWakeReconciler {
|
|
3
|
+
db;
|
|
4
|
+
logger;
|
|
5
|
+
constructor(db, logger) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
this.logger = logger;
|
|
8
|
+
}
|
|
9
|
+
reconcile() {
|
|
10
|
+
for (const issue of this.db.issues.listIssues()) {
|
|
11
|
+
if (!TERMINAL_STATES.has(issue.factoryState) || issue.activeRunId !== undefined) {
|
|
12
|
+
continue;
|
|
13
|
+
}
|
|
14
|
+
if (!this.db.issueSessions.hasPendingIssueSessionEvents(issue.projectId, issue.linearIssueId)
|
|
15
|
+
&& issue.pendingRunType === undefined) {
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
this.db.issueSessions.clearPendingIssueSessionEventsRespectingActiveLease(issue.projectId, issue.linearIssueId);
|
|
19
|
+
this.db.issues.upsertIssue({
|
|
20
|
+
projectId: issue.projectId,
|
|
21
|
+
linearIssueId: issue.linearIssueId,
|
|
22
|
+
pendingRunType: null,
|
|
23
|
+
pendingRunContextJson: null,
|
|
24
|
+
});
|
|
25
|
+
this.logger.info({ issueKey: issue.issueKey, factoryState: issue.factoryState }, "Reconciliation: cleared stale terminal wake");
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { replaceIssueDependenciesFromLinearIssue } from "../linear-issue-projection.js";
|
|
1
2
|
import { mergeIssueMetadata } from "./decision-helpers.js";
|
|
2
3
|
/**
|
|
3
4
|
* Brings the local dependency / parent-link state for `issue` up to date.
|
|
@@ -24,17 +25,7 @@ export async function syncIssueDependencies(db, linearProvider, projectId, issue
|
|
|
24
25
|
}
|
|
25
26
|
}
|
|
26
27
|
if (source.relationsKnown) {
|
|
27
|
-
db
|
|
28
|
-
projectId,
|
|
29
|
-
linearIssueId: source.id,
|
|
30
|
-
blockers: source.blockedBy.map((blocker) => ({
|
|
31
|
-
blockerLinearIssueId: blocker.id,
|
|
32
|
-
...(blocker.identifier ? { blockerIssueKey: blocker.identifier } : {}),
|
|
33
|
-
...(blocker.title ? { blockerTitle: blocker.title } : {}),
|
|
34
|
-
...(blocker.stateName ? { blockerCurrentLinearState: blocker.stateName } : {}),
|
|
35
|
-
...(blocker.stateType ? { blockerCurrentLinearStateType: blocker.stateType } : {}),
|
|
36
|
-
})),
|
|
37
|
-
});
|
|
28
|
+
replaceIssueDependenciesFromLinearIssue(db, projectId, source);
|
|
38
29
|
}
|
|
39
30
|
db.issues.replaceIssueParentLink({
|
|
40
31
|
projectId,
|