patchrelay 0.75.0 → 0.75.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/args.js +4 -1
- package/dist/cli/cluster-health/index.js +21 -9
- package/dist/cli/cluster-health/local-issue-health.js +15 -6
- package/dist/cli/commands/maintenance.js +55 -0
- package/dist/cli/help.js +28 -0
- package/dist/cli/index.js +26 -2
- package/dist/cli/output.js +1 -1
- package/dist/codex-app-server.js +22 -2
- package/dist/config.js +6 -0
- package/dist/db/migrations.js +3 -0
- package/dist/db/run-store.js +13 -3
- package/dist/db/webhook-event-store.js +40 -0
- package/dist/db.js +13 -4
- package/dist/event-retention.js +100 -0
- package/dist/idle-reconciliation.js +26 -1
- package/dist/issue-session-projection-invalidator.js +56 -0
- package/dist/linear-client.js +39 -0
- package/dist/linear-issue-projection.js +79 -0
- package/dist/merged-linear-completion-reconciler.js +2 -11
- package/dist/queue-failure-policy.js +11 -0
- package/dist/run-admission-controller.js +23 -0
- package/dist/run-launcher.js +52 -46
- package/dist/run-orchestrator.js +20 -5
- package/dist/service-queue.js +40 -8
- package/dist/service-runtime.js +69 -1
- package/dist/service-startup-recovery.js +94 -13
- package/dist/service.js +43 -2
- package/dist/terminal-wake-reconciler.js +28 -0
- package/dist/webhooks/issue-dependency-sync.js +2 -11
- package/package.json +1 -1
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { createWriteStream } from "node:fs";
|
|
2
|
+
import { mkdir } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { once } from "node:events";
|
|
5
|
+
import { createGzip } from "node:zlib";
|
|
6
|
+
export const DEFAULT_EVENT_RETENTION_DAYS = 7;
|
|
7
|
+
const DEFAULT_BATCH_SIZE = 1_000;
|
|
8
|
+
export async function runWebhookEventRetention(params) {
|
|
9
|
+
const retentionDays = params.options?.retentionDays
|
|
10
|
+
?? params.config.database.eventRetentionDays
|
|
11
|
+
?? DEFAULT_EVENT_RETENTION_DAYS;
|
|
12
|
+
const cutoffIso = computeRetentionCutoffIso(params.options?.now ?? new Date(), retentionDays);
|
|
13
|
+
const batchSize = Math.max(1, Math.floor(params.options?.batchSize ?? DEFAULT_BATCH_SIZE));
|
|
14
|
+
const archiveOldEvents = params.options?.archiveOldEvents ?? params.config.database.archiveOldEvents === true;
|
|
15
|
+
const archivePath = params.options?.archivePath ?? params.config.database.archivePath;
|
|
16
|
+
const dryRun = params.options?.dryRun === true;
|
|
17
|
+
let scanned = 0;
|
|
18
|
+
let archived = 0;
|
|
19
|
+
let deleted = 0;
|
|
20
|
+
let writer;
|
|
21
|
+
if (dryRun) {
|
|
22
|
+
const remaining = params.db.webhookEvents.countArchiveableEventsBefore(cutoffIso);
|
|
23
|
+
return {
|
|
24
|
+
cutoffIso,
|
|
25
|
+
scanned: remaining,
|
|
26
|
+
archived: 0,
|
|
27
|
+
deleted: 0,
|
|
28
|
+
remaining,
|
|
29
|
+
dryRun,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
if (archiveOldEvents) {
|
|
34
|
+
writer = await JsonlGzipArchiveWriter.create(resolveArchiveFilePath(archivePath, params.options?.now ?? new Date()));
|
|
35
|
+
}
|
|
36
|
+
while (true) {
|
|
37
|
+
const records = params.db.webhookEvents.listArchiveableEventsBefore(cutoffIso, batchSize);
|
|
38
|
+
if (records.length === 0)
|
|
39
|
+
break;
|
|
40
|
+
scanned += records.length;
|
|
41
|
+
if (writer) {
|
|
42
|
+
await writer.writeRecords(records);
|
|
43
|
+
archived += records.length;
|
|
44
|
+
}
|
|
45
|
+
deleted += params.db.webhookEvents.deleteWebhookEventsByIds(records.map((record) => record.id));
|
|
46
|
+
if (records.length < batchSize)
|
|
47
|
+
break;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
finally {
|
|
51
|
+
await writer?.close();
|
|
52
|
+
}
|
|
53
|
+
const remaining = params.db.webhookEvents.countArchiveableEventsBefore(cutoffIso);
|
|
54
|
+
return {
|
|
55
|
+
cutoffIso,
|
|
56
|
+
scanned,
|
|
57
|
+
archived,
|
|
58
|
+
deleted,
|
|
59
|
+
remaining,
|
|
60
|
+
...(writer?.filePath ? { archiveFile: writer.filePath } : {}),
|
|
61
|
+
dryRun,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
export function computeRetentionCutoffIso(now, retentionDays) {
|
|
65
|
+
return new Date(now.getTime() - retentionDays * 24 * 60 * 60 * 1000).toISOString();
|
|
66
|
+
}
|
|
67
|
+
function resolveArchiveFilePath(archivePath, now) {
|
|
68
|
+
const root = archivePath ?? path.join(process.cwd(), "archive");
|
|
69
|
+
const stamp = now.toISOString().replaceAll(":", "-").replaceAll(".", "-");
|
|
70
|
+
return path.join(root, "webhook-events", `${stamp}.jsonl.gz`);
|
|
71
|
+
}
|
|
72
|
+
class JsonlGzipArchiveWriter {
|
|
73
|
+
filePath;
|
|
74
|
+
gzip;
|
|
75
|
+
output;
|
|
76
|
+
constructor(filePath, gzip, output) {
|
|
77
|
+
this.filePath = filePath;
|
|
78
|
+
this.gzip = gzip;
|
|
79
|
+
this.output = output;
|
|
80
|
+
}
|
|
81
|
+
static async create(filePath) {
|
|
82
|
+
await mkdir(path.dirname(filePath), { recursive: true });
|
|
83
|
+
const gzip = createGzip();
|
|
84
|
+
const output = createWriteStream(filePath, { flags: "wx" });
|
|
85
|
+
gzip.pipe(output);
|
|
86
|
+
return new JsonlGzipArchiveWriter(filePath, gzip, output);
|
|
87
|
+
}
|
|
88
|
+
async writeRecords(records) {
|
|
89
|
+
for (const record of records) {
|
|
90
|
+
const line = `${JSON.stringify(record)}\n`;
|
|
91
|
+
if (!this.gzip.write(line)) {
|
|
92
|
+
await once(this.gzip, "drain");
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
async close() {
|
|
97
|
+
this.gzip.end();
|
|
98
|
+
await once(this.output, "close");
|
|
99
|
+
}
|
|
100
|
+
}
|
|
@@ -12,6 +12,10 @@ import { fetchPullRequestSnapshot } from "./reconcile-pr-fetch.js";
|
|
|
12
12
|
import { buildPrStateUpdates } from "./reconcile-pr-state-updates.js";
|
|
13
13
|
import { buildRepairWakeDedupeKey, buildRequestedChangesWakeIdentity, reactiveWakeEventType } from "./reactive-wake-keys.js";
|
|
14
14
|
import { execCommand } from "./utils.js";
|
|
15
|
+
import { LinearIssueProjectionService } from "./linear-issue-projection.js";
|
|
16
|
+
import { TerminalWakeReconciler } from "./terminal-wake-reconciler.js";
|
|
17
|
+
const BLOCKED_DEPENDENCY_REFRESH_SUCCESS_BACKOFF_MS = 60_000;
|
|
18
|
+
const BLOCKED_DEPENDENCY_REFRESH_FAILURE_BACKOFF_MS = 5 * 60_000;
|
|
15
19
|
export class IdleIssueReconciler {
|
|
16
20
|
db;
|
|
17
21
|
config;
|
|
@@ -20,9 +24,13 @@ export class IdleIssueReconciler {
|
|
|
20
24
|
feed;
|
|
21
25
|
deployEvaluator;
|
|
22
26
|
syncIssue;
|
|
27
|
+
linearProvider;
|
|
28
|
+
blockedDependencyRefreshAfter = new Map();
|
|
29
|
+
terminalWakeReconciler;
|
|
30
|
+
linearIssueProjection;
|
|
23
31
|
constructor(db, config, wakeDispatcher, logger, feed,
|
|
24
32
|
// Injectable for tests; production uses the real `gh`-backed watcher.
|
|
25
|
-
deployEvaluator = evaluateDeploy, syncIssue) {
|
|
33
|
+
deployEvaluator = evaluateDeploy, syncIssue, linearProvider) {
|
|
26
34
|
this.db = db;
|
|
27
35
|
this.config = config;
|
|
28
36
|
this.wakeDispatcher = wakeDispatcher;
|
|
@@ -30,6 +38,11 @@ export class IdleIssueReconciler {
|
|
|
30
38
|
this.feed = feed;
|
|
31
39
|
this.deployEvaluator = deployEvaluator;
|
|
32
40
|
this.syncIssue = syncIssue;
|
|
41
|
+
this.linearProvider = linearProvider;
|
|
42
|
+
this.terminalWakeReconciler = new TerminalWakeReconciler(db, logger);
|
|
43
|
+
this.linearIssueProjection = linearProvider
|
|
44
|
+
? new LinearIssueProjectionService(db, linearProvider, logger)
|
|
45
|
+
: undefined;
|
|
33
46
|
}
|
|
34
47
|
async reconcile() {
|
|
35
48
|
// Wrap the entire reconcile pass in a dispatcher tick. Every
|
|
@@ -82,9 +95,21 @@ export class IdleIssueReconciler {
|
|
|
82
95
|
continue;
|
|
83
96
|
await this.reconcileFromGitHub(issue);
|
|
84
97
|
}
|
|
98
|
+
this.terminalWakeReconciler.reconcile();
|
|
85
99
|
for (const issue of this.db.issues.listBlockedDelegatedIssues()) {
|
|
86
100
|
if (!issue.delegatedToPatchRelay)
|
|
87
101
|
continue;
|
|
102
|
+
const dependencyKey = `${issue.projectId}::${issue.linearIssueId}`;
|
|
103
|
+
const refreshAfter = this.blockedDependencyRefreshAfter.get(dependencyKey);
|
|
104
|
+
if (this.linearIssueProjection) {
|
|
105
|
+
if (refreshAfter === undefined || refreshAfter <= Date.now()) {
|
|
106
|
+
const refresh = await this.linearIssueProjection.refreshIssue(issue.projectId, issue.linearIssueId);
|
|
107
|
+
this.blockedDependencyRefreshAfter.set(dependencyKey, Date.now() + (refresh.refreshed ? BLOCKED_DEPENDENCY_REFRESH_SUCCESS_BACKOFF_MS : BLOCKED_DEPENDENCY_REFRESH_FAILURE_BACKOFF_MS));
|
|
108
|
+
if (!refresh.refreshed) {
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
88
113
|
const unresolved = this.db.issues.countUnresolvedBlockers(issue.projectId, issue.linearIssueId);
|
|
89
114
|
if (unresolved === 0) {
|
|
90
115
|
this.wakeDispatcher.recordEventAndDispatch(issue.projectId, issue.linearIssueId, {
|
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
import { emitTelemetry, noopTelemetry } from "./telemetry.js";
|
|
2
2
|
export class ImmediateIssueSessionProjectionInvalidator {
|
|
3
3
|
deps;
|
|
4
|
+
batchDepth = 0;
|
|
5
|
+
pendingProjections = new Map();
|
|
4
6
|
constructor(deps) {
|
|
5
7
|
this.deps = deps;
|
|
6
8
|
}
|
|
9
|
+
batch(fn) {
|
|
10
|
+
this.batchDepth += 1;
|
|
11
|
+
try {
|
|
12
|
+
return fn();
|
|
13
|
+
}
|
|
14
|
+
finally {
|
|
15
|
+
this.batchDepth -= 1;
|
|
16
|
+
if (this.batchDepth === 0) {
|
|
17
|
+
this.flushPendingProjections();
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
7
21
|
issueChanged(issue, options) {
|
|
8
22
|
const dependents = this.deps.listDependents(issue.projectId, issue.linearIssueId);
|
|
9
23
|
this.emitInvalidated("issue_changed", issue.projectId, issue.linearIssueId, issue.issueKey, 1 + dependents.length);
|
|
@@ -32,12 +46,26 @@ export class ImmediateIssueSessionProjectionInvalidator {
|
|
|
32
46
|
this.projectIssueById(projectId, linearIssueId, "issue_session_events_changed");
|
|
33
47
|
}
|
|
34
48
|
projectIssueById(projectId, linearIssueId, reason) {
|
|
49
|
+
if (this.batchDepth > 0) {
|
|
50
|
+
this.queueProjection({ projectId, linearIssueId, reason });
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
35
53
|
const issue = this.deps.getIssue(projectId, linearIssueId);
|
|
36
54
|
if (issue) {
|
|
37
55
|
this.projectIssue(issue, reason);
|
|
38
56
|
}
|
|
39
57
|
}
|
|
40
58
|
projectIssue(issue, reason, options) {
|
|
59
|
+
if (this.batchDepth > 0) {
|
|
60
|
+
this.queueProjection({
|
|
61
|
+
projectId: issue.projectId,
|
|
62
|
+
linearIssueId: issue.linearIssueId,
|
|
63
|
+
issue,
|
|
64
|
+
reason,
|
|
65
|
+
...(options ? { options } : {}),
|
|
66
|
+
});
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
41
69
|
const beforeWaitingReason = this.deps.getIssueSessionWaitingReason?.(issue.projectId, issue.linearIssueId);
|
|
42
70
|
this.deps.projectIssue(issue, options);
|
|
43
71
|
this.emitReprojected(reason, issue);
|
|
@@ -87,4 +115,32 @@ export class ImmediateIssueSessionProjectionInvalidator {
|
|
|
87
115
|
...(issue.issueKey ? { issueKey: issue.issueKey } : {}),
|
|
88
116
|
});
|
|
89
117
|
}
|
|
118
|
+
queueProjection(projection) {
|
|
119
|
+
const key = `${projection.projectId}::${projection.linearIssueId}`;
|
|
120
|
+
const current = this.pendingProjections.get(key);
|
|
121
|
+
this.pendingProjections.set(key, {
|
|
122
|
+
projectId: projection.projectId,
|
|
123
|
+
linearIssueId: projection.linearIssueId,
|
|
124
|
+
issue: projection.issue ?? current?.issue,
|
|
125
|
+
reason: projection.reason,
|
|
126
|
+
options: mergeProjectionOptions(current?.options, projection.options),
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
flushPendingProjections() {
|
|
130
|
+
const pending = Array.from(this.pendingProjections.values());
|
|
131
|
+
this.pendingProjections.clear();
|
|
132
|
+
for (const projection of pending) {
|
|
133
|
+
const issue = projection.issue ?? this.deps.getIssue(projection.projectId, projection.linearIssueId);
|
|
134
|
+
if (issue) {
|
|
135
|
+
this.projectIssue(issue, projection.reason, projection.options);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
function mergeProjectionOptions(current, next) {
|
|
141
|
+
if (!current)
|
|
142
|
+
return next;
|
|
143
|
+
if (!next)
|
|
144
|
+
return current;
|
|
145
|
+
return { ...current, ...next };
|
|
90
146
|
}
|
package/dist/linear-client.js
CHANGED
|
@@ -113,6 +113,45 @@ export class LinearGraphqlClient {
|
|
|
113
113
|
}
|
|
114
114
|
return this.mapIssue(response.issue);
|
|
115
115
|
}
|
|
116
|
+
async listIssuesDelegatedTo(params) {
|
|
117
|
+
const teamIds = params.teamIds.filter((teamId) => teamId.trim().length > 0);
|
|
118
|
+
if (teamIds.length === 0) {
|
|
119
|
+
return [];
|
|
120
|
+
}
|
|
121
|
+
const first = Math.max(1, Math.min(params.first ?? 100, 100));
|
|
122
|
+
const issues = [];
|
|
123
|
+
let after;
|
|
124
|
+
do {
|
|
125
|
+
const response = await this.request(`
|
|
126
|
+
query PatchRelayDelegatedIssues($delegateId: ID!, $teamIds: [ID!], $first: Int!, $after: String) {
|
|
127
|
+
issues(
|
|
128
|
+
first: $first
|
|
129
|
+
after: $after
|
|
130
|
+
filter: {
|
|
131
|
+
delegate: { id: { eq: $delegateId } }
|
|
132
|
+
team: { id: { in: $teamIds } }
|
|
133
|
+
}
|
|
134
|
+
) {
|
|
135
|
+
nodes {
|
|
136
|
+
${LINEAR_ISSUE_SELECTION}
|
|
137
|
+
}
|
|
138
|
+
pageInfo {
|
|
139
|
+
hasNextPage
|
|
140
|
+
endCursor
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
`, {
|
|
145
|
+
delegateId: params.delegateId,
|
|
146
|
+
teamIds,
|
|
147
|
+
first,
|
|
148
|
+
after: after ?? null,
|
|
149
|
+
});
|
|
150
|
+
issues.push(...(response.issues.nodes ?? []).map((issue) => this.mapIssue(issue)));
|
|
151
|
+
after = response.issues.pageInfo.hasNextPage ? response.issues.pageInfo.endCursor ?? undefined : undefined;
|
|
152
|
+
} while (after);
|
|
153
|
+
return issues;
|
|
154
|
+
}
|
|
116
155
|
async createIssue(params) {
|
|
117
156
|
const response = await this.request(`
|
|
118
157
|
mutation PatchRelayCreateIssue($input: IssueCreateInput!) {
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
export class LinearIssueProjectionService {
|
|
2
|
+
db;
|
|
3
|
+
linearProvider;
|
|
4
|
+
logger;
|
|
5
|
+
constructor(db, linearProvider, logger) {
|
|
6
|
+
this.db = db;
|
|
7
|
+
this.linearProvider = linearProvider;
|
|
8
|
+
this.logger = logger;
|
|
9
|
+
}
|
|
10
|
+
async refreshIssue(projectId, linearIssueId) {
|
|
11
|
+
return refreshIssueFromLinear({
|
|
12
|
+
db: this.db,
|
|
13
|
+
linearProvider: this.linearProvider,
|
|
14
|
+
projectId,
|
|
15
|
+
linearIssueId,
|
|
16
|
+
logger: this.logger,
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
export async function refreshIssueFromLinear(params) {
|
|
21
|
+
const linear = await params.linearProvider.forProject(params.projectId).catch((error) => {
|
|
22
|
+
params.logger?.warn({
|
|
23
|
+
projectId: params.projectId,
|
|
24
|
+
linearIssueId: params.linearIssueId,
|
|
25
|
+
error: error instanceof Error ? error.message : String(error),
|
|
26
|
+
}, "Failed to resolve Linear client while refreshing issue projection");
|
|
27
|
+
return undefined;
|
|
28
|
+
});
|
|
29
|
+
if (!linear) {
|
|
30
|
+
return { refreshed: false, error: "linear_client_unavailable" };
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
const liveIssue = await linear.getIssue(params.linearIssueId);
|
|
34
|
+
upsertLinearIssueProjection(params.db, params.projectId, liveIssue);
|
|
35
|
+
return { refreshed: true, liveIssue };
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
39
|
+
params.logger?.warn({
|
|
40
|
+
projectId: params.projectId,
|
|
41
|
+
linearIssueId: params.linearIssueId,
|
|
42
|
+
error: message,
|
|
43
|
+
}, "Failed to refresh issue projection from Linear");
|
|
44
|
+
return { refreshed: false, error: message };
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
export function upsertLinearIssueProjection(db, projectId, liveIssue) {
|
|
48
|
+
replaceIssueDependenciesFromLinearIssue(db, projectId, liveIssue);
|
|
49
|
+
db.issues.replaceIssueParentLink({
|
|
50
|
+
projectId,
|
|
51
|
+
childLinearIssueId: liveIssue.id,
|
|
52
|
+
parentLinearIssueId: liveIssue.parentId ?? null,
|
|
53
|
+
});
|
|
54
|
+
db.issues.upsertIssue({
|
|
55
|
+
projectId,
|
|
56
|
+
linearIssueId: liveIssue.id,
|
|
57
|
+
...(liveIssue.identifier ? { issueKey: liveIssue.identifier } : {}),
|
|
58
|
+
...(liveIssue.title ? { title: liveIssue.title } : {}),
|
|
59
|
+
...(liveIssue.description ? { description: liveIssue.description } : {}),
|
|
60
|
+
...(liveIssue.url ? { url: liveIssue.url } : {}),
|
|
61
|
+
...(liveIssue.priority != null ? { priority: liveIssue.priority } : {}),
|
|
62
|
+
...(liveIssue.estimate != null ? { estimate: liveIssue.estimate } : {}),
|
|
63
|
+
...(liveIssue.stateName ? { currentLinearState: liveIssue.stateName } : {}),
|
|
64
|
+
...(liveIssue.stateType ? { currentLinearStateType: liveIssue.stateType } : {}),
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
export function replaceIssueDependenciesFromLinearIssue(db, projectId, liveIssue) {
|
|
68
|
+
db.issues.replaceIssueDependencies({
|
|
69
|
+
projectId,
|
|
70
|
+
linearIssueId: liveIssue.id,
|
|
71
|
+
blockers: liveIssue.blockedBy.map((blocker) => ({
|
|
72
|
+
blockerLinearIssueId: blocker.id,
|
|
73
|
+
...(blocker.identifier ? { blockerIssueKey: blocker.identifier } : {}),
|
|
74
|
+
...(blocker.title ? { blockerTitle: blocker.title } : {}),
|
|
75
|
+
...(blocker.stateName ? { blockerCurrentLinearState: blocker.stateName } : {}),
|
|
76
|
+
...(blocker.stateType ? { blockerCurrentLinearStateType: blocker.stateType } : {}),
|
|
77
|
+
})),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
@@ -2,6 +2,7 @@ import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
|
2
2
|
import { resolvePreferredCompletedLinearState } from "./linear-workflow.js";
|
|
3
3
|
import { isCompletedLinearState } from "./pr-state.js";
|
|
4
4
|
import { hasTrustedNoPrCompletion } from "./trusted-no-pr-completion.js";
|
|
5
|
+
import { replaceIssueDependenciesFromLinearIssue } from "./linear-issue-projection.js";
|
|
5
6
|
const COMPLETION_RECONCILE_WINDOW_MS = 60 * 60 * 1000;
|
|
6
7
|
const COMPLETION_RECONCILE_SUCCESS_BACKOFF_MS = 60 * 60 * 1000;
|
|
7
8
|
const COMPLETION_RECONCILE_FAILURE_BACKOFF_MS = 5 * 60 * 1000;
|
|
@@ -45,17 +46,7 @@ export class MergedLinearCompletionReconciler {
|
|
|
45
46
|
attemptedIssues += 1;
|
|
46
47
|
try {
|
|
47
48
|
const liveIssue = await linear.getIssue(issue.linearIssueId);
|
|
48
|
-
this.db.
|
|
49
|
-
projectId: issue.projectId,
|
|
50
|
-
linearIssueId: issue.linearIssueId,
|
|
51
|
-
blockers: liveIssue.blockedBy.map((blocker) => ({
|
|
52
|
-
blockerLinearIssueId: blocker.id,
|
|
53
|
-
...(blocker.identifier ? { blockerIssueKey: blocker.identifier } : {}),
|
|
54
|
-
...(blocker.title ? { blockerTitle: blocker.title } : {}),
|
|
55
|
-
...(blocker.stateName ? { blockerCurrentLinearState: blocker.stateName } : {}),
|
|
56
|
-
...(blocker.stateType ? { blockerCurrentLinearStateType: blocker.stateType } : {}),
|
|
57
|
-
})),
|
|
58
|
-
});
|
|
49
|
+
replaceIssueDependenciesFromLinearIssue(this.db, issue.projectId, liveIssue);
|
|
59
50
|
const latestRun = this.db.runs.getLatestRunForIssue(issue.projectId, issue.linearIssueId);
|
|
60
51
|
const trustedNoPrDone = hasTrustedNoPrCompletion(issue, latestRun);
|
|
61
52
|
if (issue.prState === "merged" || trustedNoPrDone) {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
const SQLITE_LOCK_RETRY_DELAYS_MS = [250, 1_000, 2_500, 5_000, 10_000];
|
|
2
|
+
export function retrySqliteLockedQueueFailure(error, attempt) {
|
|
3
|
+
if (!isSqliteDatabaseLockedError(error)) {
|
|
4
|
+
return undefined;
|
|
5
|
+
}
|
|
6
|
+
const delayMs = SQLITE_LOCK_RETRY_DELAYS_MS[attempt - 1];
|
|
7
|
+
return delayMs === undefined ? undefined : { delayMs };
|
|
8
|
+
}
|
|
9
|
+
export function isSqliteDatabaseLockedError(error) {
|
|
10
|
+
return /\bdatabase is locked\b/i.test(error.message);
|
|
11
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
export class RunAdmissionController {
|
|
2
|
+
db;
|
|
3
|
+
linearIssueProjection;
|
|
4
|
+
constructor(db, linearIssueProjection) {
|
|
5
|
+
this.db = db;
|
|
6
|
+
this.linearIssueProjection = linearIssueProjection;
|
|
7
|
+
}
|
|
8
|
+
async check(params) {
|
|
9
|
+
if (params.runType !== "implementation") {
|
|
10
|
+
return { allowed: true };
|
|
11
|
+
}
|
|
12
|
+
const knownDependencyRows = this.db.issues.listIssueDependencies(params.projectId, params.linearIssueId).length;
|
|
13
|
+
const refresh = await this.linearIssueProjection.refreshIssue(params.projectId, params.linearIssueId);
|
|
14
|
+
if (!refresh.refreshed && knownDependencyRows > 0) {
|
|
15
|
+
return { allowed: false, reason: "dependency_refresh_failed", knownDependencyRows };
|
|
16
|
+
}
|
|
17
|
+
const blockerCount = this.db.issues.countUnresolvedBlockers(params.projectId, params.linearIssueId);
|
|
18
|
+
if (blockerCount > 0) {
|
|
19
|
+
return { allowed: false, reason: "blocked", blockerCount };
|
|
20
|
+
}
|
|
21
|
+
return { allowed: true };
|
|
22
|
+
}
|
|
23
|
+
}
|
package/dist/run-launcher.js
CHANGED
|
@@ -104,53 +104,55 @@ export class RunLauncher {
|
|
|
104
104
|
}
|
|
105
105
|
claimRun(params) {
|
|
106
106
|
return this.db.issueSessions.withIssueSessionLease(params.item.projectId, params.item.issueId, params.leaseId, () => {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
: params.runType === "
|
|
140
|
-
: params.runType === "
|
|
141
|
-
: params.runType === "
|
|
142
|
-
: "
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
107
|
+
return this.db.batchIssueSessionProjections(() => {
|
|
108
|
+
const fresh = this.db.issues.getIssue(params.item.projectId, params.item.issueId);
|
|
109
|
+
if (!fresh || fresh.activeRunId !== undefined)
|
|
110
|
+
return undefined;
|
|
111
|
+
const wakeIssue = params.materializeLegacyPendingWake(fresh, {
|
|
112
|
+
projectId: params.item.projectId,
|
|
113
|
+
linearIssueId: params.item.issueId,
|
|
114
|
+
leaseId: params.leaseId,
|
|
115
|
+
});
|
|
116
|
+
const freshWake = params.resolveRunWake(wakeIssue);
|
|
117
|
+
if (!freshWake || freshWake.runType !== params.runType)
|
|
118
|
+
return undefined;
|
|
119
|
+
const created = this.db.runs.createRun({
|
|
120
|
+
issueId: fresh.id,
|
|
121
|
+
projectId: params.item.projectId,
|
|
122
|
+
linearIssueId: params.item.issueId,
|
|
123
|
+
runType: params.runType,
|
|
124
|
+
...(params.sourceHeadSha ? { sourceHeadSha: params.sourceHeadSha } : {}),
|
|
125
|
+
promptText: params.prompt,
|
|
126
|
+
});
|
|
127
|
+
const failureHeadSha = typeof params.effectiveContext?.failureHeadSha === "string"
|
|
128
|
+
? params.effectiveContext.failureHeadSha
|
|
129
|
+
: typeof params.effectiveContext?.headSha === "string" ? params.effectiveContext.headSha : undefined;
|
|
130
|
+
const failureSignature = typeof params.effectiveContext?.failureSignature === "string" ? params.effectiveContext.failureSignature : undefined;
|
|
131
|
+
this.db.issues.upsertIssue({
|
|
132
|
+
projectId: params.item.projectId,
|
|
133
|
+
linearIssueId: params.item.issueId,
|
|
134
|
+
pendingRunType: null,
|
|
135
|
+
pendingRunContextJson: null,
|
|
136
|
+
activeRunId: created.id,
|
|
137
|
+
branchName: params.branchName,
|
|
138
|
+
worktreePath: params.worktreePath,
|
|
139
|
+
factoryState: params.runType === "implementation" ? "implementing"
|
|
140
|
+
: params.runType === "ci_repair" ? "repairing_ci"
|
|
141
|
+
: params.runType === "review_fix" || params.runType === "branch_upkeep" ? "changes_requested"
|
|
142
|
+
: params.runType === "queue_repair" ? "repairing_queue"
|
|
143
|
+
: "implementing",
|
|
144
|
+
...((params.runType === "ci_repair" || params.runType === "queue_repair") && failureSignature
|
|
145
|
+
? {
|
|
146
|
+
lastAttemptedFailureSignature: failureSignature,
|
|
147
|
+
lastAttemptedFailureHeadSha: failureHeadSha ?? null,
|
|
148
|
+
lastAttemptedFailureAt: new Date().toISOString(),
|
|
149
|
+
}
|
|
150
|
+
: {}),
|
|
151
|
+
});
|
|
152
|
+
this.db.issueSessions.consumeIssueSessionEvents(params.item.projectId, params.item.issueId, freshWake.eventIds, created.id);
|
|
153
|
+
this.db.issueSessions.setIssueSessionLastWakeReason(params.item.projectId, params.item.issueId, freshWake.wakeReason ?? null);
|
|
154
|
+
return created;
|
|
150
155
|
});
|
|
151
|
-
this.db.issueSessions.consumeIssueSessionEvents(params.item.projectId, params.item.issueId, freshWake.eventIds, created.id);
|
|
152
|
-
this.db.issueSessions.setIssueSessionLastWakeReason(params.item.projectId, params.item.issueId, freshWake.wakeReason ?? null);
|
|
153
|
-
return created;
|
|
154
156
|
});
|
|
155
157
|
}
|
|
156
158
|
async launchTurn(params) {
|
|
@@ -186,6 +188,7 @@ export class RunLauncher {
|
|
|
186
188
|
if (prepareResult.ran && prepareResult.exitCode !== 0) {
|
|
187
189
|
throw new Error(`prepare-worktree hook failed (exit ${prepareResult.exitCode}): ${prepareResult.stderr?.slice(0, 500) ?? ""}`);
|
|
188
190
|
}
|
|
191
|
+
this.db.runs.updateLaunchPhase(params.run.id, "worktree_prepared");
|
|
189
192
|
params.assertLaunchLease(params.run, "before starting the Codex turn");
|
|
190
193
|
const compactThread = shouldCompactThread(params.issue, params.issueSession?.threadGeneration, params.effectiveContext);
|
|
191
194
|
if (compactThread && params.issue.threadId) {
|
|
@@ -200,9 +203,11 @@ export class RunLauncher {
|
|
|
200
203
|
createdThreadForRun = true;
|
|
201
204
|
this.db.issueSessions.upsertIssueWithLease({ projectId: params.project.id, linearIssueId: params.issue.linearIssueId, leaseId: params.leaseId }, { projectId: params.project.id, linearIssueId: params.issue.linearIssueId, threadId });
|
|
202
205
|
}
|
|
206
|
+
this.db.runs.updateLaunchPhase(params.run.id, "thread_started");
|
|
203
207
|
try {
|
|
204
208
|
const turn = await this.codex.startTurn({ threadId, cwd: params.worktreePath, input: params.prompt });
|
|
205
209
|
turnId = turn.turnId;
|
|
210
|
+
this.db.runs.updateLaunchPhase(params.run.id, "turn_started");
|
|
206
211
|
}
|
|
207
212
|
catch (turnError) {
|
|
208
213
|
const msg = turnError instanceof Error ? turnError.message : String(turnError);
|
|
@@ -214,6 +219,7 @@ export class RunLauncher {
|
|
|
214
219
|
this.db.issueSessions.upsertIssueWithLease({ projectId: params.project.id, linearIssueId: params.issue.linearIssueId, leaseId: params.leaseId }, { projectId: params.project.id, linearIssueId: params.issue.linearIssueId, threadId });
|
|
215
220
|
const turn = await this.codex.startTurn({ threadId, cwd: params.worktreePath, input: params.prompt });
|
|
216
221
|
turnId = turn.turnId;
|
|
222
|
+
this.db.runs.updateLaunchPhase(params.run.id, "turn_started");
|
|
217
223
|
}
|
|
218
224
|
else {
|
|
219
225
|
throw turnError;
|
package/dist/run-orchestrator.js
CHANGED
|
@@ -24,6 +24,8 @@ import { buildIssueTriageHash, IssueTriageService } from "./issue-triage.js";
|
|
|
24
24
|
import { loadConfig } from "./config.js";
|
|
25
25
|
import { CodexThreadMaterializingError, isThreadMaterializingError } from "./codex-thread-errors.js";
|
|
26
26
|
import { emitTelemetry, noopTelemetry } from "./telemetry.js";
|
|
27
|
+
import { LinearIssueProjectionService } from "./linear-issue-projection.js";
|
|
28
|
+
import { RunAdmissionController } from "./run-admission-controller.js";
|
|
27
29
|
function lowerCaseFirst(value) {
|
|
28
30
|
return value ? `${value.slice(0, 1).toLowerCase()}${value.slice(1)}` : value;
|
|
29
31
|
}
|
|
@@ -64,6 +66,8 @@ export class RunOrchestrator {
|
|
|
64
66
|
runNotificationHandler;
|
|
65
67
|
runReconciler;
|
|
66
68
|
mergedLinearCompletionReconciler;
|
|
69
|
+
linearIssueProjection;
|
|
70
|
+
runAdmission;
|
|
67
71
|
codexRuntimeConfig;
|
|
68
72
|
threadPorts = {
|
|
69
73
|
readThreadWithRetry: (threadId, maxRetries) => this.readThreadWithRetry(threadId, maxRetries),
|
|
@@ -133,7 +137,9 @@ export class RunOrchestrator {
|
|
|
133
137
|
this.interruptedRunRecovery = new InterruptedRunRecovery(db, logger, this.linearSync, this.leasePorts.withHeldLease, this.leasePorts.releaseLease, this.recoveryPorts.failRunAndClear, this.recoveryPorts.restoreIdleWorktree, this.runCompletionPolicy, (projectId, issueId) => this.enqueueIssue(projectId, issueId), feed);
|
|
134
138
|
this.runReconciler = new RunReconciler(db, logger, linearProvider, this.linearSync, this.interruptedRunRecovery, this.runFinalizer, this.leasePorts.withHeldLease, this.leasePorts.releaseLease, this.threadPorts.readThreadWithRetry, this.recoveryPorts.recoverOrEscalate, (projectId) => this.config.projects.find((project) => project.id === projectId)?.github?.repoFullName, feed);
|
|
135
139
|
this.runWakePlanner = new RunWakePlanner(db);
|
|
136
|
-
this.
|
|
140
|
+
this.linearIssueProjection = new LinearIssueProjectionService(db, linearProvider, logger);
|
|
141
|
+
this.runAdmission = new RunAdmissionController(db, this.linearIssueProjection);
|
|
142
|
+
this.idleReconciler = new IdleIssueReconciler(db, config, this.wakeDispatcher, logger, feed, undefined, (issue) => this.linearSync.syncSession(issue), linearProvider);
|
|
137
143
|
this.mergedLinearCompletionReconciler = new MergedLinearCompletionReconciler(db, linearProvider, logger);
|
|
138
144
|
this.queueHealthMonitor = new QueueHealthMonitor(db, config, {
|
|
139
145
|
advanceIdleIssue: (issue, newState, options) => this.idleReconciler.advanceIdleIssue(issue, newState, options),
|
|
@@ -312,12 +318,21 @@ export class RunOrchestrator {
|
|
|
312
318
|
return;
|
|
313
319
|
}
|
|
314
320
|
const { runType, context, resumeThread } = wake;
|
|
315
|
-
|
|
316
|
-
|
|
321
|
+
const admission = await this.runAdmission.check({
|
|
322
|
+
projectId: item.projectId,
|
|
323
|
+
linearIssueId: item.issueId,
|
|
324
|
+
runType,
|
|
325
|
+
});
|
|
326
|
+
if (!admission.allowed) {
|
|
317
327
|
this.db.issueSessions.clearPendingIssueSessionEventsRespectingActiveLease(item.projectId, item.issueId);
|
|
318
328
|
this.releaseIssueSessionLease(item.projectId, item.issueId);
|
|
319
|
-
this.emitRunSkipped(item,
|
|
320
|
-
|
|
329
|
+
this.emitRunSkipped(item, admission.reason, issue, { runType, ...admission });
|
|
330
|
+
if (admission.reason === "dependency_refresh_failed") {
|
|
331
|
+
this.logger.info({ issueKey: issue.issueKey, projectId: item.projectId, knownDependencyRows: admission.knownDependencyRows }, "Skipped implementation launch because dependency refresh failed for an issue with known blockers");
|
|
332
|
+
}
|
|
333
|
+
else {
|
|
334
|
+
this.logger.info({ issueKey: issue.issueKey, blockerCount: admission.blockerCount }, "Skipped implementation launch because the issue is blocked");
|
|
335
|
+
}
|
|
321
336
|
return;
|
|
322
337
|
}
|
|
323
338
|
const remainingZombieDelayMs = shouldDelayZombieRecoveryLaunch(issue, issueSession, runType);
|