patchrelay 0.68.3 → 0.68.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-info.json +3 -3
- package/dist/cli/cluster-health/active-overlap.js +64 -0
- package/dist/cli/cluster-health/ci-classification.js +186 -0
- package/dist/cli/cluster-health/github-issue-health.js +212 -0
- package/dist/cli/cluster-health/github-probe.js +67 -0
- package/dist/cli/cluster-health/index.js +168 -0
- package/dist/cli/cluster-health/local-issue-health.js +91 -0
- package/dist/cli/cluster-health/review-quill-probe.js +126 -0
- package/dist/cli/cluster-health/service-probe.js +65 -0
- package/dist/cli/cluster-health/shared.js +13 -0
- package/dist/cli/cluster-health/types.js +1 -0
- package/dist/cli/commands/cluster.js +1 -1
- package/dist/codex-app-server.js +46 -9
- package/dist/idle-reconciliation.js +16 -36
- package/dist/reconcile-pr-fetch.js +23 -0
- package/dist/reconcile-pr-state-updates.js +40 -0
- package/dist/run-orchestrator.js +1 -1
- package/dist/run-reconciler.js +57 -1
- package/package.json +1 -1
- package/dist/cli/cluster-health.js +0 -976
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { hasOpenPr } from "../../pr-state.js";
|
|
2
|
+
import { collectActiveOverlapFindings } from "./active-overlap.js";
|
|
3
|
+
import { evaluateLocalIssueHealth, isResolvedDependency, needsReviewAutomation, } from "./local-issue-health.js";
|
|
4
|
+
import { evaluateGitHubIssueHealth } from "./github-issue-health.js";
|
|
5
|
+
import { collectReviewQuillAttemptOwners, } from "./review-quill-probe.js";
|
|
6
|
+
import { probeOptionalService, probePatchRelayService, } from "./service-probe.js";
|
|
7
|
+
import {} from "./shared.js";
|
|
8
|
+
export async function collectClusterHealth(config, db, runCommand) {
|
|
9
|
+
const checks = [];
|
|
10
|
+
const ciEntries = [];
|
|
11
|
+
const now = Date.now();
|
|
12
|
+
const issues = db.listIssues();
|
|
13
|
+
const openIssues = issues.filter((issue) => issue.factoryState !== "done");
|
|
14
|
+
const trackedByKey = new Map(issues
|
|
15
|
+
.filter((issue) => issue.issueKey)
|
|
16
|
+
.map((issue) => [issue.issueKey, issue]));
|
|
17
|
+
const trackedByLinearId = new Map(issues.map((issue) => [issue.linearIssueId, issue]));
|
|
18
|
+
const patchRelayProbe = await probePatchRelayService(config);
|
|
19
|
+
checks.push({
|
|
20
|
+
status: patchRelayProbe.status,
|
|
21
|
+
scope: "service:patchrelay",
|
|
22
|
+
message: patchRelayProbe.message,
|
|
23
|
+
});
|
|
24
|
+
const snapshots = openIssues.map((issue) => {
|
|
25
|
+
const tracked = db.getTrackedIssue(issue.projectId, issue.linearIssueId);
|
|
26
|
+
const deps = db.issues.listIssueDependencies(issue.projectId, issue.linearIssueId);
|
|
27
|
+
const blockedBy = deps.filter((dep) => !isResolvedDependency(dep));
|
|
28
|
+
const missingTrackedBlockers = blockedBy.filter((dep) => {
|
|
29
|
+
if (trackedByLinearId.has(dep.blockerLinearIssueId))
|
|
30
|
+
return false;
|
|
31
|
+
if (dep.blockerIssueKey && trackedByKey.has(dep.blockerIssueKey))
|
|
32
|
+
return false;
|
|
33
|
+
return true;
|
|
34
|
+
});
|
|
35
|
+
return {
|
|
36
|
+
issue,
|
|
37
|
+
session: db.issueSessions.getIssueSession(issue.projectId, issue.linearIssueId),
|
|
38
|
+
blockedBy,
|
|
39
|
+
missingTrackedBlockers,
|
|
40
|
+
ageMs: Math.max(0, now - Date.parse(issue.updatedAt || new Date(0).toISOString())),
|
|
41
|
+
readyForExecution: tracked?.readyForExecution ?? false,
|
|
42
|
+
};
|
|
43
|
+
});
|
|
44
|
+
const reviewRelevantIssues = snapshots.filter((snapshot) => needsReviewAutomation(snapshot.issue));
|
|
45
|
+
const queueRelevantIssues = snapshots.filter((snapshot) => snapshot.issue.factoryState === "awaiting_queue");
|
|
46
|
+
const reviewQuillProbe = reviewRelevantIssues.length > 0
|
|
47
|
+
? await probeOptionalService(runCommand, "review-quill", {
|
|
48
|
+
healthy: (payload) => {
|
|
49
|
+
const parsed = payload;
|
|
50
|
+
return parsed.health?.reachable === true && parsed.health?.ok === true && parsed.systemd?.ActiveState === "active";
|
|
51
|
+
},
|
|
52
|
+
summarize: (payload) => {
|
|
53
|
+
const parsed = payload;
|
|
54
|
+
return parsed.health?.reachable === true && parsed.health?.ok === true
|
|
55
|
+
? "Healthy"
|
|
56
|
+
: `Unhealthy (${parsed.health?.reachable === false ? "service not reachable" : "service health unavailable"})`;
|
|
57
|
+
},
|
|
58
|
+
})
|
|
59
|
+
: undefined;
|
|
60
|
+
if (reviewQuillProbe) {
|
|
61
|
+
checks.push({
|
|
62
|
+
status: reviewQuillProbe.status,
|
|
63
|
+
scope: "service:review-quill",
|
|
64
|
+
message: reviewQuillProbe.message,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
const reviewQuillAttemptOwners = reviewQuillProbe?.status === "pass"
|
|
68
|
+
? await collectReviewQuillAttemptOwners(reviewRelevantIssues, config, runCommand)
|
|
69
|
+
: new Map();
|
|
70
|
+
const mergeStewardProbe = queueRelevantIssues.length > 0
|
|
71
|
+
? await probeOptionalService(runCommand, "merge-steward", {
|
|
72
|
+
healthy: (payload) => {
|
|
73
|
+
const parsed = payload;
|
|
74
|
+
return parsed.health?.reachable === true && parsed.health?.ok === true && parsed.systemd?.ActiveState === "active";
|
|
75
|
+
},
|
|
76
|
+
summarize: (payload) => {
|
|
77
|
+
const parsed = payload;
|
|
78
|
+
return parsed.health?.reachable === true && parsed.health?.ok === true && parsed.systemd?.ActiveState === "active"
|
|
79
|
+
? "Healthy"
|
|
80
|
+
: `Unhealthy (${parsed.health?.reachable === false ? "service not reachable" : parsed.systemd?.ActiveState ?? "unknown"})`;
|
|
81
|
+
},
|
|
82
|
+
})
|
|
83
|
+
: undefined;
|
|
84
|
+
if (mergeStewardProbe) {
|
|
85
|
+
checks.push({
|
|
86
|
+
status: mergeStewardProbe.status,
|
|
87
|
+
scope: "service:merge-steward",
|
|
88
|
+
message: mergeStewardProbe.message,
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
for (const snapshot of snapshots) {
|
|
92
|
+
const finding = evaluateLocalIssueHealth(snapshot);
|
|
93
|
+
if (finding) {
|
|
94
|
+
checks.push({
|
|
95
|
+
...finding,
|
|
96
|
+
...(snapshot.issue.issueKey ? { issueKey: snapshot.issue.issueKey } : {}),
|
|
97
|
+
projectId: snapshot.issue.projectId,
|
|
98
|
+
...(snapshot.issue.prNumber !== undefined ? { prNumber: snapshot.issue.prNumber } : {}),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
checks.push(...await collectActiveOverlapFindings(snapshots, runCommand));
|
|
103
|
+
for (const snapshot of snapshots) {
|
|
104
|
+
if (!hasOpenPr(snapshot.issue.prNumber, snapshot.issue.prState)) {
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
const githubHealth = await evaluateGitHubIssueHealth(snapshot, config, runCommand, reviewQuillProbe, reviewQuillAttemptOwners, mergeStewardProbe);
|
|
108
|
+
if (githubHealth.ciEntry) {
|
|
109
|
+
ciEntries.push(githubHealth.ciEntry);
|
|
110
|
+
}
|
|
111
|
+
if (githubHealth.finding) {
|
|
112
|
+
checks.push({
|
|
113
|
+
...githubHealth.finding,
|
|
114
|
+
...(snapshot.issue.issueKey ? { issueKey: snapshot.issue.issueKey } : {}),
|
|
115
|
+
projectId: snapshot.issue.projectId,
|
|
116
|
+
prNumber: snapshot.issue.prNumber,
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const workflowFailures = checks.filter((check) => check.scope.startsWith("issue:") || check.scope.startsWith("github:"));
|
|
121
|
+
if (workflowFailures.every((check) => check.status === "pass" || check.status === "warn") && openIssues.length > 0) {
|
|
122
|
+
checks.push({
|
|
123
|
+
status: "pass",
|
|
124
|
+
scope: "workflow",
|
|
125
|
+
message: `All ${openIssues.length} non-done issues currently have active work, a tracked blocker, or a downstream owner`,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
if (openIssues.length === 0) {
|
|
129
|
+
checks.push({
|
|
130
|
+
status: "pass",
|
|
131
|
+
scope: "workflow",
|
|
132
|
+
message: "No non-done issues are currently tracked",
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
if (ciEntries.length > 0) {
|
|
136
|
+
const orphanedCi = ciEntries.filter((entry) => entry.orphaned);
|
|
137
|
+
checks.push({
|
|
138
|
+
status: orphanedCi.length === 0 ? "pass" : "fail",
|
|
139
|
+
scope: "ci",
|
|
140
|
+
message: orphanedCi.length === 0
|
|
141
|
+
? `Tracked ${ciEntries.length} PR-backed issue${ciEntries.length === 1 ? "" : "s"} and each PR has a visible next owner`
|
|
142
|
+
: `${orphanedCi.length} PR-backed issue${orphanedCi.length === 1 ? "" : "s"} ha${orphanedCi.length === 1 ? "s" : "ve"} no visible next owner`,
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
const summary = {
|
|
146
|
+
trackedIssues: issues.length,
|
|
147
|
+
openIssues: openIssues.length,
|
|
148
|
+
activeRuns: openIssues.filter((issue) => issue.activeRunId !== undefined).length,
|
|
149
|
+
blockedIssues: snapshots.filter((snapshot) => snapshot.blockedBy.length > 0).length,
|
|
150
|
+
readyIssues: snapshots.filter((snapshot) => snapshot.readyForExecution).length,
|
|
151
|
+
ciTrackedPrs: ciEntries.length,
|
|
152
|
+
ciPending: ciEntries.filter((entry) => entry.gateStatus === "pending").length,
|
|
153
|
+
ciSuccess: ciEntries.filter((entry) => entry.gateStatus === "success").length,
|
|
154
|
+
ciFailure: ciEntries.filter((entry) => entry.gateStatus === "failure").length,
|
|
155
|
+
ciUnknown: ciEntries.filter((entry) => entry.gateStatus === "unknown").length,
|
|
156
|
+
ciOrphaned: ciEntries.filter((entry) => entry.orphaned).length,
|
|
157
|
+
passCount: checks.filter((check) => check.status === "pass").length,
|
|
158
|
+
warnCount: checks.filter((check) => check.status === "warn").length,
|
|
159
|
+
failCount: checks.filter((check) => check.status === "fail").length,
|
|
160
|
+
};
|
|
161
|
+
return {
|
|
162
|
+
generatedAt: new Date().toISOString(),
|
|
163
|
+
ok: summary.failCount === 0,
|
|
164
|
+
summary,
|
|
165
|
+
checks,
|
|
166
|
+
ci: ciEntries,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { ACTIVE_RUN_STATES } from "../../factory-state.js";
|
|
2
|
+
import { isUndelegatedPausedNoPrWork } from "../../paused-issue-state.js";
|
|
3
|
+
import { hasOpenPr } from "../../pr-state.js";
|
|
4
|
+
import { DOWNSTREAM_STALE_MS, RECONCILIATION_GRACE_MS } from "./shared.js";
|
|
5
|
+
export function isResolvedDependency(dep) {
|
|
6
|
+
const stateType = dep.blockerCurrentLinearStateType?.trim().toLowerCase();
|
|
7
|
+
const state = dep.blockerCurrentLinearState?.trim().toLowerCase();
|
|
8
|
+
return stateType === "completed"
|
|
9
|
+
|| stateType === "canceled"
|
|
10
|
+
|| stateType === "cancelled"
|
|
11
|
+
|| state === "done"
|
|
12
|
+
|| state === "canceled"
|
|
13
|
+
|| state === "cancelled";
|
|
14
|
+
}
|
|
15
|
+
export function needsReviewAutomation(issue) {
|
|
16
|
+
if (issue.factoryState === "awaiting_queue" || issue.factoryState === "done") {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
return hasOpenPr(issue.prNumber, issue.prState);
|
|
20
|
+
}
|
|
21
|
+
export function evaluateLocalIssueHealth(snapshot) {
|
|
22
|
+
const { issue, session, missingTrackedBlockers, blockedBy, ageMs, readyForExecution } = snapshot;
|
|
23
|
+
const pausedNoPrWork = isUndelegatedPausedNoPrWork(issue);
|
|
24
|
+
if (issue.factoryState === "failed" || issue.factoryState === "escalated") {
|
|
25
|
+
return {
|
|
26
|
+
status: "fail",
|
|
27
|
+
scope: "issue:terminal",
|
|
28
|
+
message: `Issue is in terminal failure state ${issue.factoryState}`,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
if (missingTrackedBlockers.length > 0) {
|
|
32
|
+
return {
|
|
33
|
+
status: "fail",
|
|
34
|
+
scope: "issue:blockers",
|
|
35
|
+
message: `Blocked by unmanaged issue${missingTrackedBlockers.length === 1 ? "" : "s"} ${missingTrackedBlockers.map((dep) => dep.blockerIssueKey ?? dep.blockerLinearIssueId).join(", ")}`,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
if (issue.activeRunId !== undefined && session?.sessionState !== "running") {
|
|
39
|
+
return {
|
|
40
|
+
status: "fail",
|
|
41
|
+
scope: "issue:run-state",
|
|
42
|
+
message: `Issue has active run #${issue.activeRunId} but session state is ${session?.sessionState ?? "missing"}`,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
if (issue.activeRunId === undefined && session?.sessionState === "running") {
|
|
46
|
+
return {
|
|
47
|
+
status: "fail",
|
|
48
|
+
scope: "issue:run-state",
|
|
49
|
+
message: "Issue session is marked running but no active run is attached",
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
if (blockedBy.length > 0) {
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
if (readyForExecution && issue.activeRunId === undefined && ageMs >= RECONCILIATION_GRACE_MS) {
|
|
56
|
+
return {
|
|
57
|
+
status: "fail",
|
|
58
|
+
scope: "issue:dispatch",
|
|
59
|
+
message: "Issue is ready for execution but no active run has started",
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
if (!pausedNoPrWork && ACTIVE_RUN_STATES.has(issue.factoryState) && issue.activeRunId === undefined && ageMs >= RECONCILIATION_GRACE_MS) {
|
|
63
|
+
return {
|
|
64
|
+
status: "fail",
|
|
65
|
+
scope: "issue:dispatch",
|
|
66
|
+
message: `Issue is parked in ${issue.factoryState} without an active run`,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
if (!pausedNoPrWork && issue.factoryState === "delegated" && issue.activeRunId === undefined && !readyForExecution && ageMs >= RECONCILIATION_GRACE_MS) {
|
|
70
|
+
return {
|
|
71
|
+
status: "fail",
|
|
72
|
+
scope: "issue:dispatch",
|
|
73
|
+
message: "Delegated issue is idle but no wake is queued",
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
if (issue.factoryState === "awaiting_input" && ageMs >= RECONCILIATION_GRACE_MS) {
|
|
77
|
+
return {
|
|
78
|
+
status: "warn",
|
|
79
|
+
scope: "issue:operator",
|
|
80
|
+
message: "Issue is waiting on operator input",
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
if (issue.factoryState === "awaiting_queue" && ageMs >= DOWNSTREAM_STALE_MS) {
|
|
84
|
+
return {
|
|
85
|
+
status: "warn",
|
|
86
|
+
scope: "issue:downstream",
|
|
87
|
+
message: "Issue has been waiting on downstream merge automation for a long time",
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
return undefined;
|
|
91
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { probeGitHubPullRequest } from "./github-probe.js";
|
|
2
|
+
import { safeJsonParse } from "./shared.js";
|
|
3
|
+
export async function collectReviewQuillAttemptOwners(snapshots, config, runCommand) {
|
|
4
|
+
const owners = new Map();
|
|
5
|
+
const repoBacklog = await probeReviewQuillRepoBacklog(runCommand);
|
|
6
|
+
for (const snapshot of snapshots) {
|
|
7
|
+
const issueKey = snapshot.issue.issueKey;
|
|
8
|
+
const prNumber = snapshot.issue.prNumber;
|
|
9
|
+
if (!issueKey || prNumber === undefined)
|
|
10
|
+
continue;
|
|
11
|
+
const project = config.projects.find((entry) => entry.id === snapshot.issue.projectId);
|
|
12
|
+
const repoFullName = project?.github?.repoFullName;
|
|
13
|
+
if (!repoFullName)
|
|
14
|
+
continue;
|
|
15
|
+
const probe = await probeReviewQuillAttempts(runCommand, repoFullName, prNumber);
|
|
16
|
+
if (!probe.ok)
|
|
17
|
+
continue;
|
|
18
|
+
const activeAttempt = probe.attempts.find((attempt) => (attempt.status === "queued" || attempt.status === "running")
|
|
19
|
+
&& !attempt.stale
|
|
20
|
+
&& attempt.headSha === probe.currentHeadSha);
|
|
21
|
+
if (!activeAttempt) {
|
|
22
|
+
if (repoBacklog.has(repoFullName)) {
|
|
23
|
+
owners.set(issueKey, { backlog: true, headSha: probe.latestAttemptHeadSha });
|
|
24
|
+
}
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
owners.set(issueKey, {
|
|
28
|
+
id: activeAttempt.id,
|
|
29
|
+
status: activeAttempt.status,
|
|
30
|
+
headSha: activeAttempt.headSha,
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return owners;
|
|
34
|
+
}
|
|
35
|
+
export async function probeReviewQuillRepoBacklog(runCommand) {
|
|
36
|
+
let result;
|
|
37
|
+
try {
|
|
38
|
+
result = await runCommand("review-quill", ["status", "--json"]);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return new Set();
|
|
42
|
+
}
|
|
43
|
+
if (result.exitCode !== 0) {
|
|
44
|
+
return new Set();
|
|
45
|
+
}
|
|
46
|
+
const parsed = safeJsonParse(result.stdout);
|
|
47
|
+
if (!parsed || parsed.runtime?.reconcileInProgress !== true || !Array.isArray(parsed.repos)) {
|
|
48
|
+
return new Set();
|
|
49
|
+
}
|
|
50
|
+
const activeRepos = new Set();
|
|
51
|
+
for (const repo of parsed.repos) {
|
|
52
|
+
if (!repo || typeof repo !== "object")
|
|
53
|
+
continue;
|
|
54
|
+
const repoFullName = typeof repo.repoFullName === "string"
|
|
55
|
+
? String(repo.repoFullName).trim()
|
|
56
|
+
: undefined;
|
|
57
|
+
const runningAttempts = typeof repo.runningAttempts === "number"
|
|
58
|
+
? Number(repo.runningAttempts)
|
|
59
|
+
: 0;
|
|
60
|
+
const queuedAttempts = typeof repo.queuedAttempts === "number"
|
|
61
|
+
? Number(repo.queuedAttempts)
|
|
62
|
+
: 0;
|
|
63
|
+
if (!repoFullName)
|
|
64
|
+
continue;
|
|
65
|
+
if (runningAttempts > 0 || queuedAttempts > 0) {
|
|
66
|
+
activeRepos.add(repoFullName);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return activeRepos;
|
|
70
|
+
}
|
|
71
|
+
export async function probeReviewQuillAttempts(runCommand, repoFullName, prNumber) {
|
|
72
|
+
const repoRef = repoFullName.split("/").at(-1);
|
|
73
|
+
if (!repoRef) {
|
|
74
|
+
return { ok: false, error: `Unable to derive review-quill repo id from ${repoFullName}` };
|
|
75
|
+
}
|
|
76
|
+
let attemptsResult;
|
|
77
|
+
try {
|
|
78
|
+
attemptsResult = await runCommand("review-quill", ["attempts", repoRef, String(prNumber), "--json"]);
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
82
|
+
}
|
|
83
|
+
if (attemptsResult.exitCode !== 0) {
|
|
84
|
+
return {
|
|
85
|
+
ok: false,
|
|
86
|
+
error: [attemptsResult.stderr.trim(), attemptsResult.stdout.trim()].filter(Boolean).join(" ") || `review-quill exited ${attemptsResult.exitCode}`,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
const parsedAttempts = safeJsonParse(attemptsResult.stdout);
|
|
90
|
+
if (!parsedAttempts || !Array.isArray(parsedAttempts.attempts)) {
|
|
91
|
+
return { ok: false, error: "invalid JSON from review-quill attempts" };
|
|
92
|
+
}
|
|
93
|
+
const prProbe = await probeGitHubPullRequest(runCommand, repoFullName, prNumber);
|
|
94
|
+
if (!prProbe.ok) {
|
|
95
|
+
return { ok: false, error: prProbe.error };
|
|
96
|
+
}
|
|
97
|
+
let latestAttemptHeadSha;
|
|
98
|
+
const attempts = parsedAttempts.attempts.flatMap((entry) => {
|
|
99
|
+
if (!entry || typeof entry !== "object")
|
|
100
|
+
return [];
|
|
101
|
+
const id = entry.id;
|
|
102
|
+
const headSha = entry.headSha;
|
|
103
|
+
const status = entry.status;
|
|
104
|
+
const stale = entry.stale;
|
|
105
|
+
if (!latestAttemptHeadSha && typeof headSha === "string" && headSha.trim().length > 0) {
|
|
106
|
+
latestAttemptHeadSha = headSha.trim();
|
|
107
|
+
}
|
|
108
|
+
if (typeof id !== "number"
|
|
109
|
+
|| typeof headSha !== "string"
|
|
110
|
+
|| (status !== "queued" && status !== "running")) {
|
|
111
|
+
return [];
|
|
112
|
+
}
|
|
113
|
+
return [{
|
|
114
|
+
id,
|
|
115
|
+
headSha,
|
|
116
|
+
status: status,
|
|
117
|
+
stale: stale === true,
|
|
118
|
+
}];
|
|
119
|
+
});
|
|
120
|
+
return {
|
|
121
|
+
ok: true,
|
|
122
|
+
currentHeadSha: prProbe.pr.headRefOid,
|
|
123
|
+
latestAttemptHeadSha,
|
|
124
|
+
attempts,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { safeJsonParse } from "./shared.js";
|
|
2
|
+
export async function probePatchRelayService(config) {
|
|
3
|
+
const host = config.server.bind === "0.0.0.0" ? "127.0.0.1" : config.server.bind;
|
|
4
|
+
const healthUrl = `http://${host}:${config.server.port}${config.server.healthPath}`;
|
|
5
|
+
const readyUrl = `http://${host}:${config.server.port}${config.server.readinessPath}`;
|
|
6
|
+
try {
|
|
7
|
+
const [healthResponse, readyResponse] = await Promise.all([
|
|
8
|
+
fetch(healthUrl, { signal: AbortSignal.timeout(2_000) }),
|
|
9
|
+
fetch(readyUrl, { signal: AbortSignal.timeout(2_000) }),
|
|
10
|
+
]);
|
|
11
|
+
const healthBody = await healthResponse.json();
|
|
12
|
+
const readyBody = await readyResponse.json();
|
|
13
|
+
if (healthResponse.ok && readyResponse.ok && readyBody.ready) {
|
|
14
|
+
return {
|
|
15
|
+
status: "pass",
|
|
16
|
+
message: `Healthy${healthBody.version ? ` (v${healthBody.version})` : ""}`,
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
return {
|
|
20
|
+
status: "fail",
|
|
21
|
+
message: `Reachable but not ready${readyBody.codexStarted === false || readyBody.linearConnected === false
|
|
22
|
+
? ` (${[
|
|
23
|
+
readyBody.codexStarted === false ? "codex not started" : undefined,
|
|
24
|
+
readyBody.linearConnected === false ? "Linear not connected" : undefined,
|
|
25
|
+
].filter(Boolean).join(", ")})`
|
|
26
|
+
: ""}`,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
return {
|
|
31
|
+
status: "fail",
|
|
32
|
+
message: `Unavailable: ${error instanceof Error ? error.message : String(error)}`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export async function probeOptionalService(runCommand, binary, options) {
|
|
37
|
+
let result;
|
|
38
|
+
try {
|
|
39
|
+
result = await runCommand(binary, ["service", "status", "--json"]);
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
return {
|
|
43
|
+
status: "warn",
|
|
44
|
+
message: `Unavailable: ${error instanceof Error ? error.message : String(error)}`,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
if (result.exitCode !== 0) {
|
|
48
|
+
const errorText = [result.stderr.trim(), result.stdout.trim()].filter(Boolean).join(" ");
|
|
49
|
+
return {
|
|
50
|
+
status: "warn",
|
|
51
|
+
message: `Unavailable: ${errorText || `${binary} service status exited ${result.exitCode}`}`,
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
const payload = safeJsonParse(result.stdout);
|
|
55
|
+
if (!payload) {
|
|
56
|
+
return {
|
|
57
|
+
status: "warn",
|
|
58
|
+
message: "Unavailable: unable to parse JSON status output",
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
status: options.healthy(payload) ? "pass" : "fail",
|
|
63
|
+
message: options.summarize(payload),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export const RECONCILIATION_GRACE_MS = 120_000;
|
|
2
|
+
export const DOWNSTREAM_STALE_MS = 900_000;
|
|
3
|
+
export function safeJsonParse(value) {
|
|
4
|
+
try {
|
|
5
|
+
const parsed = JSON.parse(value);
|
|
6
|
+
return parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
|
7
|
+
? parsed
|
|
8
|
+
: undefined;
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { collectClusterHealth } from "../cluster-health.js";
|
|
1
|
+
import { collectClusterHealth } from "../cluster-health/index.js";
|
|
2
2
|
import { CliUsageError } from "../errors.js";
|
|
3
3
|
import { formatJson } from "../formatters/json.js";
|
|
4
4
|
import { writeOutput } from "../output.js";
|
package/dist/codex-app-server.js
CHANGED
|
@@ -286,12 +286,22 @@ export class CodexAppServerClient extends EventEmitter {
|
|
|
286
286
|
},
|
|
287
287
|
});
|
|
288
288
|
});
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
289
|
+
try {
|
|
290
|
+
this.writeMessage({
|
|
291
|
+
jsonrpc: "2.0",
|
|
292
|
+
id,
|
|
293
|
+
method,
|
|
294
|
+
params,
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
catch (error) {
|
|
298
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
299
|
+
const pending = this.pending.get(id);
|
|
300
|
+
if (pending) {
|
|
301
|
+
this.pending.delete(id);
|
|
302
|
+
pending.reject(err);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
295
305
|
return promise.catch((error) => {
|
|
296
306
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
297
307
|
this.logger.error({
|
|
@@ -390,10 +400,37 @@ export class CodexAppServerClient extends EventEmitter {
|
|
|
390
400
|
}
|
|
391
401
|
}
|
|
392
402
|
writeMessage(message) {
|
|
393
|
-
|
|
394
|
-
|
|
403
|
+
const child = this.child;
|
|
404
|
+
const stdin = child?.stdin;
|
|
405
|
+
if (!stdin || stdin.destroyed || stdin.writableEnded || !stdin.writable) {
|
|
406
|
+
const error = new Error("Codex app-server stdin is closed");
|
|
407
|
+
this.handleTransportFailure(error);
|
|
408
|
+
throw error;
|
|
395
409
|
}
|
|
396
|
-
|
|
410
|
+
try {
|
|
411
|
+
stdin.write(`${JSON.stringify(message)}\n`, (error) => {
|
|
412
|
+
if (error) {
|
|
413
|
+
this.handleTransportFailure(error instanceof Error ? error : new Error(String(error)));
|
|
414
|
+
}
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
catch (error) {
|
|
418
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
419
|
+
this.handleTransportFailure(err);
|
|
420
|
+
throw err;
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
handleTransportFailure(error) {
|
|
424
|
+
const child = this.child;
|
|
425
|
+
this.started = false;
|
|
426
|
+
this.child = undefined;
|
|
427
|
+
this.stdoutBuffer = "";
|
|
428
|
+
this.logger.error({
|
|
429
|
+
error: sanitizeDiagnosticText(error.message),
|
|
430
|
+
pendingRequestCount: this.pending.size,
|
|
431
|
+
}, "Codex app-server transport failed");
|
|
432
|
+
this.rejectAllPending(error);
|
|
433
|
+
child?.kill("SIGTERM");
|
|
397
434
|
}
|
|
398
435
|
drainMessages() {
|
|
399
436
|
while (true) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { buildBranchUpkeepContext, buildFailureContext, getGateCheckNames, hasCompletedReviewQuillVerdict, hasFailureProvenance, isDuplicateRepairAttempt, isFailingCheckStatus, isReviewDecisionApproved,
|
|
1
|
+
import { buildBranchUpkeepContext, buildFailureContext, getGateCheckNames, hasCompletedReviewQuillVerdict, hasFailureProvenance, isDuplicateRepairAttempt, isFailingCheckStatus, isReviewDecisionApproved, isReviewDecisionReviewRequired, } from "./idle-reconciliation-helpers.js";
|
|
2
2
|
import { isMainRepairIssue } from "./main-repair.js";
|
|
3
3
|
import { resolveMergeQueueProtocol } from "./merge-queue-protocol.js";
|
|
4
4
|
import { deriveGateCheckStatusFromRollup } from "./github-rollup.js";
|
|
@@ -6,6 +6,8 @@ import { deriveIssueSessionReactiveIntent } from "./issue-session.js";
|
|
|
6
6
|
import { buildClosedPrCleanupFields, resolveClosedPrDisposition } from "./pr-state.js";
|
|
7
7
|
import { getReviewFixBudget } from "./run-budgets.js";
|
|
8
8
|
import { queueSettledOrchestrationIssue } from "./orchestration-parent-wake.js";
|
|
9
|
+
import { fetchPullRequestSnapshot } from "./reconcile-pr-fetch.js";
|
|
10
|
+
import { buildPrStateUpdates } from "./reconcile-pr-state-updates.js";
|
|
9
11
|
import { execCommand } from "./utils.js";
|
|
10
12
|
export class IdleIssueReconciler {
|
|
11
13
|
db;
|
|
@@ -373,13 +375,18 @@ export class IdleIssueReconciler {
|
|
|
373
375
|
const project = this.config.projects.find((p) => p.id === issue.projectId);
|
|
374
376
|
if (!project?.github?.repoFullName || !issue.prNumber)
|
|
375
377
|
return;
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
"
|
|
381
|
-
|
|
382
|
-
|
|
378
|
+
const snapshot = await fetchPullRequestSnapshot(project.github.repoFullName, issue.prNumber);
|
|
379
|
+
if (!snapshot.ok) {
|
|
380
|
+
this.logger.debug({ issueKey: issue.issueKey, error: snapshot.error.message }, "Failed to query GitHub PR state during reconciliation");
|
|
381
|
+
if (issue.prReviewState === "approved") {
|
|
382
|
+
if (issue.factoryState !== "awaiting_queue" || hasFailureProvenance(issue)) {
|
|
383
|
+
this.advanceIdleIssue(issue, "awaiting_queue", hasFailureProvenance(issue) ? { clearFailureProvenance: true } : {});
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
const pr = snapshot.pr;
|
|
389
|
+
{
|
|
383
390
|
const previousHeadSha = issue.prHeadSha;
|
|
384
391
|
const gateCheckNames = getGateCheckNames(project);
|
|
385
392
|
const gateCheckStatus = deriveGateCheckStatusFromRollup(pr.statusCheckRollup, gateCheckNames);
|
|
@@ -389,24 +396,7 @@ export class IdleIssueReconciler {
|
|
|
389
396
|
this.db.issues.upsertIssue({
|
|
390
397
|
projectId: issue.projectId,
|
|
391
398
|
linearIssueId: issue.linearIssueId,
|
|
392
|
-
...(pr
|
|
393
|
-
...(pr.state === "OPEN" ? { prState: "open" } : {}),
|
|
394
|
-
...(isReviewDecisionApproved(pr.reviewDecision)
|
|
395
|
-
? { prReviewState: "approved" }
|
|
396
|
-
: isReviewDecisionChangesRequested(pr.reviewDecision)
|
|
397
|
-
? { prReviewState: "changes_requested" }
|
|
398
|
-
: isReviewDecisionReviewRequired(pr.reviewDecision)
|
|
399
|
-
? { prReviewState: "commented" }
|
|
400
|
-
: {}),
|
|
401
|
-
...(gateCheckStatus ? { prCheckStatus: gateCheckStatus } : {}),
|
|
402
|
-
...(pr.headRefOid && gateCheckStatus
|
|
403
|
-
? {
|
|
404
|
-
lastGitHubCiSnapshotHeadSha: pr.headRefOid,
|
|
405
|
-
lastGitHubCiSnapshotGateCheckName: gateCheckNames[0] ?? "verify",
|
|
406
|
-
lastGitHubCiSnapshotGateCheckStatus: gateCheckStatus,
|
|
407
|
-
lastGitHubCiSnapshotSettledAt: gateCheckStatus === "pending" ? null : new Date().toISOString(),
|
|
408
|
-
}
|
|
409
|
-
: {}),
|
|
399
|
+
...buildPrStateUpdates(pr, gateCheckStatus, gateCheckNames[0] ?? "verify"),
|
|
410
400
|
});
|
|
411
401
|
if (pr.state === "MERGED") {
|
|
412
402
|
this.db.issues.upsertIssue({ projectId: issue.projectId, linearIssueId: issue.linearIssueId, prState: "merged" });
|
|
@@ -600,16 +590,6 @@ export class IdleIssueReconciler {
|
|
|
600
590
|
this.logger.debug({ issueKey: issue.issueKey, prNumber: issue.prNumber, mergeable: pr.mergeable, mergeStateStatus: pr.mergeStateStatus }, "Reconciliation: PR is dirty but no automation owner was derived");
|
|
601
591
|
}
|
|
602
592
|
}
|
|
603
|
-
catch (error) {
|
|
604
|
-
this.logger.debug({ issueKey: issue.issueKey, error: error instanceof Error ? error.message : String(error) }, "Failed to query GitHub PR state during reconciliation");
|
|
605
|
-
if (issue.prReviewState === "approved") {
|
|
606
|
-
if (issue.factoryState !== "awaiting_queue" || hasFailureProvenance(issue)) {
|
|
607
|
-
this.advanceIdleIssue(issue, "awaiting_queue", {
|
|
608
|
-
...(hasFailureProvenance(issue) ? { clearFailureProvenance: true } : {}),
|
|
609
|
-
});
|
|
610
|
-
}
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
593
|
}
|
|
614
594
|
deriveTerminalRecoveryState(issue, reviewDecision, gateCheckStatus, headAdvanced) {
|
|
615
595
|
if (issue.factoryState !== "escalated" && issue.factoryState !== "failed") {
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { execCommand } from "./utils.js";
|
|
2
|
+
/**
|
|
3
|
+
* Snapshots a PR via `gh pr view --json`, used during idle reconciliation
|
|
4
|
+
* to verify the local state machine against fresh GitHub truth before
|
|
5
|
+
* dispatching repair runs.
|
|
6
|
+
*
|
|
7
|
+
* Caller is responsible for logging / acting on `ok: false`; this helper
|
|
8
|
+
* never throws.
|
|
9
|
+
*/
|
|
10
|
+
export async function fetchPullRequestSnapshot(repoFullName, prNumber, options = {}) {
|
|
11
|
+
try {
|
|
12
|
+
const { stdout } = await execCommand("gh", [
|
|
13
|
+
"pr", "view", String(prNumber),
|
|
14
|
+
"--repo", repoFullName,
|
|
15
|
+
"--json", "headRefOid,state,reviewDecision,mergeable,mergeStateStatus,statusCheckRollup",
|
|
16
|
+
], { timeoutMs: options.timeoutMs ?? 10_000 });
|
|
17
|
+
const pr = JSON.parse(stdout);
|
|
18
|
+
return { ok: true, pr };
|
|
19
|
+
}
|
|
20
|
+
catch (error) {
|
|
21
|
+
return { ok: false, error: error instanceof Error ? error : new Error(String(error)) };
|
|
22
|
+
}
|
|
23
|
+
}
|