pi-claude-supervisor 0.9.0 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.cn.md +16 -8
- package/README.md +19 -8
- package/docs/architecture.md +42 -10
- package/docs/autonomy-target.md +1 -1
- package/docs/testing.md +40 -3
- package/package.json +2 -1
- package/src/acceptance.ts +2 -1
- package/src/config.ts +78 -32
- package/src/cwd-lease.ts +34 -1
- package/src/decision-session-store.ts +5 -14
- package/src/decision-worker.ts +89 -11
- package/src/events.ts +5 -11
- package/src/hooks/install.ts +2 -11
- package/src/hooks/server.ts +2 -10
- package/src/hooks/settings.ts +2 -11
- package/src/hooks/types.ts +17 -9
- package/src/index.ts +83 -4
- package/src/json-extract.ts +41 -2
- package/src/lock-owner.ts +55 -0
- package/src/redaction.ts +15 -0
- package/src/reviewer.ts +228 -53
- package/src/supervisor.ts +197 -16
- package/src/worker/process-adapter.ts +40 -5
- package/src/worker/tmux-adapter.ts +105 -21
package/src/supervisor.ts
CHANGED
|
@@ -3,7 +3,7 @@ import { join } from "node:path";
|
|
|
3
3
|
import { EventLog, type SupervisorEvent } from "./events.ts";
|
|
4
4
|
import { SupervisorStateMachine } from "./state.ts";
|
|
5
5
|
import { evaluatePermission, isCommitId, isProtectedBranch, isRoutinePermission, publishCommand, pullRequestCommand, type PermissionPolicyOptions, type PolicyResult, type RemoteGrant } from "./policy.ts";
|
|
6
|
-
import { PiDecisionWorker, type DecisionAction, type DecisionContext, type DecisionDeadlineContext, type DecisionWorkerFactory, type DecisionWorkerLike, type PiModel } from "./decision-worker.ts";
|
|
6
|
+
import { PiDecisionWorker, STALE_VERIFICATION_TURNS, type DecisionAction, type DecisionContext, type DecisionDeadlineContext, type DecisionWorkerFactory, type DecisionWorkerLike, type PiModel } from "./decision-worker.ts";
|
|
7
7
|
import { DEFAULT_DEADLINE_GRACE_MS, DEFAULT_DEADLINE_MS, DEFAULT_DEADLINE_WARNING_MS, DEFAULT_NO_OUTPUT_TIMEOUT_MS, formatDurationMs } from "./config.ts";
|
|
8
8
|
import { collectRepositoryEvidence, remoteBranchHead, remoteUrl, runReadOnly, repositoryBranch, repositoryCommitExists, repositoryHead, repositoryIsAncestor, repositoryWorkTree, verifyAll, repositoryClean, repositoryGitDirectoryIsLocal, repositorySlug, sameDestination, type RemoteBranchLookup, type RemoteDestination, type RepositoryEvidence, type VerificationCommand } from "./verifier.ts";
|
|
9
9
|
import { normalizeTaskSpec } from "./acceptance.ts";
|
|
@@ -214,11 +214,17 @@ export class Supervisor {
|
|
|
214
214
|
#lastObservedBranch?: string;
|
|
215
215
|
#handle?: WorkerHandle;
|
|
216
216
|
#lastVerification?: AcceptanceReport;
|
|
217
|
+
/** The Worker turn the last verification judged; later turns are not reflected in it. */
|
|
218
|
+
#lastVerificationTurn?: number;
|
|
219
|
+
/** Set when a Decision Worker `stop` became a verification: no repair turn may follow it. */
|
|
220
|
+
#stopVerification?: string;
|
|
217
221
|
#workerOutput = "";
|
|
218
222
|
#lastWorkerResult?: Record<string, unknown>;
|
|
219
223
|
#lastTurnCompleted?: WorkerEvent;
|
|
220
224
|
/** Key of the completed turn whose decision is still in flight; the deadline must not verify underneath it. */
|
|
221
225
|
#pendingDecisionKey?: string;
|
|
226
|
+
/** When the pending decision was requested; bounds how long an idle timeout defers to it. */
|
|
227
|
+
#pendingDecisionSince = 0;
|
|
222
228
|
/**
|
|
223
229
|
* Publish phase. Authority exists only between "acceptance and Reviewer
|
|
224
230
|
* passed" and "the publish turn completed", and only for `#verifiedHead` on
|
|
@@ -275,6 +281,7 @@ export class Supervisor {
|
|
|
275
281
|
#usageRecordedEvents = new Set<string>();
|
|
276
282
|
#pendingPermissions = new Map<string, WorkerPermissionRequest>();
|
|
277
283
|
#humanRequired = false;
|
|
284
|
+
#watchdogTickPending = false;
|
|
278
285
|
#humanGate: "permission" | "other" | undefined;
|
|
279
286
|
#candidateParked = false;
|
|
280
287
|
#stopRequested?: string;
|
|
@@ -315,6 +322,9 @@ export class Supervisor {
|
|
|
315
322
|
get task() { return this.#task; }
|
|
316
323
|
get handle() { return this.#handle; }
|
|
317
324
|
get lastVerification() { return this.#lastVerification; }
|
|
325
|
+
/** Worker turns sent so far (the task's own opening turn is 0). */
|
|
326
|
+
get turn() { return this.#turn; }
|
|
327
|
+
get repairRound() { return this.#repairRound; }
|
|
318
328
|
/** True only after an explicit takeover, never for ordinary uncertainty. */
|
|
319
329
|
get humanRequired() { return this.#humanRequired; }
|
|
320
330
|
get candidateParked() { return this.#candidateParked; }
|
|
@@ -346,6 +356,8 @@ export class Supervisor {
|
|
|
346
356
|
this.#handle = undefined;
|
|
347
357
|
this.#lastObservedBranch = undefined;
|
|
348
358
|
this.#lastVerification = undefined;
|
|
359
|
+
this.#lastVerificationTurn = undefined;
|
|
360
|
+
this.#stopVerification = undefined;
|
|
349
361
|
this.#workerOutput = "";
|
|
350
362
|
this.#lastWorkerResult = undefined;
|
|
351
363
|
this.#lastTurnCompleted = undefined;
|
|
@@ -1076,6 +1088,8 @@ export class Supervisor {
|
|
|
1076
1088
|
}
|
|
1077
1089
|
if (action.action === "continue" || action.action === "redirect" || action.action === "answer") {
|
|
1078
1090
|
if (await this.#decisionIsStale(event)) return;
|
|
1091
|
+
if (await this.#verifyOnTurnBudget(handle, event, action)) return;
|
|
1092
|
+
if (await this.#verifyStaleFailure(handle, event, action)) return;
|
|
1079
1093
|
await this.#sendInternal(action.message);
|
|
1080
1094
|
return;
|
|
1081
1095
|
}
|
|
@@ -1106,6 +1120,10 @@ export class Supervisor {
|
|
|
1106
1120
|
return;
|
|
1107
1121
|
}
|
|
1108
1122
|
if (action.action === "verify") {
|
|
1123
|
+
// Background work can re-invoke the Worker while this decision was in
|
|
1124
|
+
// flight; verifying then would judge a tree that is still changing
|
|
1125
|
+
// and stop a busy Worker. Its next completed turn is decided afresh.
|
|
1126
|
+
if (await this.#decisionIsStale(event)) return;
|
|
1109
1127
|
await this.#startVerification(handle, event, "Decision Worker");
|
|
1110
1128
|
return;
|
|
1111
1129
|
}
|
|
@@ -1125,17 +1143,81 @@ export class Supervisor {
|
|
|
1125
1143
|
return;
|
|
1126
1144
|
}
|
|
1127
1145
|
if (action.action === "stop") {
|
|
1146
|
+
if (await this.#verifyStop(handle, event, action)) return;
|
|
1128
1147
|
await this.#stopInternal(`Decision Worker: ${action.reason}`);
|
|
1129
1148
|
return;
|
|
1130
1149
|
}
|
|
1131
1150
|
if (action.action === "retry") {
|
|
1132
|
-
|
|
1151
|
+
// The prompt invites a bare retry after a Worker API error; resume the
|
|
1152
|
+
// turn rather than parking a task over a single transient failure.
|
|
1153
|
+
const message = action.message?.trim() ? action.message : RETRY_RESUME_MESSAGE;
|
|
1133
1154
|
if (await this.#decisionIsStale(event)) return;
|
|
1134
|
-
await this.#
|
|
1155
|
+
if (await this.#verifyOnTurnBudget(handle, event, action)) return;
|
|
1156
|
+
if (await this.#verifyStaleFailure(handle, event, action)) return;
|
|
1157
|
+
await this.#sendInternal(message);
|
|
1135
1158
|
}
|
|
1136
1159
|
});
|
|
1137
1160
|
}
|
|
1138
1161
|
|
|
1162
|
+
/**
|
|
1163
|
+
* The turn budget is spent: another message would only throw (and park the
|
|
1164
|
+
* task as a "Decision Worker failure"). Judge the work that exists instead.
|
|
1165
|
+
*/
|
|
1166
|
+
async #verifyOnTurnBudget(handle: WorkerHandle, event: WorkerEvent, action: DecisionAction): Promise<boolean> {
|
|
1167
|
+
const maxTurns = this.#task?.maxTurns ?? 100;
|
|
1168
|
+
if (this.#turn + 1 <= maxTurns) return false;
|
|
1169
|
+
await this.#appendEvent({ type: "decision_overridden", taskId: this.#task?.taskId, workerId: handle.id, data: { action: action.action, override: "verify", reason: `turn budget of ${maxTurns} is exhausted`, eventType: event.type } }).catch(() => {});
|
|
1170
|
+
await this.#startVerification(handle, event, "turn budget");
|
|
1171
|
+
return true;
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
/**
|
|
1175
|
+
* A verification failed and the Worker has since taken several turns with
|
|
1176
|
+
* no new one: judge the work as it is now instead of sending more guidance
|
|
1177
|
+
* based on the old failure. Another failure goes through the ordinary
|
|
1178
|
+
* repair-round budget, so a stuck task parks instead of running to its
|
|
1179
|
+
* deadline.
|
|
1180
|
+
*/
|
|
1181
|
+
async #verifyStaleFailure(handle: WorkerHandle, event: WorkerEvent, action: DecisionAction): Promise<boolean> {
|
|
1182
|
+
if (event.type !== "turn_completed" || !this.#lastVerification || this.#lastVerification.ok || this.#lastVerificationTurn === undefined) return false;
|
|
1183
|
+
// A turn cut short by the Worker's own API error is half-done work: let
|
|
1184
|
+
// the Decision Worker resume it rather than judging it.
|
|
1185
|
+
if (event.result.is_error === true || event.result.subtype === "error") return false;
|
|
1186
|
+
const turnsSince = this.#turn - this.#lastVerificationTurn;
|
|
1187
|
+
if (turnsSince < STALE_VERIFICATION_TURNS) return false;
|
|
1188
|
+
await this.#appendEvent({ type: "decision_overridden", taskId: this.#task?.taskId, workerId: handle.id, data: { action: action.action, override: "verify", reason: `${turnsSince} Worker turns since the last failed verification`, eventType: event.type } }).catch(() => {});
|
|
1189
|
+
await this.#startVerification(handle, event, "stale failed verification");
|
|
1190
|
+
return true;
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
/**
|
|
1194
|
+
* An unattended `stop` on a finished turn: models reach for it when they
|
|
1195
|
+
* believe the work is done, and a plain stop would end the task unverified
|
|
1196
|
+
* with its work unreported. The stop still happens first — the Worker is
|
|
1197
|
+
* stopped and never kept open — and then its finished work is judged
|
|
1198
|
+
* (acceptance and the Reviewer only read): a failure blocks the candidate
|
|
1199
|
+
* rather than starting a repair round. A stop elsewhere (a pending
|
|
1200
|
+
* permission, a turn the Worker has already resumed) or on a task that may
|
|
1201
|
+
* publish stays a plain stop.
|
|
1202
|
+
*/
|
|
1203
|
+
async #verifyStop(handle: WorkerHandle, event: WorkerEvent, action: DecisionAction): Promise<boolean> {
|
|
1204
|
+
const task = this.#task;
|
|
1205
|
+
if (!task || !this.#automation || this.#humanRequired || event.type !== "turn_completed" || this.#machine.state !== "waiting") return false;
|
|
1206
|
+
if (task.spec.autonomy.remoteAuthority !== "none") return false;
|
|
1207
|
+
if (await this.#decisionIsStale(event)) return false;
|
|
1208
|
+
this.#stopVerification = action.reason;
|
|
1209
|
+
await this.#appendEvent({ type: "decision_overridden", taskId: task.taskId, workerId: handle.id, data: { action: action.action, override: "verify", reason: "the Worker is stopped and its finished work verified before the task ends; no further Worker turns", decisionReason: action.reason, eventType: event.type } }).catch(() => {});
|
|
1210
|
+
await this.#adapter.stop(handle, `Decision Worker: ${action.reason}`);
|
|
1211
|
+
await this.#pollInternal(true);
|
|
1212
|
+
// Re-read after the awaits: the poll moves the stopped Worker on.
|
|
1213
|
+
const after: string = this.#machine.state;
|
|
1214
|
+
// An operator stop arrived meanwhile: it owns the outcome ("stopped").
|
|
1215
|
+
if (this.#stopRequested !== undefined) return true;
|
|
1216
|
+
if (after === "verifying") await this.#verifyInternal();
|
|
1217
|
+
else await this.#parkCandidate(`Decision Worker stop left the task in state ${after} before verification`, event);
|
|
1218
|
+
return true;
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1139
1221
|
/**
|
|
1140
1222
|
* Move an idle Worker into verification the way a `verify` decision does:
|
|
1141
1223
|
* in place when the transport supports it, otherwise by stopping the
|
|
@@ -1163,7 +1245,10 @@ export class Supervisor {
|
|
|
1163
1245
|
// nothing must be marked as pending on its account.
|
|
1164
1246
|
if (replay && !this.#decision.replay) return;
|
|
1165
1247
|
this.#decision.updateContext(this.#decisionContextPatch());
|
|
1166
|
-
if (event.type === "turn_completed")
|
|
1248
|
+
if (event.type === "turn_completed") {
|
|
1249
|
+
this.#pendingDecisionKey = workerEventKey(event);
|
|
1250
|
+
this.#pendingDecisionSince = Date.now();
|
|
1251
|
+
}
|
|
1167
1252
|
if (replay) this.#decision.replay!(event);
|
|
1168
1253
|
else this.#decision.notify(event);
|
|
1169
1254
|
}
|
|
@@ -1379,6 +1464,10 @@ export class Supervisor {
|
|
|
1379
1464
|
const nextTurn = this.#turn + 1;
|
|
1380
1465
|
if (nextTurn > (this.#task?.maxTurns ?? 100)) throw new Error("supervisor turn budget exhausted");
|
|
1381
1466
|
await this.#adapter.send(handle, message, `${taskId}:turn:${nextTurn}`);
|
|
1467
|
+
// The silence being timed starts now, not at the Worker's last output: a
|
|
1468
|
+
// repair or publish turn follows an acceptance/Review run that can easily
|
|
1469
|
+
// outlast the no-output timeout on its own.
|
|
1470
|
+
this.#noOutputBaselineAt = Date.now();
|
|
1382
1471
|
this.#turn = nextTurn;
|
|
1383
1472
|
if (this.#machine.state === "waiting") this.#machine.transition("running");
|
|
1384
1473
|
await this.#appendEvent({ type: "worker_message_sent", taskId, workerId: handle.id, idempotencyKey: `${taskId}:turn:${this.#turn}`, data: { message } });
|
|
@@ -1603,6 +1692,7 @@ export class Supervisor {
|
|
|
1603
1692
|
if (!this.#task) throw new Error("no active task");
|
|
1604
1693
|
if (this.#machine.state === "waiting" && canRepairInPlace(this.#adapter)) this.#machine.transition("verifying");
|
|
1605
1694
|
if (this.#machine.state !== "verifying") throw new Error(`cannot verify from ${this.#machine.state}`);
|
|
1695
|
+
this.#lastVerificationTurn = this.#turn;
|
|
1606
1696
|
const verificationAbortController = new AbortController();
|
|
1607
1697
|
this.#verificationAbortController = verificationAbortController;
|
|
1608
1698
|
|
|
@@ -1757,6 +1847,9 @@ export class Supervisor {
|
|
|
1757
1847
|
this.#reportProgress("review", "collecting repository evidence and running independent Reviewer", true);
|
|
1758
1848
|
let review: ReviewReport;
|
|
1759
1849
|
let reviewUsageReceived = false;
|
|
1850
|
+
// Until this round finishes, #lastVerification still holds the round
|
|
1851
|
+
// whose findings the Worker was just asked to repair.
|
|
1852
|
+
const previousFindings = this.#repairRound > 0 ? this.#lastVerification?.review?.findings : undefined;
|
|
1760
1853
|
try {
|
|
1761
1854
|
const evidence = repositoryEvidence ?? redactRepositoryEvidence(await collectRepositoryEvidence(this.#task.cwd, { signal: verificationAbortController.signal, baseRef: this.#task.baseCommit }));
|
|
1762
1855
|
judgedEvidence = evidence;
|
|
@@ -1769,6 +1862,7 @@ export class Supervisor {
|
|
|
1769
1862
|
workerOutput: String(redactSensitive(this.#workerOutput)),
|
|
1770
1863
|
workerResult: this.#lastWorkerResult ? redactSensitive(this.#lastWorkerResult) as Record<string, unknown> : undefined,
|
|
1771
1864
|
round: this.#repairRound,
|
|
1865
|
+
...(previousFindings?.length ? { previousFindings } : {}),
|
|
1772
1866
|
signal: verificationAbortController.signal,
|
|
1773
1867
|
onUsage: (sample) => { reviewUsageReceived = true; this.#recordPiUsage(sample); },
|
|
1774
1868
|
} satisfies ReviewInput), this.#reviewTimeoutMs + 30_000, "independent Reviewer", verificationAbortController.signal);
|
|
@@ -1781,8 +1875,12 @@ export class Supervisor {
|
|
|
1781
1875
|
if (error instanceof Error && error.name === "TimeoutError") verificationAbortController.abort(error.message);
|
|
1782
1876
|
review = { verdict: "human" as const, summary: `independent Reviewer failed: ${safeMessage(error)}`, findings: [], round: this.#repairRound, checkedAt: new Date().toISOString() };
|
|
1783
1877
|
}
|
|
1878
|
+
// A P0/P1 finding blocks a pass, but it is a repair input like any other
|
|
1879
|
+
// concrete finding: the bounded repair loop is where serious, fixable
|
|
1880
|
+
// defects get fixed. Only a `human` verdict (or an exhausted/repeating
|
|
1881
|
+
// repair loop) parks the candidate.
|
|
1784
1882
|
const hasBlockingFinding = review.findings.some((finding) => finding.severity === "P0" || finding.severity === "P1");
|
|
1785
|
-
if (hasBlockingFinding) review = { ...review, verdict: "
|
|
1883
|
+
if (hasBlockingFinding && review.verdict === "pass") review = { ...review, verdict: "revise" as const, summary: `${review.summary}; blocking findings must be repaired before the candidate can pass` };
|
|
1786
1884
|
if (review.verdict === "revise") {
|
|
1787
1885
|
const signature = findingSignature(review);
|
|
1788
1886
|
if (signature === this.#lastFindingSignature) {
|
|
@@ -2153,6 +2251,10 @@ export class Supervisor {
|
|
|
2153
2251
|
const task = this.#task;
|
|
2154
2252
|
const handle = this.#handle;
|
|
2155
2253
|
if (!task || !this.#automation || this.#humanRequired) return false;
|
|
2254
|
+
if (this.#stopVerification !== undefined) {
|
|
2255
|
+
await this.#appendEvent({ type: "candidate_blocked", taskId: task.taskId, workerId: handle?.id, data: { reason: `${reason}; the Decision Worker chose stop, so no repair round is sent` } });
|
|
2256
|
+
return false;
|
|
2257
|
+
}
|
|
2156
2258
|
if (this.#repairRound >= task.spec.maxRepairRounds) {
|
|
2157
2259
|
await this.#appendEvent({ type: "repair_round_exhausted", taskId: task.taskId, workerId: handle?.id, data: { maxRepairRounds: task.spec.maxRepairRounds, reason } });
|
|
2158
2260
|
await this.#appendEvent({ type: "candidate_blocked", taskId: task.taskId, workerId: handle?.id, data: { reason: `${reason}; automatic repair budget is exhausted` } });
|
|
@@ -2222,7 +2324,7 @@ export class Supervisor {
|
|
|
2222
2324
|
// A completed interactive task may keep its persistent session open for
|
|
2223
2325
|
// the operator instead of tearing it down; a stop requested mid-verify or
|
|
2224
2326
|
// a blocked/failed outcome always falls back to today's stop behavior.
|
|
2225
|
-
const keepOpen = !stopRequested && outcome === "completed" && result.ok
|
|
2327
|
+
const keepOpen = !stopRequested && this.#stopVerification === undefined && outcome === "completed" && result.ok
|
|
2226
2328
|
&& this.#keepWorkerOnCompletion && Boolean(this.#adapter.release) && Boolean(this.#handle);
|
|
2227
2329
|
let cleanupError: unknown;
|
|
2228
2330
|
let releasedInteractive = false;
|
|
@@ -2373,7 +2475,16 @@ export class Supervisor {
|
|
|
2373
2475
|
|
|
2374
2476
|
#armWatchdog(): void {
|
|
2375
2477
|
if (!this.#automation && this.#deadlineMs <= 0 && this.#noOutputTimeoutMs <= 0) return;
|
|
2376
|
-
|
|
2478
|
+
// One tick at a time: a tick queues behind #exclusive, so while a long
|
|
2479
|
+
// acceptance/Review run holds it, un-guarded ticks would pile up by the
|
|
2480
|
+
// thousand and then all run back to back.
|
|
2481
|
+
this.#watchdog = setInterval(() => {
|
|
2482
|
+
if (this.#watchdogTickPending) return;
|
|
2483
|
+
this.#watchdogTickPending = true;
|
|
2484
|
+
void this.#checkWatchdog()
|
|
2485
|
+
.catch(() => { /* lifecycle state is retained for the next explicit operation */ })
|
|
2486
|
+
.finally(() => { this.#watchdogTickPending = false; });
|
|
2487
|
+
}, 1_000);
|
|
2377
2488
|
this.#watchdog.unref();
|
|
2378
2489
|
}
|
|
2379
2490
|
|
|
@@ -2428,7 +2539,9 @@ export class Supervisor {
|
|
|
2428
2539
|
// for the grace period as well (an empty grace keeps the old immediate stop).
|
|
2429
2540
|
const reason = deadlineReached && elapsed >= this.#deadlineMs + this.#deadlineGraceMs
|
|
2430
2541
|
? "worker deadline exceeded"
|
|
2431
|
-
|
|
2542
|
+
// Under human takeover (including every recovered task until
|
|
2543
|
+
// resume-auto) an idle Worker is waiting for the operator, not stuck.
|
|
2544
|
+
: this.#machine.state !== "paused" && !this.#humanRequired && this.#noOutputTimeoutMs > 0 && now - lastOutputAt >= this.#noOutputTimeoutMs
|
|
2432
2545
|
? "worker produced no output before timeout"
|
|
2433
2546
|
: undefined;
|
|
2434
2547
|
if (!reason) {
|
|
@@ -2438,6 +2551,40 @@ export class Supervisor {
|
|
|
2438
2551
|
}
|
|
2439
2552
|
return;
|
|
2440
2553
|
}
|
|
2554
|
+
// An *idle* automatic Worker that stayed silent is not hung: it finished a
|
|
2555
|
+
// turn and is waiting (typically on background work that never came
|
|
2556
|
+
// back). Judge the work instead of killing it and discarding the chance
|
|
2557
|
+
// of a candidate; a Worker silent in the middle of a turn is still stopped.
|
|
2558
|
+
// A Worker that never completed a turn under this Supervisor (recovered,
|
|
2559
|
+
// or adopted) still reads `running`; classify it first, as the close-out does.
|
|
2560
|
+
if (reason === "worker produced no output before timeout" && this.#automation && this.#machine.state === "running" && !status.activeRequests) {
|
|
2561
|
+
await this.#pollInternal();
|
|
2562
|
+
// The Worker exited in between: the poll has already classified it.
|
|
2563
|
+
// As at the top of this function, a classified exit is verified here
|
|
2564
|
+
// too, in case its `exited` event (which normally drives it) was lost.
|
|
2565
|
+
const polled: string = this.#machine.state;
|
|
2566
|
+
if (polled !== "running" && polled !== "waiting") {
|
|
2567
|
+
if (polled === "verifying" && !this.#verificationAbortController) {
|
|
2568
|
+
try {
|
|
2569
|
+
await this.#verifyInternal();
|
|
2570
|
+
} catch (error) {
|
|
2571
|
+
await this.#appendEvent({ type: "worker_event_error", taskId, workerId, data: { error: safeMessage(error), eventType: "watchdog_verify" } }).catch(() => {});
|
|
2572
|
+
}
|
|
2573
|
+
}
|
|
2574
|
+
return;
|
|
2575
|
+
}
|
|
2576
|
+
}
|
|
2577
|
+
if (reason === "worker produced no output before timeout" && this.#automation && this.#machine.state === "waiting" && !status.activeRequests) {
|
|
2578
|
+
// A decision about this idle Worker is still being made (it may be
|
|
2579
|
+
// backing off a provider outage); let it land rather than race it —
|
|
2580
|
+
// but not forever: one that has not landed within another timeout
|
|
2581
|
+
// period never will.
|
|
2582
|
+
if (this.#pendingDecisionKey && now - this.#pendingDecisionSince < this.#noOutputTimeoutMs) return;
|
|
2583
|
+
this.#clearWaitTimer();
|
|
2584
|
+
await this.#appendEvent({ type: "worker_idle_timeout", taskId, workerId, data: { reason, action: "verify", noOutputTimeoutMs: this.#noOutputTimeoutMs } }).catch(() => {});
|
|
2585
|
+
await this.#startVerification(this.#handle, this.#lastTurnCompleted, "no-output timeout");
|
|
2586
|
+
return;
|
|
2587
|
+
}
|
|
2441
2588
|
// A close-out window that was skipped entirely (a task recovered past its
|
|
2442
2589
|
// budget) still leaves the deadline notice ahead of the stop in the log.
|
|
2443
2590
|
if (deadlineReached && this.#deadlineGraceMs > 0) await this.#noteDeadlineReached(elapsed).catch(() => {});
|
|
@@ -2486,7 +2633,17 @@ export class Supervisor {
|
|
|
2486
2633
|
/** The per-event context refresh sent to the Decision Worker before every notification or replay. */
|
|
2487
2634
|
#decisionContextPatch(): Partial<DecisionContext> {
|
|
2488
2635
|
const deadline = this.#deadlineContext();
|
|
2489
|
-
|
|
2636
|
+
const verification = this.#lastVerification;
|
|
2637
|
+
// The Worker's own repair prompt carries these; without them the Decision
|
|
2638
|
+
// Worker judges a repair turn only by Claude's claim that it is done.
|
|
2639
|
+
const lastVerification = verification ? {
|
|
2640
|
+
ok: verification.ok,
|
|
2641
|
+
...(this.#lastVerificationTurn !== undefined ? { atTurn: this.#lastVerificationTurn } : {}),
|
|
2642
|
+
failedChecks: verification.checks.filter((check) => check.check.required && !check.ok).map((check) => check.check.id).slice(0, 16),
|
|
2643
|
+
...(verification.review ? { reviewVerdict: verification.review.verdict } : {}),
|
|
2644
|
+
findings: bySeverity(verification.review?.findings ?? []).slice(0, 8).map((finding) => `${finding.id} [${finding.severity}] ${finding.message}`.slice(0, 200)),
|
|
2645
|
+
} : undefined;
|
|
2646
|
+
return { state: this.#machine.state, turn: this.#turn, repairRound: this.#repairRound, ...(deadline ? { deadline } : {}), ...(lastVerification ? { lastVerification } : {}) };
|
|
2490
2647
|
}
|
|
2491
2648
|
|
|
2492
2649
|
/**
|
|
@@ -2823,11 +2980,13 @@ function removeFlagWithValue(args: readonly string[], flag: string): string[] {
|
|
|
2823
2980
|
/**
|
|
2824
2981
|
* The deadline (in ms since the task started) that grants `extendMs` more
|
|
2825
2982
|
* from now: measured from the later of the current deadline and the present,
|
|
2826
|
-
* so extending an expired task by 30 minutes means 30 minutes from now
|
|
2827
|
-
*
|
|
2983
|
+
* so extending an expired task by 30 minutes means 30 minutes from now.
|
|
2984
|
+
* Extending by 0 means "close out now" for any task, expired or not: the
|
|
2985
|
+
* deadline lands on the present so the close-out opens at once.
|
|
2828
2986
|
*/
|
|
2829
2987
|
export function extendedDeadlineMs(currentDeadlineMs: number, elapsedMs: number, extendMs: number): number {
|
|
2830
|
-
|
|
2988
|
+
if (extendMs <= 0) return elapsedMs;
|
|
2989
|
+
return Math.max(currentDeadlineMs, elapsedMs) + extendMs;
|
|
2831
2990
|
}
|
|
2832
2991
|
|
|
2833
2992
|
function canRepairInPlace(adapter: WorkerAdapter): boolean {
|
|
@@ -2905,15 +3064,37 @@ function cancelledAcceptanceReport(reason: string): AcceptanceReport {
|
|
|
2905
3064
|
};
|
|
2906
3065
|
}
|
|
2907
3066
|
|
|
3067
|
+
function tailText(value: string, maxChars: number): string {
|
|
3068
|
+
return value.length <= maxChars ? value : `[…${value.length - maxChars} earlier characters omitted]\n${value.slice(-maxChars)}`;
|
|
3069
|
+
}
|
|
3070
|
+
|
|
3071
|
+
const RETRY_RESUME_MESSAGE = "Your previous turn stopped before finishing. Resume the task where you left off.";
|
|
3072
|
+
|
|
3073
|
+
/** Findings with the most severe first, keeping the Reviewer's order within a severity. */
|
|
3074
|
+
function bySeverity<T extends { severity: string }>(findings: readonly T[]): T[] {
|
|
3075
|
+
const rank = (severity: string) => { const at = ["P0", "P1", "P2", "P3"].indexOf(severity); return at === -1 ? 4 : at; };
|
|
3076
|
+
return [...findings].sort((a, b) => rank(a.severity) - rank(b.severity));
|
|
3077
|
+
}
|
|
3078
|
+
|
|
2908
3079
|
function repairInstruction(result: AcceptanceReport, reason: string, round: number): string {
|
|
3080
|
+
// Each check keeps the *end* of its output: that is where test runners
|
|
3081
|
+
// print the failure summary, while the head is usually progress noise.
|
|
2909
3082
|
const failedChecks = result.checks
|
|
2910
3083
|
.filter((check) => check.check.required && !check.ok)
|
|
2911
|
-
.map((check) => `${check.check.id}: ${check.output}`)
|
|
3084
|
+
.map((check) => `${check.check.id}: ${tailText(check.output, 4_000)}`)
|
|
2912
3085
|
.join("\n");
|
|
2913
|
-
|
|
2914
|
-
|
|
3086
|
+
// Findings carry their location and evidence: they are what points the
|
|
3087
|
+
// Worker at the fix. They go first so the 16 KB bound never cuts them, and
|
|
3088
|
+
// the most severe lead, so long lesser findings cannot crowd out a P0.
|
|
3089
|
+
const findings = bySeverity(result.review?.findings ?? [])
|
|
3090
|
+
.map((finding) => {
|
|
3091
|
+
const location = finding.file ? ` ${finding.file}${finding.line ? `:${finding.line}` : ""}` : "";
|
|
3092
|
+
const fix = finding.requiredFix ? `; required fix: ${finding.requiredFix}` : "";
|
|
3093
|
+
const evidence = finding.evidence ? `; evidence: ${tailText(finding.evidence, 1_000)}` : "";
|
|
3094
|
+
return `${finding.id} [${finding.severity}]${location}: ${finding.message}${fix}${evidence}`;
|
|
3095
|
+
})
|
|
2915
3096
|
.join("\n") ?? "";
|
|
2916
|
-
const evidence = [
|
|
3097
|
+
const evidence = [findings ? `Reviewer findings:\n${findings}` : "", failedChecks ? `Failed acceptance checks:\n${failedChecks}` : ""].filter(Boolean).join("\n\n");
|
|
2917
3098
|
const commitRequirement = reason.includes("local commit") || reason.includes("uncommitted")
|
|
2918
3099
|
? "Before reporting completion, inspect the final diff, run the relevant checks, and create a local git commit on the task branch. Do not push, merge, publish, or modify main/integration."
|
|
2919
3100
|
: "";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
2
|
import { spawn, type ChildProcess } from "node:child_process";
|
|
3
3
|
import { constants as fsConstants, readFileSync } from "node:fs";
|
|
4
|
-
import { access, mkdir, readFile, readdir, rmdir, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { access, mkdir, readFile, readdir, rmdir, stat, statfs, writeFile } from "node:fs/promises";
|
|
5
5
|
import { delimiter, isAbsolute, join } from "node:path";
|
|
6
6
|
import type {
|
|
7
7
|
WorkerAdapter,
|
|
@@ -59,6 +59,8 @@ interface ProcessRecord {
|
|
|
59
59
|
seenResultIds: Set<string>;
|
|
60
60
|
seenPermissionRequestIds: Set<string>;
|
|
61
61
|
protocolBuffer: string;
|
|
62
|
+
/** Byte length of protocolBuffer, kept incrementally so a long line is not re-measured per chunk. */
|
|
63
|
+
protocolBufferBytes: number;
|
|
62
64
|
discardProtocolLine: boolean;
|
|
63
65
|
exitCode?: number | null;
|
|
64
66
|
signal?: NodeJS.Signals;
|
|
@@ -114,7 +116,11 @@ export class ProcessWorkerAdapter implements WorkerAdapter {
|
|
|
114
116
|
this.#killGraceMs = boundedDelay(options.killGraceMs ?? 500);
|
|
115
117
|
this.#maxOutputChunks = boundedPositiveInteger(options.maxOutputChunks ?? 10_000, "maxOutputChunks");
|
|
116
118
|
this.#maxOutputBytes = boundedPositiveInteger(options.maxOutputBytes ?? 8 * 1024 * 1024, "maxOutputBytes");
|
|
117
|
-
|
|
119
|
+
// One JSONL record, not the stream: a permission request for a Write of a
|
|
120
|
+
// large file, or a long final result, is a single line. A dropped
|
|
121
|
+
// control_request leaves Claude waiting forever for its answer and a
|
|
122
|
+
// dropped result never ends the turn, so the bound only guards memory.
|
|
123
|
+
this.#maxProtocolBufferBytes = boundedPositiveInteger(options.maxProtocolBufferBytes ?? 32 * 1024 * 1024, "maxProtocolBufferBytes");
|
|
118
124
|
this.#inputWriteTimeoutMs = boundedDelay(options.inputWriteTimeoutMs ?? 10_000);
|
|
119
125
|
this.#cgroupMode = options.cgroupMode ?? "auto";
|
|
120
126
|
this.#cgroupParentPath = options.cgroupParentPath;
|
|
@@ -258,6 +264,7 @@ export class ProcessWorkerAdapter implements WorkerAdapter {
|
|
|
258
264
|
seenResultIds: new Set(),
|
|
259
265
|
seenPermissionRequestIds: new Set(),
|
|
260
266
|
protocolBuffer: "",
|
|
267
|
+
protocolBufferBytes: 0,
|
|
261
268
|
discardProtocolLine: false,
|
|
262
269
|
exited,
|
|
263
270
|
resolveExit,
|
|
@@ -300,6 +307,12 @@ export class ProcessWorkerAdapter implements WorkerAdapter {
|
|
|
300
307
|
this.#emit(record, { type: "output", handle: record.handle, chunk: outputChunk });
|
|
301
308
|
if (stream === "stdout" && this.#mode === "claude-jsonl") this.#observeJsonl(record, text);
|
|
302
309
|
};
|
|
310
|
+
// A stream decoder keeps a multi-byte character that straddles two pipe
|
|
311
|
+
// reads intact; String(buffer) per chunk turned it into U+FFFD, and a
|
|
312
|
+
// permission `updatedInput` echoed back from that text then wrote the
|
|
313
|
+
// corruption into the Worker's files.
|
|
314
|
+
child.stdout?.setEncoding("utf8");
|
|
315
|
+
child.stderr?.setEncoding("utf8");
|
|
303
316
|
child.stdout?.on("data", capture("stdout"));
|
|
304
317
|
child.stderr?.on("data", capture("stderr"));
|
|
305
318
|
child.stdin?.on("error", (error) => {
|
|
@@ -602,28 +615,33 @@ export class ProcessWorkerAdapter implements WorkerAdapter {
|
|
|
602
615
|
if (newline < 0) return;
|
|
603
616
|
record.discardProtocolLine = false;
|
|
604
617
|
record.protocolBuffer = "";
|
|
618
|
+
record.protocolBufferBytes = 0;
|
|
605
619
|
offset = newline + 1;
|
|
606
620
|
continue;
|
|
607
621
|
}
|
|
608
622
|
const newline = chunk.indexOf("\n", offset);
|
|
609
623
|
if (newline < 0) {
|
|
610
624
|
const tail = chunk.slice(offset);
|
|
611
|
-
|
|
625
|
+
const tailBytes = Buffer.byteLength(tail, "utf8");
|
|
626
|
+
if (record.protocolBufferBytes + tailBytes > this.#maxProtocolBufferBytes) {
|
|
612
627
|
record.protocolBuffer = "";
|
|
628
|
+
record.protocolBufferBytes = 0;
|
|
613
629
|
record.discardProtocolLine = true;
|
|
614
630
|
record.outputTruncated = true;
|
|
615
631
|
} else {
|
|
616
632
|
record.protocolBuffer += tail;
|
|
633
|
+
record.protocolBufferBytes += tailBytes;
|
|
617
634
|
}
|
|
618
635
|
return;
|
|
619
636
|
}
|
|
620
637
|
const linePart = chunk.slice(offset, newline);
|
|
621
|
-
if (
|
|
638
|
+
if (record.protocolBufferBytes + Buffer.byteLength(linePart, "utf8") > this.#maxProtocolBufferBytes) {
|
|
622
639
|
record.outputTruncated = true;
|
|
623
640
|
} else {
|
|
624
641
|
this.#processJsonlLine(record, `${record.protocolBuffer}${linePart}`.trim());
|
|
625
642
|
}
|
|
626
643
|
record.protocolBuffer = "";
|
|
644
|
+
record.protocolBufferBytes = 0;
|
|
627
645
|
offset = newline + 1;
|
|
628
646
|
}
|
|
629
647
|
}
|
|
@@ -983,10 +1001,24 @@ async function assertExecutable(command: string, pathValue: string | undefined):
|
|
|
983
1001
|
throw new Error(`worker executable preflight failed (ENOENT): ${command}`);
|
|
984
1002
|
}
|
|
985
1003
|
|
|
1004
|
+
/** statfs(2) f_type of a cgroup v2 mount (CGROUP2_SUPER_MAGIC). */
|
|
1005
|
+
const CGROUP2_SUPER_MAGIC = 0x63677270;
|
|
1006
|
+
|
|
986
1007
|
export async function currentCgroupPath(): Promise<string> {
|
|
987
1008
|
const contents = await readFile("/proc/self/cgroup", "utf8");
|
|
988
1009
|
const match = contents.match(/^0::([^\n]*)$/mu);
|
|
989
1010
|
if (!match) throw new Error("cgroup v2 is not active");
|
|
1011
|
+
// A hybrid (v1 + v2) host also lists a `0::` line, but its unified
|
|
1012
|
+
// hierarchy is not mounted at /sys/fs/cgroup — that is a tmpfs holding the
|
|
1013
|
+
// v1 controllers. Creating a "cgroup" there only makes an ordinary directory
|
|
1014
|
+
// with no controls, so require the real cgroup2 filesystem first.
|
|
1015
|
+
let type: number;
|
|
1016
|
+
try {
|
|
1017
|
+
type = (await statfs("/sys/fs/cgroup")).type;
|
|
1018
|
+
} catch (error) {
|
|
1019
|
+
throw new Error(`cgroup v2 is not mounted at /sys/fs/cgroup: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
1020
|
+
}
|
|
1021
|
+
if (type !== CGROUP2_SUPER_MAGIC) throw new Error("cgroup v2 is not mounted at /sys/fs/cgroup (a cgroup v1 or hybrid host)");
|
|
990
1022
|
// /proc/self/cgroup uses the same escaped component spelling as the cgroup
|
|
991
1023
|
// filesystem (for example, a literal `\\x2d` in a systemd scope name).
|
|
992
1024
|
return `/sys/fs/cgroup${match[1]}`;
|
|
@@ -1043,7 +1075,10 @@ export async function preflightCgroupContainment(parentPath?: string): Promise<v
|
|
|
1043
1075
|
throw new Error(`required cgroup preflight failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
1044
1076
|
} finally {
|
|
1045
1077
|
if (probePath) {
|
|
1046
|
-
|
|
1078
|
+
// O_WRONLY without O_CREAT: kill through an existing control file only.
|
|
1079
|
+
// The default `w` flag would create a regular file in a directory that
|
|
1080
|
+
// turned out not to be a cgroup, and that file then blocks its rmdir.
|
|
1081
|
+
try { await writeFile(`${probePath}/cgroup.kill`, "1" + String.fromCharCode(10), { flag: fsConstants.O_WRONLY }); } catch {}
|
|
1047
1082
|
}
|
|
1048
1083
|
const child = probeChild;
|
|
1049
1084
|
if (child && child.exitCode === null) {
|