pi-claude-supervisor 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -2
- package/README.cn.md +3 -3
- package/README.md +1 -0
- package/docs/architecture.md +39 -7
- package/docs/releasing.md +2 -1
- package/docs/testing.md +15 -6
- package/docs/transport-spike-2026-09-12.md +8 -8
- package/package.json +2 -1
- package/src/config.ts +2 -0
- package/src/cwd-lease.ts +345 -0
- package/src/index.ts +188 -38
- package/src/supervisor.ts +100 -22
- package/src/types.ts +22 -1
- package/src/worker/process-adapter.ts +98 -26
- package/src/worker/tmux-adapter.ts +61 -25
package/src/index.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
2
3
|
import { homedir } from "node:os";
|
|
3
|
-
import {
|
|
4
|
+
import { join } from "node:path";
|
|
4
5
|
import { realpath } from "node:fs/promises";
|
|
5
6
|
import { EventLog } from "./events.ts";
|
|
6
7
|
import { redactSensitive } from "./redaction.ts";
|
|
@@ -11,6 +12,7 @@ import { evaluateCommand } from "./policy.ts";
|
|
|
11
12
|
import { HumanWebhookNotifier } from "./notifications.ts";
|
|
12
13
|
import { loadSupervisorEnvironment } from "./config.ts";
|
|
13
14
|
import { DecisionSessionStore, type DecisionSessionRecord } from "./decision-session-store.ts";
|
|
15
|
+
import { CwdLeaseStore, type CwdLeaseHandle, pathsOverlap, workerIdentity } from "./cwd-lease.ts";
|
|
14
16
|
|
|
15
17
|
/**
|
|
16
18
|
* Pi Claude Supervisor.
|
|
@@ -25,15 +27,23 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
25
27
|
const stateDir = process.env.PI_CLAUDE_SUPERVISOR_STATE_DIR ?? join(homedir(), ".pi", "agent", "claude-supervisor");
|
|
26
28
|
const configuredTransport = process.env.PI_CLAUDE_SUPERVISOR_TRANSPORT;
|
|
27
29
|
const transport = configuredTransport ?? (automation ? "jsonl" : "process-pipe");
|
|
30
|
+
const cgroupMode = process.env.PI_CLAUDE_SUPERVISOR_CGROUP_MODE ?? "auto";
|
|
28
31
|
if (!(["process-pipe", "jsonl", "tmux"] as string[]).includes(transport)) {
|
|
29
32
|
throw new Error(`Unsupported PI_CLAUDE_SUPERVISOR_TRANSPORT: ${transport}; expected process-pipe, jsonl, or tmux`);
|
|
30
33
|
}
|
|
34
|
+
if (!["off", "auto", "required"].includes(cgroupMode)) {
|
|
35
|
+
throw new Error(`Unsupported PI_CLAUDE_SUPERVISOR_CGROUP_MODE: ${cgroupMode}; expected off, auto, or required`);
|
|
36
|
+
}
|
|
37
|
+
if (transport === "tmux" && cgroupMode === "required") {
|
|
38
|
+
throw new Error("PI_CLAUDE_SUPERVISOR_CGROUP_MODE=required is unsupported with tmux; use process-pipe/jsonl or set cgroup mode to auto/off");
|
|
39
|
+
}
|
|
31
40
|
const adapter = transport === "tmux"
|
|
32
41
|
? new TmuxWorkerAdapter({ stateDir })
|
|
33
42
|
: new ProcessWorkerAdapter({
|
|
34
43
|
// Automatic decisions require Claude's structured event stream. The pipe
|
|
35
44
|
// transport remains available for manual/compatibility sessions.
|
|
36
45
|
mode: automation || transport === "jsonl" ? "claude-jsonl" : "process-pipe",
|
|
46
|
+
cgroupMode: cgroupMode as "off" | "auto" | "required",
|
|
37
47
|
});
|
|
38
48
|
const humanWebhook = new HumanWebhookNotifier({
|
|
39
49
|
url: process.env.PI_CLAUDE_SUPERVISOR_HUMAN_WEBHOOK_URL,
|
|
@@ -42,7 +52,10 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
42
52
|
});
|
|
43
53
|
const events = new EventLog(join(stateDir, "events.jsonl"));
|
|
44
54
|
const decisionStore = new DecisionSessionStore(join(stateDir, "decision-sessions"));
|
|
55
|
+
const cwdLeaseStore = new CwdLeaseStore(process.env.PI_CLAUDE_SUPERVISOR_CWD_LEASE_DIR ?? join(homedir(), ".pi", "agent", "claude-supervisor", "cwd-leases"));
|
|
45
56
|
const sessions = new Map<string, Supervisor>();
|
|
57
|
+
const cwdLeases = new Map<string, CwdLeaseHandle>();
|
|
58
|
+
const cleanupRequiredTasks = new Set<string>();
|
|
46
59
|
const reservedCwds = new Map<string, string>();
|
|
47
60
|
const pendingCwds = new Set<string>();
|
|
48
61
|
const pendingStarts = new Set<Promise<void>>();
|
|
@@ -56,30 +69,54 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
56
69
|
};
|
|
57
70
|
const activeSessions = () => [...sessions.entries()].filter(([, session]) =>
|
|
58
71
|
["starting", "running", "waiting", "paused"].includes(session.state));
|
|
72
|
+
const releaseLease = async (taskId: string): Promise<boolean> => {
|
|
73
|
+
const lease = cwdLeases.get(taskId);
|
|
74
|
+
if (!lease) return false;
|
|
75
|
+
try {
|
|
76
|
+
await lease.release();
|
|
77
|
+
cwdLeases.delete(taskId);
|
|
78
|
+
return true;
|
|
79
|
+
} catch (error) {
|
|
80
|
+
console.error(`pi-claude-supervisor cwd lease release failed: ${redactText(error instanceof Error ? error.message : String(error))}`);
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
59
84
|
const releaseSettledReservations = async (): Promise<void> => {
|
|
60
85
|
for (const [taskId, session] of sessions) {
|
|
61
86
|
if (!["completed", "stopped", "failed"].includes(session.state)) continue;
|
|
62
87
|
if (!session.handle) {
|
|
63
|
-
|
|
88
|
+
if (await releaseLease(taskId)) {
|
|
89
|
+
reservedCwds.delete(taskId);
|
|
90
|
+
cleanupRequiredTasks.delete(taskId);
|
|
91
|
+
}
|
|
64
92
|
continue;
|
|
65
93
|
}
|
|
66
94
|
try {
|
|
67
95
|
const status = await adapter.getStatus(session.handle);
|
|
68
|
-
if (!status.running && status.processGroupCleaned === true && !status.cleanupError
|
|
96
|
+
if (!status.running && status.processGroupCleaned === true && !status.cleanupError && (!status.cgroupError || status.cgroupRequired === false)) {
|
|
97
|
+
if (await releaseLease(taskId)) {
|
|
98
|
+
reservedCwds.delete(taskId);
|
|
99
|
+
cleanupRequiredTasks.delete(taskId);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
69
102
|
} catch {
|
|
70
103
|
// Keep the reservation when cleanup status cannot be confirmed.
|
|
71
104
|
}
|
|
72
105
|
}
|
|
73
106
|
};
|
|
74
|
-
const stopSession = async (session: Supervisor, reason: string): Promise<void> => {
|
|
107
|
+
const stopSession = async (session: Supervisor, reason: string, releasePersistent = false): Promise<void> => {
|
|
75
108
|
const handle = session.handle;
|
|
76
109
|
const persistent = adapter.capabilities().persistentSession && Boolean(handle);
|
|
110
|
+
const taskId = session.task?.taskId;
|
|
77
111
|
const adoptedPersistent = persistent && handle?.ownership === "adopted";
|
|
78
112
|
const healthyPersistent = persistent && !["failed", "completed", "stopped"].includes(session.state);
|
|
79
|
-
|
|
113
|
+
const cleanupRequired = taskId ? cleanupRequiredTasks.has(taskId) : false;
|
|
114
|
+
const markCleanupRequired = () => { if (taskId) cleanupRequiredTasks.add(taskId); };
|
|
115
|
+
if (adoptedPersistent || (releasePersistent && healthyPersistent && !cleanupRequired)) {
|
|
80
116
|
await session.release(reason);
|
|
81
117
|
return;
|
|
82
118
|
}
|
|
119
|
+
if (!handle && ["failed", "completed", "stopped"].includes(session.state)) return;
|
|
83
120
|
let lifecycleError: unknown;
|
|
84
121
|
try {
|
|
85
122
|
await session.stop(reason);
|
|
@@ -88,7 +125,10 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
88
125
|
}
|
|
89
126
|
|
|
90
127
|
if (!handle) {
|
|
91
|
-
if (lifecycleError)
|
|
128
|
+
if (lifecycleError) {
|
|
129
|
+
markCleanupRequired();
|
|
130
|
+
throw lifecycleError;
|
|
131
|
+
}
|
|
92
132
|
return;
|
|
93
133
|
}
|
|
94
134
|
|
|
@@ -97,8 +137,12 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
97
137
|
while (Date.now() <= deadline) {
|
|
98
138
|
try {
|
|
99
139
|
const status = await adapter.getStatus(handle);
|
|
100
|
-
if (!status.running && status.processGroupCleaned === true && !status.cleanupError) {
|
|
101
|
-
if (lifecycleError)
|
|
140
|
+
if (!status.running && status.processGroupCleaned === true && !status.cleanupError && (!status.cgroupError || status.cgroupRequired === false)) {
|
|
141
|
+
if (lifecycleError) {
|
|
142
|
+
markCleanupRequired();
|
|
143
|
+
throw lifecycleError;
|
|
144
|
+
}
|
|
145
|
+
if (taskId) cleanupRequiredTasks.delete(taskId);
|
|
102
146
|
return;
|
|
103
147
|
}
|
|
104
148
|
} catch (error) {
|
|
@@ -112,6 +156,7 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
112
156
|
await delay(25);
|
|
113
157
|
}
|
|
114
158
|
|
|
159
|
+
markCleanupRequired();
|
|
115
160
|
if (lifecycleError) throw lifecycleError;
|
|
116
161
|
throw cleanupError instanceof Error
|
|
117
162
|
? cleanupError
|
|
@@ -152,6 +197,15 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
152
197
|
if (reservedPaths.some((reserved) => pathsOverlap(reserved, cwdKey))) {
|
|
153
198
|
throw new Error("An active, starting, or unreaped worker uses an overlapping cwd; use a separate worktree for concurrent sessions");
|
|
154
199
|
}
|
|
200
|
+
const taskId = randomUUID();
|
|
201
|
+
const lease = await cwdLeaseStore.acquire(cwdKey, taskId, adapter.capabilities().transport, tmuxSession
|
|
202
|
+
? { handoff: { sessionName: tmuxSession, tmuxSocket: process.env.PI_CLAUDE_SUPERVISOR_TMUX_SOCKET } }
|
|
203
|
+
: {});
|
|
204
|
+
cwdLeases.set(taskId, lease);
|
|
205
|
+
if (shuttingDown) {
|
|
206
|
+
await releaseLease(taskId);
|
|
207
|
+
throw new Error("Pi session is shutting down");
|
|
208
|
+
}
|
|
155
209
|
pendingCwds.add(cwdKey);
|
|
156
210
|
const session = new Supervisor(adapter, events, {
|
|
157
211
|
onHumanRequired: async (notice) => {
|
|
@@ -168,11 +222,13 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
168
222
|
},
|
|
169
223
|
});
|
|
170
224
|
pendingStartSessions.add(session);
|
|
225
|
+
let startupCleanupCompleted = false;
|
|
171
226
|
const startOperation = (async () => {
|
|
172
227
|
try {
|
|
173
228
|
const handle = await session.start({
|
|
229
|
+
taskId,
|
|
174
230
|
task,
|
|
175
|
-
cwd:
|
|
231
|
+
cwd: cwdKey,
|
|
176
232
|
command,
|
|
177
233
|
args: workerArgs,
|
|
178
234
|
env: selectedWorkerEnvironment(),
|
|
@@ -180,6 +236,14 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
180
236
|
automation,
|
|
181
237
|
tmuxSession,
|
|
182
238
|
tmuxSocket: process.env.PI_CLAUDE_SUPERVISOR_TMUX_SOCKET,
|
|
239
|
+
tmuxExpectedIdentity: tmuxSession && lease.record.worker ? {
|
|
240
|
+
pid: lease.record.worker.pid,
|
|
241
|
+
startTime: lease.record.worker.startTime,
|
|
242
|
+
tmuxTarget: lease.record.worker.tmuxTarget,
|
|
243
|
+
tmuxPaneId: lease.record.worker.tmuxPaneId,
|
|
244
|
+
paneStartTime: lease.record.worker.paneStartTime,
|
|
245
|
+
paneCommand: lease.record.worker.paneCommand,
|
|
246
|
+
} : undefined,
|
|
183
247
|
sendInitialInput: !tmuxSession,
|
|
184
248
|
decisionSessionDir: decisionStore.directory,
|
|
185
249
|
onDecisionSessionReady: (info) => decisionStore.save({
|
|
@@ -200,37 +264,70 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
200
264
|
onDecisionSessionProgress: (info) => decisionStore.update(info.taskId, { turn: info.turn }),
|
|
201
265
|
onDecisionSessionClosed: (taskId) => decisionStore.close(taskId),
|
|
202
266
|
});
|
|
203
|
-
const
|
|
204
|
-
if (!
|
|
267
|
+
const startedTaskId = session.task?.taskId;
|
|
268
|
+
if (!startedTaskId) throw new Error("worker started without a task id");
|
|
269
|
+
try {
|
|
270
|
+
await lease.updateWorker({
|
|
271
|
+
transport: adapter.capabilities().transport,
|
|
272
|
+
...(await workerIdentity(handle)),
|
|
273
|
+
sessionName: handle.sessionName,
|
|
274
|
+
tmuxSocket: handle.tmuxSocket,
|
|
275
|
+
ownership: handle.ownership,
|
|
276
|
+
});
|
|
277
|
+
} catch (error) {
|
|
278
|
+
const registrationError = error instanceof Error ? error : new Error(String(error));
|
|
279
|
+
if (handle.ownership !== "adopted") {
|
|
280
|
+
try {
|
|
281
|
+
await stopSession(session, "cwd lease metadata registration failed");
|
|
282
|
+
startupCleanupCompleted = await releaseLease(taskId);
|
|
283
|
+
} catch (cleanupError) {
|
|
284
|
+
const message = cleanupError instanceof Error ? cleanupError.message : String(cleanupError);
|
|
285
|
+
registrationError.message = `${registrationError.message}; worker cleanup failed: ${message}`;
|
|
286
|
+
Object.defineProperty(registrationError, "workerCleanupRequired", { value: true, enumerable: false });
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
throw registrationError;
|
|
290
|
+
}
|
|
205
291
|
// Register immediately after spawn so shutdown can retry cleanup if
|
|
206
292
|
// the first stop attempt fails.
|
|
207
|
-
sessions.set(
|
|
208
|
-
reservedCwds.set(
|
|
209
|
-
activeTaskId =
|
|
293
|
+
sessions.set(startedTaskId, session);
|
|
294
|
+
reservedCwds.set(startedTaskId, cwdKey);
|
|
295
|
+
activeTaskId = startedTaskId;
|
|
210
296
|
if (shuttingDown) {
|
|
211
297
|
try {
|
|
212
|
-
await stopSession(session, "Pi session shutdown during worker start");
|
|
213
|
-
sessions.delete(
|
|
214
|
-
reservedCwds.delete(
|
|
298
|
+
await stopSession(session, "Pi session shutdown during worker start", true);
|
|
299
|
+
sessions.delete(startedTaskId);
|
|
300
|
+
reservedCwds.delete(startedTaskId);
|
|
215
301
|
} finally {
|
|
216
302
|
if (session.state !== "stopped") {
|
|
217
303
|
// Keep the session registered for the shutdown retry below.
|
|
218
|
-
sessions.set(
|
|
219
|
-
reservedCwds.set(
|
|
304
|
+
sessions.set(startedTaskId, session);
|
|
305
|
+
reservedCwds.set(startedTaskId, cwdKey);
|
|
220
306
|
}
|
|
221
307
|
}
|
|
222
308
|
throw new Error("Pi session shut down during worker start");
|
|
223
309
|
}
|
|
224
310
|
const attach = handle.sessionName ? ` attach=${attachCommand(handle)}` : "";
|
|
225
|
-
message = `${tmuxSession ? "Tmux worker adopted" : "Worker started"}: task=${
|
|
311
|
+
message = `${tmuxSession ? "Tmux worker adopted" : "Worker started"}: task=${startedTaskId} worker=${handle.id} (pid ${handle.pid ?? "unknown"}); transport=${adapter.capabilities().transport}${attach}`;
|
|
226
312
|
} catch (error) {
|
|
227
313
|
// Register failed starts before the promise settles, so shutdown
|
|
228
314
|
// cannot snapshot sessions before a returned handle is retained.
|
|
229
315
|
const failedTaskId = session.task?.taskId;
|
|
230
|
-
|
|
316
|
+
const cleanupRequired = requiresWorkerCleanup(error);
|
|
317
|
+
if (cleanupRequired && failedTaskId) cleanupRequiredTasks.add(failedTaskId);
|
|
318
|
+
const retainHandle = !startupCleanupCompleted && failedTaskId && session.handle
|
|
319
|
+
&& (cleanupRequired || session.handle.ownership === "adopted");
|
|
320
|
+
if (retainHandle) {
|
|
231
321
|
sessions.set(failedTaskId, session);
|
|
232
322
|
reservedCwds.set(failedTaskId, cwdKey);
|
|
233
323
|
activeTaskId = failedTaskId;
|
|
324
|
+
} else if (!cleanupRequired && !startupCleanupCompleted) {
|
|
325
|
+
const released = await releaseLease(taskId);
|
|
326
|
+
if (!released && failedTaskId) {
|
|
327
|
+
sessions.set(failedTaskId, session);
|
|
328
|
+
reservedCwds.set(failedTaskId, cwdKey);
|
|
329
|
+
activeTaskId = failedTaskId;
|
|
330
|
+
}
|
|
234
331
|
}
|
|
235
332
|
throw error;
|
|
236
333
|
}
|
|
@@ -242,10 +339,21 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
242
339
|
// Preserve a failed startup in the registry when the adapter
|
|
243
340
|
// returned a handle but lifecycle/event setup failed.
|
|
244
341
|
const failedTaskId = session.task?.taskId;
|
|
245
|
-
|
|
342
|
+
const cleanupRequired = requiresWorkerCleanup(error);
|
|
343
|
+
if (cleanupRequired && failedTaskId) cleanupRequiredTasks.add(failedTaskId);
|
|
344
|
+
const retainHandle = !startupCleanupCompleted && failedTaskId && session.handle
|
|
345
|
+
&& (cleanupRequired || session.handle.ownership === "adopted");
|
|
346
|
+
if (retainHandle) {
|
|
246
347
|
sessions.set(failedTaskId, session);
|
|
247
348
|
reservedCwds.set(failedTaskId, cwdKey);
|
|
248
349
|
activeTaskId = failedTaskId;
|
|
350
|
+
} else if (!cleanupRequired && !startupCleanupCompleted) {
|
|
351
|
+
const released = await releaseLease(taskId);
|
|
352
|
+
if (!released && failedTaskId) {
|
|
353
|
+
sessions.set(failedTaskId, session);
|
|
354
|
+
reservedCwds.set(failedTaskId, cwdKey);
|
|
355
|
+
activeTaskId = failedTaskId;
|
|
356
|
+
}
|
|
249
357
|
}
|
|
250
358
|
throw error;
|
|
251
359
|
} finally {
|
|
@@ -282,6 +390,12 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
282
390
|
if (!approved) throw new Error("Worker command not approved");
|
|
283
391
|
approval = { actor: "human", reason: policy.reason };
|
|
284
392
|
}
|
|
393
|
+
const lease = await cwdLeaseStore.acquire(cwdKey, record.taskId, adapter.capabilities().transport);
|
|
394
|
+
cwdLeases.set(record.taskId, lease);
|
|
395
|
+
if (shuttingDown) {
|
|
396
|
+
await releaseLease(record.taskId);
|
|
397
|
+
throw new Error("Pi session is shutting down");
|
|
398
|
+
}
|
|
285
399
|
pendingCwds.add(cwdKey);
|
|
286
400
|
const session = new Supervisor(adapter, events, {
|
|
287
401
|
onHumanRequired: async (notice) => {
|
|
@@ -295,6 +409,7 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
295
409
|
},
|
|
296
410
|
});
|
|
297
411
|
pendingStartSessions.add(session);
|
|
412
|
+
let recoveryCleanupCompleted = false;
|
|
298
413
|
const recoveryOperation = (async () => {
|
|
299
414
|
try {
|
|
300
415
|
const handle = await session.start({
|
|
@@ -305,7 +420,7 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
305
420
|
task: record.task,
|
|
306
421
|
initialInput: "",
|
|
307
422
|
sendInitialInput: false,
|
|
308
|
-
cwd:
|
|
423
|
+
cwd: cwdKey,
|
|
309
424
|
command: record.command,
|
|
310
425
|
args: record.args,
|
|
311
426
|
env: selectedWorkerEnvironment(),
|
|
@@ -336,20 +451,51 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
336
451
|
onDecisionSessionProgress: (info) => decisionStore.update(info.taskId, { turn: info.turn }),
|
|
337
452
|
onDecisionSessionClosed: (closedTaskId) => decisionStore.close(closedTaskId),
|
|
338
453
|
});
|
|
454
|
+
try {
|
|
455
|
+
await lease.updateWorker({
|
|
456
|
+
transport: adapter.capabilities().transport,
|
|
457
|
+
...(await workerIdentity(handle)),
|
|
458
|
+
sessionName: handle.sessionName,
|
|
459
|
+
tmuxSocket: handle.tmuxSocket,
|
|
460
|
+
ownership: handle.ownership,
|
|
461
|
+
});
|
|
462
|
+
} catch (error) {
|
|
463
|
+
const registrationError = error instanceof Error ? error : new Error(String(error));
|
|
464
|
+
try {
|
|
465
|
+
await stopSession(session, "cwd lease metadata registration failed");
|
|
466
|
+
recoveryCleanupCompleted = await releaseLease(record.taskId);
|
|
467
|
+
} catch (cleanupError) {
|
|
468
|
+
const message = cleanupError instanceof Error ? cleanupError.message : String(cleanupError);
|
|
469
|
+
registrationError.message = `${registrationError.message}; worker cleanup failed: ${message}`;
|
|
470
|
+
Object.defineProperty(registrationError, "workerCleanupRequired", { value: true, enumerable: false });
|
|
471
|
+
}
|
|
472
|
+
throw registrationError;
|
|
473
|
+
}
|
|
339
474
|
sessions.set(record.taskId, session);
|
|
340
475
|
reservedCwds.set(record.taskId, cwdKey);
|
|
341
476
|
activeTaskId = record.taskId;
|
|
342
477
|
await session.takeover();
|
|
343
478
|
if (shuttingDown) {
|
|
344
|
-
await stopSession(session, "Pi session shutdown during recovery");
|
|
479
|
+
await stopSession(session, "Pi session shutdown during recovery", true);
|
|
345
480
|
throw new Error("Pi session shut down during recovery");
|
|
346
481
|
}
|
|
347
482
|
message = `Worker recovered idle: task=${record.taskId} worker=${handle.id}; original task was not replayed; send an explicit continuation, then use resume-auto`;
|
|
348
483
|
} catch (error) {
|
|
349
|
-
|
|
484
|
+
const cleanupRequired = requiresWorkerCleanup(error);
|
|
485
|
+
if (cleanupRequired) cleanupRequiredTasks.add(record.taskId);
|
|
486
|
+
const retainHandle = !recoveryCleanupCompleted && session.handle
|
|
487
|
+
&& (cleanupRequired || session.handle.ownership === "adopted");
|
|
488
|
+
if (retainHandle) {
|
|
350
489
|
sessions.set(record.taskId, session);
|
|
351
490
|
reservedCwds.set(record.taskId, cwdKey);
|
|
352
491
|
activeTaskId = record.taskId;
|
|
492
|
+
} else if (!cleanupRequired && !recoveryCleanupCompleted) {
|
|
493
|
+
const released = await releaseLease(record.taskId);
|
|
494
|
+
if (!released) {
|
|
495
|
+
sessions.set(record.taskId, session);
|
|
496
|
+
reservedCwds.set(record.taskId, cwdKey);
|
|
497
|
+
activeTaskId = record.taskId;
|
|
498
|
+
}
|
|
353
499
|
}
|
|
354
500
|
throw error;
|
|
355
501
|
}
|
|
@@ -395,9 +541,12 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
395
541
|
} else if (operation === "resume") {
|
|
396
542
|
await session.resume(); message = `Worker resumed: ${sessionId}.`;
|
|
397
543
|
} else if (operation === "stop") {
|
|
398
|
-
await session
|
|
544
|
+
await stopSession(session, remaining.join(" ") || "human requested stop");
|
|
545
|
+
await releaseSettledReservations();
|
|
546
|
+
message = `Worker stopped: ${sessionId}.`;
|
|
399
547
|
} else if (operation === "verify") {
|
|
400
548
|
const result = await session.verify();
|
|
549
|
+
await releaseSettledReservations();
|
|
401
550
|
message = `${result.ok ? "Verification passed" : "Verification failed"}: ${result.command}\n${result.output}`.trim();
|
|
402
551
|
} else if (operation === "approve") {
|
|
403
552
|
const behavior = remaining[0];
|
|
@@ -429,12 +578,18 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
429
578
|
if (pendingStarts.size > 0) {
|
|
430
579
|
const abortResults = await Promise.allSettled([...pendingStartSessions].map((session) => session.abortStart("Pi session shutdown during startup")));
|
|
431
580
|
startupFailures = abortResults.filter((result): result is PromiseRejectedResult => result.status === "rejected");
|
|
432
|
-
//
|
|
433
|
-
//
|
|
434
|
-
//
|
|
435
|
-
|
|
581
|
+
// Do not snapshot sessions or release leases until every startup has
|
|
582
|
+
// settled. Otherwise a provider that resolves after a fixed timeout
|
|
583
|
+
// could register a live handle after shutdown cleanup had completed.
|
|
584
|
+
// Adapters expose bounded cancellation; an uncooperative provider must
|
|
585
|
+
// keep shutdown fail-closed rather than allowing a late worker escape.
|
|
586
|
+
while (pendingStarts.size > 0) {
|
|
587
|
+
const results = await Promise.allSettled([...pendingStarts]);
|
|
588
|
+
startupFailures.push(...results.filter((result): result is PromiseRejectedResult => result.status === "rejected"));
|
|
589
|
+
}
|
|
436
590
|
}
|
|
437
|
-
const results = await Promise.allSettled([...sessions.values()].map((session) => stopSession(session, "Pi session shutdown")));
|
|
591
|
+
const results = await Promise.allSettled([...sessions.values()].map((session) => stopSession(session, "Pi session shutdown", true)));
|
|
592
|
+
await releaseSettledReservations();
|
|
438
593
|
const failures = [...startupFailures, ...results.filter((result): result is PromiseRejectedResult => result.status === "rejected")];
|
|
439
594
|
if (failures.length > 0) {
|
|
440
595
|
for (const failure of failures) console.error(`pi-claude-supervisor shutdown cleanup failed: ${redactText(failure.reason instanceof Error ? failure.reason.message : String(failure.reason))}`);
|
|
@@ -477,13 +632,8 @@ async function canonicalCwd(cwd: string): Promise<string> {
|
|
|
477
632
|
return realpath(cwd);
|
|
478
633
|
}
|
|
479
634
|
|
|
480
|
-
function
|
|
481
|
-
return
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
function isWithin(parent: string, child: string): boolean {
|
|
485
|
-
const childRelative = relative(parent, child);
|
|
486
|
-
return childRelative === "" || (childRelative !== ".." && !childRelative.startsWith(`..${sep}`) && !isAbsolute(childRelative));
|
|
635
|
+
function requiresWorkerCleanup(error: unknown): boolean {
|
|
636
|
+
return Boolean(error && typeof error === "object" && (error as { workerCleanupRequired?: unknown }).workerCleanupRequired === true);
|
|
487
637
|
}
|
|
488
638
|
|
|
489
639
|
function formatSessions(sessions: Map<string, Supervisor>, recoverable: DecisionSessionRecord[] = []): string {
|
package/src/supervisor.ts
CHANGED
|
@@ -53,6 +53,8 @@ export interface SupervisorStartOptions {
|
|
|
53
53
|
tmuxSession?: string;
|
|
54
54
|
/** Optional socket path for an existing non-default tmux server. */
|
|
55
55
|
tmuxSocket?: string;
|
|
56
|
+
/** Persisted identity required when handing off an existing tmux lease. */
|
|
57
|
+
tmuxExpectedIdentity?: WorkerStartInput["tmuxExpectedIdentity"];
|
|
56
58
|
/** Do not replay the task when adopting an existing interactive session. */
|
|
57
59
|
sendInitialInput?: boolean;
|
|
58
60
|
/** Enable the event-driven Pi Decision Worker. Requires claude-jsonl or tmux. */
|
|
@@ -101,6 +103,11 @@ export class Supervisor {
|
|
|
101
103
|
#humanRequired = false;
|
|
102
104
|
#onDecisionSessionProgress?: (info: { taskId: string; turn: number }) => Promise<void> | void;
|
|
103
105
|
#onDecisionSessionClosed?: (taskId: string) => Promise<void> | void;
|
|
106
|
+
#startAbortController?: AbortController;
|
|
107
|
+
#startToken?: string;
|
|
108
|
+
#startStopReason?: string;
|
|
109
|
+
#startAbortError?: unknown;
|
|
110
|
+
#startAbortCompletion?: Promise<void>;
|
|
104
111
|
|
|
105
112
|
constructor(adapter: WorkerAdapter, events = new EventLog(), hooks: { onHumanRequired?: (notice: HumanInterventionNotice) => Promise<void> | void } = {}) {
|
|
106
113
|
this.#adapter = adapter;
|
|
@@ -138,6 +145,11 @@ export class Supervisor {
|
|
|
138
145
|
this.#noOutputTimeoutMs = options.noOutputTimeoutMs ?? 20 * 60_000;
|
|
139
146
|
this.#clearWatchdog();
|
|
140
147
|
this.#machine.transition("starting");
|
|
148
|
+
const startAbortController = new AbortController();
|
|
149
|
+
this.#startAbortController = startAbortController;
|
|
150
|
+
this.#startToken = randomUUID();
|
|
151
|
+
this.#startStopReason = undefined;
|
|
152
|
+
this.#startAbortError = undefined;
|
|
141
153
|
try {
|
|
142
154
|
await this.#appendEvent({
|
|
143
155
|
type: "task_started",
|
|
@@ -151,6 +163,7 @@ export class Supervisor {
|
|
|
151
163
|
if (this.#automation && !["jsonl", "tmux"].includes(this.#adapter.capabilities().transport)) {
|
|
152
164
|
throw new Error("automatic supervision requires claude-jsonl or tmux transport");
|
|
153
165
|
}
|
|
166
|
+
this.#assertStartNotAborted(startAbortController.signal);
|
|
154
167
|
if (this.#automation) {
|
|
155
168
|
this.#decision = new PiDecisionWorker({
|
|
156
169
|
context: { taskId, task: options.task, cwd: options.cwd, state: this.#machine.state, turn: this.#turn, maxTurns: this.#task.maxTurns },
|
|
@@ -173,6 +186,7 @@ export class Supervisor {
|
|
|
173
186
|
});
|
|
174
187
|
await this.#decision.start();
|
|
175
188
|
}
|
|
189
|
+
this.#assertStartNotAborted(startAbortController.signal);
|
|
176
190
|
const input: WorkerStartInput = {
|
|
177
191
|
task: options.initialInput ?? options.task,
|
|
178
192
|
cwd: options.cwd,
|
|
@@ -182,37 +196,66 @@ export class Supervisor {
|
|
|
182
196
|
approval: options.approval,
|
|
183
197
|
tmuxSession: options.tmuxSession,
|
|
184
198
|
tmuxSocket: options.tmuxSocket,
|
|
199
|
+
tmuxExpectedIdentity: options.tmuxExpectedIdentity,
|
|
185
200
|
sendInitialInput: options.sendInitialInput,
|
|
186
201
|
eventListener: (event) => this.#receiveWorkerEvent(event),
|
|
202
|
+
abortSignal: startAbortController.signal,
|
|
203
|
+
startupToken: this.#startToken,
|
|
187
204
|
};
|
|
188
205
|
this.#handle = await this.#adapter.start(input);
|
|
206
|
+
this.#assertStartNotAborted(startAbortController.signal);
|
|
189
207
|
this.#machine.transition("running");
|
|
190
208
|
await this.#appendEvent({ type: "worker_started", taskId, workerId: this.#handle.id, data: { pid: this.#handle.pid } });
|
|
191
209
|
this.#armWatchdog();
|
|
210
|
+
this.#startAbortController = undefined;
|
|
211
|
+
this.#startToken = undefined;
|
|
212
|
+
this.#startStopReason = undefined;
|
|
213
|
+
this.#startAbortError = undefined;
|
|
192
214
|
return this.#handle;
|
|
193
215
|
} catch (error) {
|
|
194
|
-
const
|
|
216
|
+
const startupError = error instanceof Error ? error : new Error(String(error));
|
|
217
|
+
const startFailure = startupError as Error & { workerHandle?: WorkerHandle; workerCleanupRequired?: boolean };
|
|
195
218
|
const startFailureHandle = startFailure.workerHandle;
|
|
196
219
|
if (!this.#handle && startFailureHandle && (startFailure.workerCleanupRequired || startFailureHandle.ownership || startFailureHandle.sessionName)) this.#handle = startFailureHandle;
|
|
197
220
|
const handle = this.#handle;
|
|
221
|
+
const startupCancelled = Boolean(this.#startStopReason || startAbortController.signal.aborted);
|
|
222
|
+
let startupCleanupError: unknown = this.#startAbortError;
|
|
223
|
+
const abortCompletion = this.#startAbortCompletion;
|
|
224
|
+
if (abortCompletion) {
|
|
225
|
+
try { await abortCompletion; }
|
|
226
|
+
catch (error) { startupCleanupError ??= error; }
|
|
227
|
+
}
|
|
198
228
|
if (handle) {
|
|
199
229
|
try {
|
|
200
|
-
await this.#adapter.stop(handle, "startup failed");
|
|
201
|
-
} catch {
|
|
202
|
-
|
|
230
|
+
await this.#adapter.stop(handle, startupCancelled ? (this.#startStopReason ?? "startup aborted") : "startup failed");
|
|
231
|
+
} catch (error) {
|
|
232
|
+
startupCleanupError = error;
|
|
233
|
+
try { await this.#adapter.killProcessGroup(handle, "startup cleanup"); }
|
|
234
|
+
catch (cleanupError) { startupCleanupError ??= cleanupError; }
|
|
203
235
|
}
|
|
204
236
|
}
|
|
205
|
-
if (
|
|
237
|
+
if (startupCleanupError && !startFailure.workerCleanupRequired) {
|
|
238
|
+
Object.defineProperty(startupError, "workerCleanupRequired", { value: true, enumerable: false });
|
|
239
|
+
}
|
|
240
|
+
if (["starting", "running"].includes(this.#machine.state)) this.#machine.transition(startupCancelled && !startupCleanupError ? "stopped" : "failed");
|
|
206
241
|
try {
|
|
207
|
-
await this.#appendEvent({ type: "worker_start_failed", taskId, data: { error: safeMessage(
|
|
242
|
+
await this.#appendEvent({ type: "worker_start_failed", taskId, data: { error: safeMessage(startupError) } });
|
|
208
243
|
} catch { /* logging failure must not hide the startup failure */ }
|
|
209
244
|
await this.#decision?.close().catch(() => {});
|
|
210
245
|
this.#decision = undefined;
|
|
211
246
|
await Promise.resolve(this.#onDecisionSessionClosed?.(taskId)).catch(() => {});
|
|
212
|
-
|
|
247
|
+
this.#startAbortController = undefined;
|
|
248
|
+
this.#startToken = undefined;
|
|
249
|
+
this.#startStopReason = undefined;
|
|
250
|
+
this.#startAbortError = undefined;
|
|
251
|
+
throw startupError;
|
|
213
252
|
}
|
|
214
253
|
}
|
|
215
254
|
|
|
255
|
+
#assertStartNotAborted(signal: AbortSignal): void {
|
|
256
|
+
if (signal.aborted) throw new Error(`worker startup aborted: ${this.#startStopReason ?? "startup cancellation requested"}`);
|
|
257
|
+
}
|
|
258
|
+
|
|
216
259
|
async poll(): Promise<{ status: WorkerStatus; output: WorkerOutputChunk[] }> {
|
|
217
260
|
return this.#exclusive(() => this.#pollInternal());
|
|
218
261
|
}
|
|
@@ -496,21 +539,35 @@ export class Supervisor {
|
|
|
496
539
|
}
|
|
497
540
|
|
|
498
541
|
async abortStart(reason = "startup aborted"): Promise<void> {
|
|
542
|
+
if (this.#machine.state !== "starting" && !this.#startAbortController) return;
|
|
499
543
|
// This path intentionally bypasses #exclusive(): start() may be blocked in
|
|
500
544
|
// a Decision Worker model call and shutdown must still dispose that session.
|
|
501
|
-
|
|
502
|
-
const
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
545
|
+
this.#startStopReason = reason;
|
|
546
|
+
const completion = (async () => {
|
|
547
|
+
this.#startAbortController?.abort(reason);
|
|
548
|
+
let cleanupError: unknown;
|
|
549
|
+
const abort = this.#adapter.abortStart?.(reason, this.#startToken);
|
|
550
|
+
if (abort) {
|
|
551
|
+
try { await abort; }
|
|
552
|
+
catch (error) { cleanupError = error; }
|
|
553
|
+
}
|
|
554
|
+
await this.#decision?.close().catch(() => {});
|
|
555
|
+
this.#decision = undefined;
|
|
556
|
+
if (this.#handle) {
|
|
557
|
+
try { await this.#adapter.stop(this.#handle, reason); }
|
|
558
|
+
catch (error) { cleanupError ??= error; }
|
|
559
|
+
}
|
|
560
|
+
if (cleanupError) throw cleanupError;
|
|
561
|
+
})();
|
|
562
|
+
this.#startAbortCompletion = completion;
|
|
563
|
+
try {
|
|
564
|
+
await completion;
|
|
565
|
+
} catch (error) {
|
|
566
|
+
this.#startAbortError = error;
|
|
567
|
+
throw error;
|
|
568
|
+
} finally {
|
|
569
|
+
if (this.#startAbortCompletion === completion) this.#startAbortCompletion = undefined;
|
|
512
570
|
}
|
|
513
|
-
if (cleanupError) throw cleanupError;
|
|
514
571
|
}
|
|
515
572
|
|
|
516
573
|
async release(reason = "Pi session disconnected"): Promise<void> {
|
|
@@ -519,7 +576,9 @@ export class Supervisor {
|
|
|
519
576
|
? this.#adapter.release
|
|
520
577
|
? this.#adapter.release(handle, reason)
|
|
521
578
|
: this.#adapter.stop(handle, reason)
|
|
522
|
-
: this.#
|
|
579
|
+
: this.#machine.state === "starting" || this.#startAbortController
|
|
580
|
+
? this.abortStart(reason)
|
|
581
|
+
: Promise.resolve();
|
|
523
582
|
await this.#decision?.close().catch(() => {});
|
|
524
583
|
this.#decision = undefined;
|
|
525
584
|
await withTimeout(this.#exclusive(async () => {
|
|
@@ -530,6 +589,16 @@ export class Supervisor {
|
|
|
530
589
|
}
|
|
531
590
|
|
|
532
591
|
async stop(reason = "human requested stop"): Promise<void> {
|
|
592
|
+
// A stop must be able to preempt startup rather than waiting behind a
|
|
593
|
+
// startup operation that is blocked in a provider or adapter call.
|
|
594
|
+
if (this.#machine.state === "starting") {
|
|
595
|
+
this.#startStopReason = reason;
|
|
596
|
+
this.#startAbortController?.abort(reason);
|
|
597
|
+
if (!this.#handle) {
|
|
598
|
+
await withTimeout(this.abortStart(reason), 15_000, "worker startup cancellation");
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
}
|
|
533
602
|
// Start the adapter stop immediately so a queued/hung send cannot delay
|
|
534
603
|
// process termination. State/event changes still remain serialized below.
|
|
535
604
|
if (!this.#preemptiveStop && this.#handle && ["starting", "running", "waiting", "paused"].includes(this.#machine.state)) {
|
|
@@ -542,12 +611,20 @@ export class Supervisor {
|
|
|
542
611
|
async #stopInternal(reason: string, flushPendingEvents = true): Promise<void> {
|
|
543
612
|
if (flushPendingEvents) await this.#flushPendingEvents();
|
|
544
613
|
if (!this.#handle) throw new Error("no active task");
|
|
545
|
-
if (this.#machine.state === "stopped")
|
|
614
|
+
if (this.#machine.state === "stopped") {
|
|
615
|
+
const preemptiveStop = this.#preemptiveStop;
|
|
616
|
+
this.#preemptiveStop = undefined;
|
|
617
|
+
if (preemptiveStop) await preemptiveStop;
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
546
620
|
if (this.#machine.state === "failed") {
|
|
547
621
|
// A failed startup or cleanup attempt may still retain a live handle.
|
|
548
622
|
// Retry group termination during shutdown instead of treating the state
|
|
549
623
|
// as fully reclaimed.
|
|
550
|
-
|
|
624
|
+
const preemptiveStop = this.#preemptiveStop;
|
|
625
|
+
this.#preemptiveStop = undefined;
|
|
626
|
+
if (preemptiveStop) await preemptiveStop;
|
|
627
|
+
else if (this.#handle) await this.#adapter.stop(this.#handle, reason);
|
|
551
628
|
await this.#decision?.close().catch(() => {});
|
|
552
629
|
this.#decision = undefined;
|
|
553
630
|
await Promise.resolve(this.#onDecisionSessionClosed?.(this.#task?.taskId ?? "")).catch(() => {});
|
|
@@ -615,6 +692,7 @@ export class Supervisor {
|
|
|
615
692
|
await this.#drainOutputAfterStop(this.#handle);
|
|
616
693
|
const cleanup = await this.#adapter.getStatus(this.#handle);
|
|
617
694
|
if (cleanup.cleanupError) throw new Error(`worker cleanup failed after verification: ${cleanup.cleanupError}`);
|
|
695
|
+
if (cleanup.cgroupError && cleanup.cgroupRequired !== false) throw new Error(`worker cgroup cleanup failed after verification: ${cleanup.cgroupError}`);
|
|
618
696
|
if (this.#handle.ownership === "owned" && (cleanup.running || cleanup.processGroupCleaned !== true)) {
|
|
619
697
|
throw new Error("owned worker cleanup was not confirmed after verification");
|
|
620
698
|
}
|