pi-claude-supervisor 0.5.4 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.cn.md +15 -13
- package/README.md +61 -24
- package/docs/architecture.md +67 -32
- package/docs/autonomy-target.md +42 -25
- package/docs/engineering-plan.md +15 -12
- package/docs/implementation-review.md +25 -22
- package/docs/independent-review.md +7 -6
- package/docs/testing.md +73 -37
- package/package.json +1 -1
- package/src/cwd-lease.ts +736 -49
- package/src/index.ts +83 -8
- package/src/policy.ts +21 -34
- package/src/supervisor.ts +19 -3
- package/src/types.ts +16 -3
- package/src/worker/environment.ts +187 -124
- package/src/worker/process-adapter.ts +60 -105
- package/src/worker/process-tree.ts +1 -59
- package/src/worker/tmux-adapter.ts +264 -121
package/src/cwd-lease.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
1
2
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { chmod, lstat, mkdir, readFile, readdir, realpath, rm, stat, writeFile, rename } from "node:fs/promises";
|
|
3
|
-
import { join, resolve } from "node:path";
|
|
3
|
+
import { chmod, lstat, mkdir, readFile, readdir, realpath, rm, rmdir, stat, writeFile, rename } from "node:fs/promises";
|
|
4
|
+
import { basename, join, resolve } from "node:path";
|
|
5
|
+
import { promisify } from "node:util";
|
|
4
6
|
import { redactSensitive } from "./redaction.ts";
|
|
5
7
|
|
|
6
8
|
export type CwdLeaseTransport = "process-pipe" | "jsonl" | "pty" | "tmux";
|
|
@@ -12,13 +14,36 @@ export interface CwdLeaseWorker {
|
|
|
12
14
|
sessionName?: string;
|
|
13
15
|
tmuxSocket?: string;
|
|
14
16
|
ownership?: "owned" | "adopted";
|
|
17
|
+
workerId?: string;
|
|
15
18
|
cgroupPath?: string;
|
|
19
|
+
cgroupIdentity?: { device: string; inode: string };
|
|
20
|
+
retainCgroupUntilLeaseRelease?: boolean;
|
|
21
|
+
tmuxServerPid?: number;
|
|
22
|
+
tmuxServerStartTime?: string;
|
|
16
23
|
tmuxTarget?: string;
|
|
17
24
|
tmuxPaneId?: string;
|
|
18
25
|
paneStartTime?: string;
|
|
19
26
|
paneCommand?: string;
|
|
20
27
|
}
|
|
21
28
|
|
|
29
|
+
export interface CwdLeaseCleanup {
|
|
30
|
+
/** prepared = old record; replacement = new task record in the same file. */
|
|
31
|
+
phase: "prepared" | "replacement";
|
|
32
|
+
transport: CwdLeaseTransport;
|
|
33
|
+
workerId: string;
|
|
34
|
+
cgroupPath: string;
|
|
35
|
+
cgroupIdentity: { device: string; inode: string };
|
|
36
|
+
/** Set only after the verified cgroup directory has been removed. */
|
|
37
|
+
cgroupCleaned?: boolean;
|
|
38
|
+
sessionName?: string;
|
|
39
|
+
tmuxSocket?: string;
|
|
40
|
+
tmuxServerPid?: number;
|
|
41
|
+
tmuxServerStartTime?: string;
|
|
42
|
+
socketMarkerIdentity?: { device: string; inode: string };
|
|
43
|
+
/** Startup recovery may have no tmux server identity before the server is recorded. */
|
|
44
|
+
startupResource?: boolean;
|
|
45
|
+
}
|
|
46
|
+
|
|
22
47
|
export interface CwdLeaseRecord {
|
|
23
48
|
version: 1;
|
|
24
49
|
leaseId: string;
|
|
@@ -29,6 +54,10 @@ export interface CwdLeaseRecord {
|
|
|
29
54
|
acquiredAt: string;
|
|
30
55
|
updatedAt: string;
|
|
31
56
|
worker?: CwdLeaseWorker;
|
|
57
|
+
/** Automatic adapter startup has not yet persisted its final pre-spawn identity. */
|
|
58
|
+
pendingStartup?: { transport: CwdLeaseTransport };
|
|
59
|
+
/** Durable takeover transaction state; never start a new Worker while set. */
|
|
60
|
+
pendingCleanup?: CwdLeaseCleanup;
|
|
32
61
|
}
|
|
33
62
|
|
|
34
63
|
export interface CwdLeaseHandoff {
|
|
@@ -42,23 +71,52 @@ export interface CwdLeaseHandoff {
|
|
|
42
71
|
*/
|
|
43
72
|
export interface CwdLeaseTakeover {
|
|
44
73
|
taskId: string;
|
|
45
|
-
/** Run before the
|
|
74
|
+
/** Run before the replacement is committed; failure leaves the old lease intact. */
|
|
46
75
|
beforeReplace?: (lease: CwdLeaseRecord) => Promise<void>;
|
|
47
76
|
}
|
|
48
77
|
|
|
49
78
|
export interface CwdLeaseAcquireOptions {
|
|
50
79
|
handoff?: CwdLeaseHandoff;
|
|
51
80
|
takeover?: CwdLeaseTakeover;
|
|
81
|
+
/** Persist the no-spawn startup phase as part of lease acquisition. */
|
|
82
|
+
startup?: boolean;
|
|
52
83
|
}
|
|
53
84
|
|
|
54
85
|
export interface CwdLeaseHandle {
|
|
55
86
|
readonly record: CwdLeaseRecord;
|
|
56
87
|
/** Task id whose lease was explicitly handed off/taken over, if any. */
|
|
57
88
|
readonly replacedTaskId?: string;
|
|
58
|
-
updateWorker(worker: CwdLeaseWorker): Promise<void>;
|
|
89
|
+
updateWorker(worker: CwdLeaseWorker, options?: { preserveStartup?: boolean }): Promise<void>;
|
|
59
90
|
release(): Promise<void>;
|
|
60
91
|
}
|
|
61
92
|
|
|
93
|
+
type LockIdentity = {
|
|
94
|
+
device: number;
|
|
95
|
+
inode: number;
|
|
96
|
+
token: string;
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
type SocketReservation = {
|
|
100
|
+
identity: { device: string; inode: string };
|
|
101
|
+
release(): Promise<boolean>;
|
|
102
|
+
};
|
|
103
|
+
|
|
104
|
+
type TakeoverProof = {
|
|
105
|
+
/** Undefined for a durable pre-spawn lease reclaim with no Worker yet. */
|
|
106
|
+
pendingCleanup?: CwdLeaseCleanup;
|
|
107
|
+
startupOnly?: boolean;
|
|
108
|
+
reserve(): Promise<boolean>;
|
|
109
|
+
cleanup(): Promise<boolean>;
|
|
110
|
+
release(): Promise<boolean>;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
type TakeoverTransaction = {
|
|
114
|
+
proof: TakeoverProof;
|
|
115
|
+
replacementWritten: boolean;
|
|
116
|
+
cleanupFinalized: boolean;
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
const execFileAsync = promisify(execFile);
|
|
62
120
|
const LOCK_TIMEOUT_MS = 10_000;
|
|
63
121
|
const STALE_LOCK_MS = 5_000;
|
|
64
122
|
|
|
@@ -95,35 +153,122 @@ export class CwdLeaseStore {
|
|
|
95
153
|
acquiredAt: now,
|
|
96
154
|
updatedAt: now,
|
|
97
155
|
worker: { transport },
|
|
156
|
+
...(options.startup ? { pendingStartup: { transport } } : {}),
|
|
98
157
|
};
|
|
99
158
|
let replacedTaskId: string | undefined;
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
159
|
+
const takeoverTransactions: TakeoverTransaction[] = [];
|
|
160
|
+
try {
|
|
161
|
+
await this.#withLock(async () => {
|
|
162
|
+
const leases = await this.#readAll();
|
|
163
|
+
let handoffLease: CwdLeaseRecord | undefined;
|
|
164
|
+
let takeoverLease: CwdLeaseRecord | undefined;
|
|
165
|
+
let takeoverProof: TakeoverProof | undefined;
|
|
166
|
+
for (const existing of leases) {
|
|
167
|
+
if (options.handoff && existing.cwd === canonicalCwd && !existing.pendingCleanup && await matchesHandoff(existing, options.handoff)) {
|
|
168
|
+
if (handoffLease) throw new Error("multiple matching tmux cwd leases; refusing ambiguous handoff");
|
|
169
|
+
handoffLease = existing;
|
|
170
|
+
// Adopt in place so a crash cannot leave both the old and new
|
|
171
|
+
// overlapping lease records between write and deletion.
|
|
172
|
+
lease.leaseId = existing.leaseId;
|
|
173
|
+
lease.worker = { ...existing.worker!, transport };
|
|
174
|
+
if (options.startup) lease.pendingStartup = { transport };
|
|
175
|
+
else delete lease.pendingStartup;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (!pathsOverlap(existing.cwd, canonicalCwd)) continue;
|
|
179
|
+
if (options.takeover?.taskId === existing.taskId
|
|
180
|
+
&& existing.cwd === canonicalCwd
|
|
181
|
+
&& !existing.pendingCleanup) {
|
|
182
|
+
const proof = await canTakeoverLease(existing);
|
|
183
|
+
if (proof) {
|
|
184
|
+
if (takeoverLease) throw new Error("multiple matching cwd leases; refusing ambiguous takeover");
|
|
185
|
+
takeoverLease = existing;
|
|
186
|
+
takeoverProof = proof;
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
throw new Error(`working-directory lease is held by task ${existing.taskId}: ${redactText(existing.cwd)}`);
|
|
109
191
|
}
|
|
110
|
-
if (
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
192
|
+
if (takeoverLease && takeoverProof) {
|
|
193
|
+
if (takeoverProof.startupOnly) {
|
|
194
|
+
// Startup recovery proved that no adapter-owned resource remains
|
|
195
|
+
// (the planned cgroup/socket were either never created or are
|
|
196
|
+
// already gone). A replacement can be committed in place without
|
|
197
|
+
// inventing a cleanup proof.
|
|
198
|
+
try {
|
|
199
|
+
await options.takeover!.beforeReplace?.(takeoverLease);
|
|
200
|
+
} catch (error) {
|
|
201
|
+
await this.#write(takeoverLease).catch(() => {});
|
|
202
|
+
throw error;
|
|
203
|
+
}
|
|
204
|
+
lease.leaseId = takeoverLease.leaseId;
|
|
205
|
+
lease.pendingStartup = { transport };
|
|
206
|
+
replacedTaskId = takeoverLease.taskId;
|
|
207
|
+
await this.#write(lease);
|
|
208
|
+
} else {
|
|
209
|
+
const pendingCleanup = takeoverProof.pendingCleanup;
|
|
210
|
+
if (!pendingCleanup) throw new Error("automatic takeover cleanup proof is incomplete");
|
|
211
|
+
const transaction: TakeoverTransaction = { proof: takeoverProof, replacementWritten: false, cleanupFinalized: false };
|
|
212
|
+
takeoverTransactions.push(transaction);
|
|
213
|
+
|
|
214
|
+
// Persist the transaction before reserving the socket. If the
|
|
215
|
+
// Supervisor dies in any later startup window, the next lease
|
|
216
|
+
// reader can verify and finish this cleanup instead of losing
|
|
217
|
+
// the recovery evidence.
|
|
218
|
+
await this.#write({ ...takeoverLease, pendingCleanup, updatedAt: new Date().toISOString() });
|
|
219
|
+
if (!await takeoverProof.reserve()) throw new Error(`working-directory lease is held by task ${takeoverLease.taskId}: automatic takeover socket reservation failed`);
|
|
220
|
+
// Persist the marker identity immediately after mkdir. Recovery
|
|
221
|
+
// refuses to remove an unbound directory, so a failure in this
|
|
222
|
+
// write is fail-closed rather than silently guessing ownership.
|
|
223
|
+
await this.#write({ ...takeoverLease, pendingCleanup, updatedAt: new Date().toISOString() });
|
|
224
|
+
try {
|
|
225
|
+
await options.takeover!.beforeReplace?.(takeoverLease);
|
|
226
|
+
} catch (error) {
|
|
227
|
+
// Restore the active record when the hook rejects. If the
|
|
228
|
+
// restore itself fails, the pending transaction remains the
|
|
229
|
+
// safer durable state and will be reconciled on the next read.
|
|
230
|
+
await this.#write(takeoverLease).catch(() => {});
|
|
231
|
+
throw error;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// Reuse the old filename and replace it atomically. There is
|
|
235
|
+
// never a durable interval with two overlapping cwd leases.
|
|
236
|
+
pendingCleanup.phase = "replacement";
|
|
237
|
+
lease.leaseId = takeoverLease.leaseId;
|
|
238
|
+
lease.pendingCleanup = pendingCleanup;
|
|
239
|
+
replacedTaskId = takeoverLease.taskId;
|
|
240
|
+
await this.#write(lease);
|
|
241
|
+
transaction.replacementWritten = true;
|
|
242
|
+
|
|
243
|
+
// The cgroup/socket cleanup occurs only after the replacement
|
|
244
|
+
// lease is durable. Keep pendingCleanup until every cleanup
|
|
245
|
+
// proof is complete so a crash can resume it safely.
|
|
246
|
+
if (!await takeoverProof.cleanup()) {
|
|
247
|
+
throw new Error("automatic takeover cleanup remains pending; refusing Worker startup");
|
|
248
|
+
}
|
|
249
|
+
// Persist the cleanup stage before releasing the socket marker. If
|
|
250
|
+
// the process dies after cgroup removal, recovery can distinguish
|
|
251
|
+
// our verified removal from an unexplained missing cgroup.
|
|
252
|
+
pendingCleanup.cgroupCleaned = true;
|
|
253
|
+
await this.#write(lease);
|
|
254
|
+
if (!await takeoverProof.release()) {
|
|
255
|
+
throw new Error("automatic takeover cleanup remains pending; refusing Worker startup");
|
|
256
|
+
}
|
|
257
|
+
delete lease.pendingCleanup;
|
|
258
|
+
await this.#write(lease);
|
|
259
|
+
transaction.cleanupFinalized = true;
|
|
260
|
+
}
|
|
261
|
+
} else {
|
|
262
|
+
await this.#write(lease);
|
|
263
|
+
if (handoffLease) replacedTaskId = handoffLease.taskId;
|
|
118
264
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
await
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
});
|
|
265
|
+
});
|
|
266
|
+
} finally {
|
|
267
|
+
await Promise.allSettled(takeoverTransactions.map((transaction) => {
|
|
268
|
+
if (transaction.replacementWritten && !transaction.cleanupFinalized) return Promise.resolve(true);
|
|
269
|
+
return transaction.proof.release();
|
|
270
|
+
}));
|
|
271
|
+
}
|
|
127
272
|
return this.#handle(lease, replacedTaskId);
|
|
128
273
|
}
|
|
129
274
|
|
|
@@ -137,10 +282,15 @@ export class CwdLeaseStore {
|
|
|
137
282
|
return {
|
|
138
283
|
get record() { return current; },
|
|
139
284
|
get replacedTaskId() { return replacedTaskId; },
|
|
140
|
-
updateWorker: async (worker) => {
|
|
285
|
+
updateWorker: async (worker, options = {}) => {
|
|
141
286
|
if (released) throw new Error("cwd lease has already been released");
|
|
142
287
|
assertWorker(worker);
|
|
143
|
-
const next = {
|
|
288
|
+
const next = {
|
|
289
|
+
...current,
|
|
290
|
+
worker: { ...worker },
|
|
291
|
+
...(options.preserveStartup && current.pendingStartup ? { pendingStartup: current.pendingStartup } : { pendingStartup: undefined }),
|
|
292
|
+
updatedAt: new Date().toISOString(),
|
|
293
|
+
};
|
|
144
294
|
await this.#withLock(async () => {
|
|
145
295
|
const leases = await this.#readAll();
|
|
146
296
|
const existing = leases.find((lease) => lease.leaseId === current.leaseId);
|
|
@@ -156,6 +306,8 @@ export class CwdLeaseStore {
|
|
|
156
306
|
const leases = await this.#readAll();
|
|
157
307
|
const existing = leases.find((lease) => lease.leaseId === current.leaseId);
|
|
158
308
|
if (!existing) throw new Error("cwd lease disappeared before release");
|
|
309
|
+
if (existing.cwd !== current.cwd || existing.taskId !== current.taskId) throw new Error("cwd lease identity changed; refusing release");
|
|
310
|
+
if (existing.worker?.retainCgroupUntilLeaseRelease) await releaseRetainedCgroup(existing.worker);
|
|
159
311
|
await rm(this.#path(existing.leaseId), { force: true });
|
|
160
312
|
});
|
|
161
313
|
released = true;
|
|
@@ -177,7 +329,35 @@ export class CwdLeaseStore {
|
|
|
177
329
|
const leaseInfo = await lstat(leasePath);
|
|
178
330
|
if (!leaseInfo.isFile()) throw new Error(`cwd lease entry is not a regular file: ${redactText(leasePath)}`);
|
|
179
331
|
const value = JSON.parse(await readFile(leasePath, "utf8")) as Partial<CwdLeaseRecord>;
|
|
180
|
-
|
|
332
|
+
let lease = normalizeLease(value);
|
|
333
|
+
if (lease.pendingCleanup) {
|
|
334
|
+
// A takeover transaction is not usable until the old cgroup/session
|
|
335
|
+
// and any socket marker have been independently confirmed clean.
|
|
336
|
+
const cgroupCleanedBefore = lease.pendingCleanup.cgroupCleaned === true;
|
|
337
|
+
let cleanupComplete = await cleanupPendingLease(lease.pendingCleanup);
|
|
338
|
+
// Reconciliation may have removed the cgroup successfully but failed
|
|
339
|
+
// on a later marker operation. Persist that stage before returning a
|
|
340
|
+
// still-pending record, so the next reader can continue fail-closed.
|
|
341
|
+
if (!cgroupCleanedBefore && lease.pendingCleanup.cgroupCleaned === true) {
|
|
342
|
+
await this.#write({ ...lease, updatedAt: new Date().toISOString() });
|
|
343
|
+
// The first pass deliberately stops before socket-marker removal;
|
|
344
|
+
// the cleanup stage must be durable before that second proof.
|
|
345
|
+
cleanupComplete = await cleanupPendingLease(lease.pendingCleanup);
|
|
346
|
+
}
|
|
347
|
+
if (cleanupComplete) {
|
|
348
|
+
if (lease.pendingCleanup.phase === "replacement") {
|
|
349
|
+
// The replacement is the live lease after recovery. Clear only
|
|
350
|
+
// its transaction marker; do not delete the cwd reservation.
|
|
351
|
+
delete lease.pendingCleanup;
|
|
352
|
+
await this.#write(lease);
|
|
353
|
+
} else {
|
|
354
|
+
// The old record was still in the preparation phase, so no new
|
|
355
|
+
// Worker lease exists to retain after cleanup.
|
|
356
|
+
await rm(leasePath, { force: true });
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
}
|
|
181
361
|
// A missing or replaced cwd invalidates the registry evidence. Do not
|
|
182
362
|
// silently drop that record and allow a second worker to start.
|
|
183
363
|
lease.cwd = await realpath(lease.cwd);
|
|
@@ -203,14 +383,20 @@ export class CwdLeaseStore {
|
|
|
203
383
|
await this.#ensureDirectory();
|
|
204
384
|
const lockPath = join(this.#directory, ".lock");
|
|
205
385
|
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
386
|
+
const lockToken = randomUUID();
|
|
387
|
+
let lockIdentity: LockIdentity | undefined;
|
|
206
388
|
while (true) {
|
|
207
389
|
try {
|
|
208
390
|
await mkdir(lockPath);
|
|
209
391
|
await writeFile(join(lockPath, "owner.json"), JSON.stringify({
|
|
210
392
|
pid: process.pid,
|
|
211
393
|
startTime: await processStartTime(process.pid),
|
|
394
|
+
token: lockToken,
|
|
212
395
|
at: new Date().toISOString(),
|
|
213
396
|
}), { mode: 0o600 });
|
|
397
|
+
const acquired = await lstat(lockPath);
|
|
398
|
+
if (!acquired.isDirectory()) throw new Error("cwd lease lock is not a directory");
|
|
399
|
+
lockIdentity = { device: acquired.dev, inode: acquired.ino, token: lockToken };
|
|
214
400
|
break;
|
|
215
401
|
} catch (error) {
|
|
216
402
|
if (!(error instanceof Error) || !/EEXIST/u.test(error.message)) throw error;
|
|
@@ -222,7 +408,7 @@ export class CwdLeaseStore {
|
|
|
222
408
|
try {
|
|
223
409
|
return await operation();
|
|
224
410
|
} finally {
|
|
225
|
-
await
|
|
411
|
+
if (lockIdentity) await removeOwnedLock(lockPath, lockIdentity);
|
|
226
412
|
}
|
|
227
413
|
}
|
|
228
414
|
|
|
@@ -240,11 +426,21 @@ export function pathsOverlap(first: string, second: string): boolean {
|
|
|
240
426
|
return left === right || left.startsWith(`${right}/`) || right.startsWith(`${left}/`);
|
|
241
427
|
}
|
|
242
428
|
|
|
243
|
-
|
|
429
|
+
type WorkerIdentityInput = Pick<CwdLeaseWorker, "pid" | "cgroupPath" | "tmuxTarget" | "tmuxPaneId" | "paneStartTime" | "paneCommand" | "tmuxServerPid" | "tmuxServerStartTime" | "cgroupIdentity" | "workerId" | "retainCgroupUntilLeaseRelease"> & { id?: string };
|
|
430
|
+
|
|
431
|
+
type WorkerIdentityResult = Pick<CwdLeaseWorker, "workerId" | "pid" | "startTime" | "cgroupPath" | "cgroupIdentity" | "tmuxServerPid" | "tmuxServerStartTime" | "tmuxTarget" | "tmuxPaneId" | "paneStartTime" | "paneCommand" | "retainCgroupUntilLeaseRelease">;
|
|
432
|
+
|
|
433
|
+
export async function workerIdentity(worker: WorkerIdentityInput): Promise<WorkerIdentityResult> {
|
|
434
|
+
const cgroupIdentity = worker.cgroupPath ? await readCgroupIdentity(worker.cgroupPath) : undefined;
|
|
244
435
|
return {
|
|
436
|
+
workerId: worker.workerId ?? worker.id,
|
|
245
437
|
pid: worker.pid,
|
|
246
438
|
startTime: worker.pid ? await processStartTime(worker.pid) : undefined,
|
|
247
439
|
cgroupPath: worker.cgroupPath,
|
|
440
|
+
cgroupIdentity,
|
|
441
|
+
retainCgroupUntilLeaseRelease: worker.retainCgroupUntilLeaseRelease,
|
|
442
|
+
tmuxServerPid: worker.tmuxServerPid,
|
|
443
|
+
tmuxServerStartTime: worker.tmuxServerStartTime,
|
|
248
444
|
tmuxTarget: worker.tmuxTarget,
|
|
249
445
|
tmuxPaneId: worker.tmuxPaneId,
|
|
250
446
|
paneStartTime: worker.paneStartTime,
|
|
@@ -261,7 +457,8 @@ async function removeStaleLock(lockPath: string): Promise<boolean> {
|
|
|
261
457
|
if (!ownerInfo.isFile()) throw new Error("cwd lease lock owner is not a regular file");
|
|
262
458
|
const info = await stat(ownerPath);
|
|
263
459
|
if (Date.now() - info.mtimeMs < STALE_LOCK_MS) return false;
|
|
264
|
-
const owner = JSON.parse(await readFile(ownerPath, "utf8")) as { pid?: unknown; startTime?: unknown };
|
|
460
|
+
const owner = JSON.parse(await readFile(ownerPath, "utf8")) as { pid?: unknown; startTime?: unknown; token?: unknown };
|
|
461
|
+
if (typeof owner.token !== "string" || owner.token.length === 0) return false;
|
|
265
462
|
if (typeof owner.pid === "number") {
|
|
266
463
|
const currentStart = await processStartTime(owner.pid);
|
|
267
464
|
if (currentStart && typeof owner.startTime === "string" && currentStart === owner.startTime) return false;
|
|
@@ -272,14 +469,71 @@ async function removeStaleLock(lockPath: string): Promise<boolean> {
|
|
|
272
469
|
if (error instanceof Error && /EPERM/u.test(error.message)) return false;
|
|
273
470
|
}
|
|
274
471
|
}
|
|
275
|
-
|
|
276
|
-
|
|
472
|
+
|
|
473
|
+
// Never recursively remove the path that was inspected. Move exactly that
|
|
474
|
+
// lock-directory instance to a private quarantine name first; another
|
|
475
|
+
// acquirer may create a new lock at lockPath while quarantine is removed.
|
|
476
|
+
// The inode and optional owner token bind cleanup to the observed owner.
|
|
477
|
+
const quarantine = `${lockPath}.reap-${process.pid}-${randomUUID()}`;
|
|
478
|
+
try {
|
|
479
|
+
await rename(lockPath, quarantine);
|
|
480
|
+
} catch (error) {
|
|
481
|
+
if (error instanceof Error && /ENOENT/u.test(error.message)) return true;
|
|
482
|
+
return false;
|
|
483
|
+
}
|
|
484
|
+
try {
|
|
485
|
+
const restore = () => rename(quarantine, lockPath).catch(() => {});
|
|
486
|
+
const quarantinedInfo = await lstat(quarantine);
|
|
487
|
+
if (!quarantinedInfo.isDirectory() || quarantinedInfo.dev !== lockInfo.dev || quarantinedInfo.ino !== lockInfo.ino) {
|
|
488
|
+
await restore();
|
|
489
|
+
return false;
|
|
490
|
+
}
|
|
491
|
+
const quarantinedOwner = JSON.parse(await readFile(join(quarantine, "owner.json"), "utf8")) as { token?: unknown };
|
|
492
|
+
if (quarantinedOwner.token !== owner.token) {
|
|
493
|
+
await restore();
|
|
494
|
+
return false;
|
|
495
|
+
}
|
|
496
|
+
await rm(quarantine, { recursive: true, force: true });
|
|
497
|
+
return true;
|
|
498
|
+
} catch (error) {
|
|
499
|
+
if (error instanceof Error && /ENOENT/u.test(error.message)) return true;
|
|
500
|
+
return false;
|
|
501
|
+
}
|
|
277
502
|
} catch (error) {
|
|
278
503
|
if (error instanceof Error && /ENOENT/u.test(error.message)) return true;
|
|
279
504
|
return false;
|
|
280
505
|
}
|
|
281
506
|
}
|
|
282
507
|
|
|
508
|
+
async function removeOwnedLock(lockPath: string, identity: LockIdentity): Promise<void> {
|
|
509
|
+
// Rename the exact directory instance away before deleting it. A check
|
|
510
|
+
// followed by recursive rm would let a waiter replace the path between the
|
|
511
|
+
// two syscalls and could delete the waiter's new lock.
|
|
512
|
+
const quarantine = `${lockPath}.release-${process.pid}-${randomUUID()}`;
|
|
513
|
+
try {
|
|
514
|
+
const current = await lstat(lockPath);
|
|
515
|
+
if (!current.isDirectory() || current.dev !== identity.device || current.ino !== identity.inode) return;
|
|
516
|
+
const owner = JSON.parse(await readFile(join(lockPath, "owner.json"), "utf8")) as { token?: unknown };
|
|
517
|
+
if (owner.token !== identity.token) return;
|
|
518
|
+
await rename(lockPath, quarantine);
|
|
519
|
+
const restore = () => rename(quarantine, lockPath).catch(() => {});
|
|
520
|
+
const quarantined = await lstat(quarantine);
|
|
521
|
+
if (!quarantined.isDirectory() || quarantined.dev !== identity.device || quarantined.ino !== identity.inode) {
|
|
522
|
+
await restore();
|
|
523
|
+
return;
|
|
524
|
+
}
|
|
525
|
+
const quarantinedOwner = JSON.parse(await readFile(join(quarantine, "owner.json"), "utf8")) as { token?: unknown };
|
|
526
|
+
if (quarantinedOwner.token !== identity.token) {
|
|
527
|
+
await restore();
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
await rm(quarantine, { recursive: true, force: true });
|
|
531
|
+
} catch {
|
|
532
|
+
// Another process may already have quarantined or removed this instance.
|
|
533
|
+
// Any uncertain ownership is intentionally left for stale-lock handling.
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
|
|
283
537
|
async function processIdentityLive(pid: number, expectedStartTime?: string): Promise<boolean> {
|
|
284
538
|
const currentStartTime = await processStartTime(pid);
|
|
285
539
|
if (currentStartTime && expectedStartTime) return currentStartTime === expectedStartTime;
|
|
@@ -318,6 +572,10 @@ async function cgroupHasProcesses(path: string): Promise<boolean> {
|
|
|
318
572
|
if (!cgroupInfo.isDirectory() || cgroupInfo.isSymbolicLink()) return true;
|
|
319
573
|
const canonicalPath = await realpath(cgroupPath);
|
|
320
574
|
if (canonicalPath !== cgroupPath || !canonicalPath.startsWith(`${cgroupRoot}/`)) return true;
|
|
575
|
+
const eventsPath = join(canonicalPath, "cgroup.events");
|
|
576
|
+
const eventsInfo = await lstat(eventsPath);
|
|
577
|
+
if (!eventsInfo.isFile() || eventsInfo.isSymbolicLink()) return true;
|
|
578
|
+
if (!/^populated 0$/mu.test(await readFile(eventsPath, "utf8"))) return true;
|
|
321
579
|
const procsPath = join(canonicalPath, "cgroup.procs");
|
|
322
580
|
const procsInfo = await lstat(procsPath);
|
|
323
581
|
if (!procsInfo.isFile() || procsInfo.isSymbolicLink()) return true;
|
|
@@ -328,21 +586,376 @@ async function cgroupHasProcesses(path: string): Promise<boolean> {
|
|
|
328
586
|
}
|
|
329
587
|
}
|
|
330
588
|
|
|
331
|
-
async function canTakeoverLease(lease: CwdLeaseRecord): Promise<
|
|
589
|
+
async function canTakeoverLease(lease: CwdLeaseRecord): Promise<TakeoverProof | undefined> {
|
|
332
590
|
// The old supervisor owner must be gone. A dead owner is not enough when
|
|
333
|
-
// the detached Worker itself is still alive.
|
|
334
|
-
|
|
591
|
+
// the detached Worker itself is still alive. Compare start time as well as
|
|
592
|
+
// PID so a reused PID cannot be mistaken for the recorded owner.
|
|
593
|
+
if (lease.pendingCleanup || await processIdentityLive(lease.ownerPid, lease.ownerStartTime)) return undefined;
|
|
335
594
|
const worker = lease.worker;
|
|
336
|
-
if (
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
595
|
+
if (lease.pendingStartup && lease.pendingStartup.transport === worker?.transport
|
|
596
|
+
&& !worker.workerId && !worker.cgroupPath && worker.pid === undefined) {
|
|
597
|
+
// The startup callback has not even persisted the adapter's planned
|
|
598
|
+
// resource identity, so no adapter-owned cgroup/session can exist yet.
|
|
599
|
+
return startupOnlyTakeoverProof();
|
|
600
|
+
}
|
|
601
|
+
if (!worker) return undefined;
|
|
602
|
+
|
|
603
|
+
// The adapter persists a generated cgroup path before creating it, then
|
|
604
|
+
// persists its inode after creation. If recovery sees the marker during that
|
|
605
|
+
// interval, inspect and clean the planned resource rather than assuming it
|
|
606
|
+
// does not exist. Tmux may also have a server without its identity callback
|
|
607
|
+
// having completed, so its private session/socket are checked here too.
|
|
608
|
+
if (lease.pendingStartup && lease.pendingStartup.transport === worker.transport
|
|
609
|
+
&& worker.workerId && worker.cgroupPath && worker.pid === undefined
|
|
610
|
+
&& (!worker.cgroupIdentity || (worker.transport === "tmux" && !worker.tmuxServerPid && !worker.tmuxServerStartTime))) {
|
|
611
|
+
return startupResourceTakeoverProof(worker);
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
// A provisional automatic identity is persisted before spawn and has no
|
|
615
|
+
// leader PID yet. Its generated cgroup is the cleanup boundary; a populated
|
|
616
|
+
// boundary still blocks takeover. Once a PID exists, retain the stronger
|
|
617
|
+
// process and process-group identity checks as well.
|
|
618
|
+
if (worker.pid !== undefined) {
|
|
619
|
+
if (await processExists(worker.pid)) return undefined;
|
|
620
|
+
if (await processGroupExists(worker.pid)) return undefined;
|
|
621
|
+
} else if (!worker.retainCgroupUntilLeaseRelease) {
|
|
622
|
+
return undefined;
|
|
623
|
+
}
|
|
624
|
+
if (worker.tmuxServerPid && worker.tmuxServerStartTime && await processIdentityLive(worker.tmuxServerPid, worker.tmuxServerStartTime)) return undefined;
|
|
341
625
|
// A process-group check cannot see a setsid descendant. Explicit takeover
|
|
342
626
|
// therefore requires the verified cgroup boundary used by the adapter.
|
|
343
|
-
if (!worker.cgroupPath || !
|
|
344
|
-
|
|
345
|
-
|
|
627
|
+
if (!worker.workerId || !worker.cgroupPath || !isCgroupPath(worker.cgroupPath)
|
|
628
|
+
|| !matchesGeneratedCgroupName(worker.cgroupPath, worker.transport, worker.workerId)
|
|
629
|
+
|| !worker.cgroupIdentity) return undefined;
|
|
630
|
+
if (!await cgroupIdentityMatches(worker.cgroupPath, worker.cgroupIdentity)) return undefined;
|
|
631
|
+
if (await cgroupHasProcesses(worker.cgroupPath)) return undefined;
|
|
632
|
+
|
|
633
|
+
const pendingCleanup: CwdLeaseCleanup = {
|
|
634
|
+
phase: "prepared",
|
|
635
|
+
transport: worker.transport,
|
|
636
|
+
workerId: worker.workerId,
|
|
637
|
+
cgroupPath: worker.cgroupPath,
|
|
638
|
+
cgroupIdentity: worker.cgroupIdentity,
|
|
639
|
+
};
|
|
640
|
+
if (worker.transport === "tmux") {
|
|
641
|
+
// Only Supervisor-owned automatic sessions may be reclaimed. Adopted
|
|
642
|
+
// native TUI sessions intentionally remain human-owned and require an
|
|
643
|
+
// explicit identity-bound handoff instead.
|
|
644
|
+
if (worker.ownership !== "owned"
|
|
645
|
+
|| worker.sessionName !== `pi-supervisor-${worker.workerId}`
|
|
646
|
+
|| !worker.tmuxSocket
|
|
647
|
+
|| basename(resolve(worker.tmuxSocket)) !== `pi-cs-${worker.workerId}.sock`
|
|
648
|
+
|| !worker.tmuxServerPid
|
|
649
|
+
|| !worker.tmuxServerStartTime
|
|
650
|
+
|| !await tmuxSessionGone(worker.tmuxSocket, worker.sessionName)) return undefined;
|
|
651
|
+
pendingCleanup.sessionName = worker.sessionName;
|
|
652
|
+
pendingCleanup.tmuxSocket = worker.tmuxSocket;
|
|
653
|
+
pendingCleanup.tmuxServerPid = worker.tmuxServerPid;
|
|
654
|
+
pendingCleanup.tmuxServerStartTime = worker.tmuxServerStartTime;
|
|
655
|
+
let reservation: SocketReservation | undefined;
|
|
656
|
+
return {
|
|
657
|
+
pendingCleanup,
|
|
658
|
+
reserve: async () => {
|
|
659
|
+
reservation = await reserveTmuxSocket(worker.tmuxSocket!);
|
|
660
|
+
if (!reservation) return false;
|
|
661
|
+
pendingCleanup.socketMarkerIdentity = reservation.identity;
|
|
662
|
+
return true;
|
|
663
|
+
},
|
|
664
|
+
cleanup: async () => removeEmptyCgroup(worker.cgroupPath!, worker.cgroupIdentity!),
|
|
665
|
+
release: async () => reservation ? reservation.release() : true,
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
// The guarded automatic process bootstrap follows the same contract as the
|
|
669
|
+
// tmux guardian after a parent death: it leaves an empty cgroup for recovery.
|
|
670
|
+
return {
|
|
671
|
+
pendingCleanup,
|
|
672
|
+
reserve: async () => true,
|
|
673
|
+
cleanup: async () => removeEmptyCgroup(worker.cgroupPath!, worker.cgroupIdentity!),
|
|
674
|
+
release: async () => true,
|
|
675
|
+
};
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
function startupOnlyTakeoverProof(): TakeoverProof {
|
|
679
|
+
return {
|
|
680
|
+
startupOnly: true,
|
|
681
|
+
reserve: async () => true,
|
|
682
|
+
cleanup: async () => true,
|
|
683
|
+
release: async () => true,
|
|
684
|
+
};
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
async function startupResourceTakeoverProof(worker: CwdLeaseWorker): Promise<TakeoverProof | undefined> {
|
|
688
|
+
if (!worker.workerId || !worker.cgroupPath || !isCgroupPath(worker.cgroupPath)
|
|
689
|
+
|| !matchesGeneratedCgroupName(worker.cgroupPath, worker.transport, worker.workerId)
|
|
690
|
+
|| worker.pid !== undefined || !worker.retainCgroupUntilLeaseRelease) return undefined;
|
|
691
|
+
if (worker.tmuxServerPid && worker.tmuxServerStartTime
|
|
692
|
+
&& await processIdentityLive(worker.tmuxServerPid, worker.tmuxServerStartTime)) return undefined;
|
|
693
|
+
|
|
694
|
+
let cgroupIdentity: { device: string; inode: string };
|
|
695
|
+
try {
|
|
696
|
+
cgroupIdentity = await readCgroupIdentity(worker.cgroupPath);
|
|
697
|
+
if (worker.cgroupIdentity
|
|
698
|
+
&& (worker.cgroupIdentity.device !== cgroupIdentity.device || worker.cgroupIdentity.inode !== cgroupIdentity.inode)) return undefined;
|
|
699
|
+
} catch (error) {
|
|
700
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") return undefined;
|
|
701
|
+
// The adapter can have completed guardian cleanup after the startup plan
|
|
702
|
+
// was persisted. Missing cgroup plus a gone private tmux session is a
|
|
703
|
+
// verified no-resource startup, not permission to ignore a live session.
|
|
704
|
+
if (worker.transport === "tmux") {
|
|
705
|
+
if (worker.ownership !== "owned"
|
|
706
|
+
|| worker.sessionName !== `pi-supervisor-${worker.workerId}`
|
|
707
|
+
|| !worker.tmuxSocket
|
|
708
|
+
|| basename(resolve(worker.tmuxSocket)) !== `pi-cs-${worker.workerId}.sock`
|
|
709
|
+
|| !await tmuxSessionGone(worker.tmuxSocket, worker.sessionName)) return undefined;
|
|
710
|
+
// A socket with no matching session can still belong to a live tmux
|
|
711
|
+
// server. Startup-only replacement is allowed only once the private
|
|
712
|
+
// socket itself has disappeared (or can be atomically reserved below).
|
|
713
|
+
try {
|
|
714
|
+
await lstat(worker.tmuxSocket);
|
|
715
|
+
return undefined;
|
|
716
|
+
} catch (socketError) {
|
|
717
|
+
if ((socketError as NodeJS.ErrnoException).code !== "ENOENT") return undefined;
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
return startupOnlyTakeoverProof();
|
|
721
|
+
}
|
|
722
|
+
if (await cgroupHasProcesses(worker.cgroupPath)) return undefined;
|
|
723
|
+
|
|
724
|
+
const pendingCleanup: CwdLeaseCleanup = {
|
|
725
|
+
phase: "prepared",
|
|
726
|
+
transport: worker.transport,
|
|
727
|
+
workerId: worker.workerId,
|
|
728
|
+
cgroupPath: worker.cgroupPath,
|
|
729
|
+
cgroupIdentity,
|
|
730
|
+
...(worker.transport === "tmux" ? { startupResource: true } : {}),
|
|
731
|
+
};
|
|
732
|
+
if (worker.transport === "tmux") {
|
|
733
|
+
if (worker.ownership !== "owned"
|
|
734
|
+
|| worker.sessionName !== `pi-supervisor-${worker.workerId}`
|
|
735
|
+
|| !worker.tmuxSocket
|
|
736
|
+
|| basename(resolve(worker.tmuxSocket)) !== `pi-cs-${worker.workerId}.sock`
|
|
737
|
+
|| !await tmuxSessionGone(worker.tmuxSocket, worker.sessionName)) return undefined;
|
|
738
|
+
pendingCleanup.sessionName = worker.sessionName;
|
|
739
|
+
pendingCleanup.tmuxSocket = worker.tmuxSocket;
|
|
740
|
+
let reservation: SocketReservation | undefined;
|
|
741
|
+
return {
|
|
742
|
+
pendingCleanup,
|
|
743
|
+
reserve: async () => {
|
|
744
|
+
reservation = await reserveTmuxSocket(worker.tmuxSocket!);
|
|
745
|
+
if (!reservation) return false;
|
|
746
|
+
pendingCleanup.socketMarkerIdentity = reservation.identity;
|
|
747
|
+
return true;
|
|
748
|
+
},
|
|
749
|
+
cleanup: async () => removeEmptyCgroup(worker.cgroupPath!, cgroupIdentity),
|
|
750
|
+
release: async () => reservation ? reservation.release() : true,
|
|
751
|
+
};
|
|
752
|
+
}
|
|
753
|
+
return {
|
|
754
|
+
pendingCleanup,
|
|
755
|
+
reserve: async () => true,
|
|
756
|
+
cleanup: async () => removeEmptyCgroup(worker.cgroupPath!, cgroupIdentity),
|
|
757
|
+
release: async () => true,
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
async function cleanupPendingLease(cleanup: CwdLeaseCleanup): Promise<boolean> {
|
|
762
|
+
if (!isCgroupPath(cleanup.cgroupPath) || !matchesGeneratedCgroupName(cleanup.cgroupPath, cleanup.transport, cleanup.workerId)) return false;
|
|
763
|
+
if (cleanup.transport === "tmux") {
|
|
764
|
+
if (!cleanup.sessionName || !cleanup.tmuxSocket) return false;
|
|
765
|
+
if (!cleanup.startupResource) {
|
|
766
|
+
if (!cleanup.tmuxServerPid || !cleanup.tmuxServerStartTime) return false;
|
|
767
|
+
if (await processIdentityLive(cleanup.tmuxServerPid, cleanup.tmuxServerStartTime)) return false;
|
|
768
|
+
}
|
|
769
|
+
if (!await tmuxSessionGoneOrReserved(cleanup.tmuxSocket, cleanup.sessionName, cleanup.socketMarkerIdentity)) return false;
|
|
770
|
+
}
|
|
771
|
+
const cgroupWasCleaned = cleanup.cgroupCleaned === true;
|
|
772
|
+
if (!await cgroupGoneOrCleaned(cleanup)) return false;
|
|
773
|
+
// Persist cgroupCleaned before removing a tmux marker. A crash after marker
|
|
774
|
+
// removal but before the record write must remain recoverable on the next
|
|
775
|
+
// lease read.
|
|
776
|
+
if (!cgroupWasCleaned && cleanup.cgroupCleaned === true) return false;
|
|
777
|
+
return cleanup.transport !== "tmux" || await removeSocketMarker(cleanup.tmuxSocket!, cleanup.socketMarkerIdentity);
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
async function cgroupGoneOrCleaned(cleanup: CwdLeaseCleanup): Promise<boolean> {
|
|
781
|
+
try {
|
|
782
|
+
await lstat(cleanup.cgroupPath);
|
|
783
|
+
} catch (error) {
|
|
784
|
+
// A missing cgroup is not proof of cleanup unless the transaction already
|
|
785
|
+
// durably recorded that this Supervisor removed the verified empty one.
|
|
786
|
+
return (error as NodeJS.ErrnoException).code === "ENOENT" && cleanup.cgroupCleaned === true;
|
|
787
|
+
}
|
|
788
|
+
const removed = await removeEmptyCgroup(cleanup.cgroupPath, cleanup.cgroupIdentity);
|
|
789
|
+
if (removed) cleanup.cgroupCleaned = true;
|
|
790
|
+
return removed;
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
async function tmuxSessionGoneOrReserved(socket: string, session: string, markerIdentity?: { device: string; inode: string }): Promise<boolean> {
|
|
794
|
+
try {
|
|
795
|
+
const info = await lstat(socket);
|
|
796
|
+
if (info.isDirectory()) {
|
|
797
|
+
return Boolean(markerIdentity)
|
|
798
|
+
&& String(info.dev) === markerIdentity!.device
|
|
799
|
+
&& String(info.ino) === markerIdentity!.inode;
|
|
800
|
+
}
|
|
801
|
+
} catch (error) {
|
|
802
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return true;
|
|
803
|
+
return false;
|
|
804
|
+
}
|
|
805
|
+
return tmuxSessionGone(socket, session);
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
async function removeSocketMarker(socket: string, markerIdentity?: { device: string; inode: string }): Promise<boolean> {
|
|
809
|
+
if (!markerIdentity) return false;
|
|
810
|
+
const quarantine = `${socket}.release-${process.pid}-${randomUUID()}`;
|
|
811
|
+
try {
|
|
812
|
+
const current = await lstat(socket);
|
|
813
|
+
if (!current.isDirectory() || current.isSymbolicLink()) return false;
|
|
814
|
+
if (String(current.dev) !== markerIdentity.device || String(current.ino) !== markerIdentity.inode) return false;
|
|
815
|
+
if ((await readdir(socket)).length !== 0) return false;
|
|
816
|
+
// Move the observed marker instance before removing it. A path check
|
|
817
|
+
// followed by rmdir could delete a replacement directory created by a
|
|
818
|
+
// competing recovery process.
|
|
819
|
+
await rename(socket, quarantine);
|
|
820
|
+
const moved = await lstat(quarantine);
|
|
821
|
+
if (!moved.isDirectory() || moved.isSymbolicLink()
|
|
822
|
+
|| String(moved.dev) !== markerIdentity.device || String(moved.ino) !== markerIdentity.inode) {
|
|
823
|
+
await rename(quarantine, socket).catch(() => {});
|
|
824
|
+
return false;
|
|
825
|
+
}
|
|
826
|
+
if ((await readdir(quarantine)).length !== 0) {
|
|
827
|
+
await rename(quarantine, socket).catch(() => {});
|
|
828
|
+
return false;
|
|
829
|
+
}
|
|
830
|
+
await rmdir(quarantine);
|
|
831
|
+
return true;
|
|
832
|
+
} catch (error) {
|
|
833
|
+
return (error as NodeJS.ErrnoException).code === "ENOENT";
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
function isCgroupPath(path: string): boolean {
|
|
838
|
+
const root = resolve("/sys/fs/cgroup");
|
|
839
|
+
const candidate = resolve(path);
|
|
840
|
+
return candidate !== root && candidate.startsWith(`${root}/`);
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
function matchesGeneratedCgroupName(path: string, transport: CwdLeaseTransport, workerId: string): boolean {
|
|
844
|
+
const prefix = transport === "tmux" ? "pi-claude-supervisor-tmux-" : "pi-claude-supervisor-";
|
|
845
|
+
return basename(resolve(path)) === `${prefix}${workerId}`;
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
async function readCgroupIdentity(path: string): Promise<{ device: string; inode: string }> {
|
|
849
|
+
if (!isCgroupPath(path)) throw new Error("invalid cgroup path");
|
|
850
|
+
const candidate = resolve(path);
|
|
851
|
+
const info = await lstat(candidate);
|
|
852
|
+
if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("cgroup path is not a real directory");
|
|
853
|
+
const canonicalPath = await realpath(candidate);
|
|
854
|
+
if (canonicalPath !== candidate || !isCgroupPath(canonicalPath)) throw new Error("cgroup path is not canonical");
|
|
855
|
+
return { device: String(info.dev), inode: String(info.ino) };
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
async function cgroupIdentityMatches(path: string, expected: { device: string; inode: string }): Promise<boolean> {
|
|
859
|
+
try {
|
|
860
|
+
const actual = await readCgroupIdentity(path);
|
|
861
|
+
return actual.device === expected.device && actual.inode === expected.inode;
|
|
862
|
+
} catch {
|
|
863
|
+
return false;
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
async function removeEmptyCgroup(path: string, expected: { device: string; inode: string }): Promise<boolean> {
|
|
868
|
+
// Re-check both the persisted directory identity and emptiness immediately
|
|
869
|
+
// before removal. The initial checks only authorize attempting takeover;
|
|
870
|
+
// they must not be reused after the pre-replacement hook has run.
|
|
871
|
+
if (!await cgroupIdentityMatches(path, expected) || await cgroupHasProcesses(path)) return false;
|
|
872
|
+
try {
|
|
873
|
+
await rmdir(resolve(path));
|
|
874
|
+
return true;
|
|
875
|
+
} catch {
|
|
876
|
+
return false;
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
async function releaseRetainedCgroup(worker: CwdLeaseWorker): Promise<void> {
|
|
881
|
+
if (!worker.workerId || !worker.cgroupPath
|
|
882
|
+
|| !isCgroupPath(worker.cgroupPath)
|
|
883
|
+
|| !matchesGeneratedCgroupName(worker.cgroupPath, worker.transport, worker.workerId)) {
|
|
884
|
+
throw new Error("retained Worker cgroup identity is invalid");
|
|
885
|
+
}
|
|
886
|
+
if (!worker.cgroupIdentity) {
|
|
887
|
+
try {
|
|
888
|
+
await lstat(worker.cgroupPath);
|
|
889
|
+
} catch (error) {
|
|
890
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
891
|
+
throw error;
|
|
892
|
+
}
|
|
893
|
+
throw new Error("retained Worker cgroup identity is unavailable");
|
|
894
|
+
}
|
|
895
|
+
try {
|
|
896
|
+
await lstat(worker.cgroupPath);
|
|
897
|
+
} catch (error) {
|
|
898
|
+
// Startup failure cleanup may have removed the cgroup before the lease
|
|
899
|
+
// release reached this path. Recovery takeover remains stricter and never
|
|
900
|
+
// treats a missing cgroup as proof unless its cleanup stage is persisted.
|
|
901
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return;
|
|
902
|
+
throw error;
|
|
903
|
+
}
|
|
904
|
+
if (!await removeEmptyCgroup(worker.cgroupPath, worker.cgroupIdentity)) {
|
|
905
|
+
throw new Error("retained Worker cgroup cleanup could not be confirmed");
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
async function reserveTmuxSocket(socket: string): Promise<SocketReservation | undefined> {
|
|
910
|
+
try {
|
|
911
|
+
const existing = await lstat(socket);
|
|
912
|
+
// A private socket must have disappeared before it can be reserved. Do
|
|
913
|
+
// not unlink a live or replaced socket as part of takeover.
|
|
914
|
+
if (existing) return undefined;
|
|
915
|
+
} catch (error) {
|
|
916
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") return undefined;
|
|
917
|
+
}
|
|
918
|
+
try {
|
|
919
|
+
// tmux removes a regular stale socket path before binding its own socket;
|
|
920
|
+
// a directory is an atomic reservation that tmux cannot replace. If a
|
|
921
|
+
// competing process wins the mkdir race, fail closed instead.
|
|
922
|
+
await mkdir(socket, { mode: 0o700 });
|
|
923
|
+
const markerInfo = await lstat(socket);
|
|
924
|
+
if (!markerInfo.isDirectory() || markerInfo.isSymbolicLink()) {
|
|
925
|
+
await rmdir(socket).catch(() => {});
|
|
926
|
+
return undefined;
|
|
927
|
+
}
|
|
928
|
+
let released = false;
|
|
929
|
+
const identity = { device: String(markerInfo.dev), inode: String(markerInfo.ino) };
|
|
930
|
+
const release = async () => {
|
|
931
|
+
if (released) return true;
|
|
932
|
+
released = true;
|
|
933
|
+
return removeSocketMarker(socket, identity);
|
|
934
|
+
};
|
|
935
|
+
return { identity, release };
|
|
936
|
+
} catch {
|
|
937
|
+
return undefined;
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
async function tmuxSessionGone(socket: string, session: string): Promise<boolean> {
|
|
942
|
+
// Automatic sessions always use a private absolute socket and a generated
|
|
943
|
+
// safe session name. Never pass lease-controlled strings through a shell.
|
|
944
|
+
if (!socket.startsWith("/") || socket.includes("\0") || !/^[A-Za-z0-9_.-]+$/u.test(session)) return false;
|
|
945
|
+
try {
|
|
946
|
+
const socketInfo = await lstat(socket);
|
|
947
|
+
if (!socketInfo.isSocket()) return false;
|
|
948
|
+
} catch (error) {
|
|
949
|
+
return (error as NodeJS.ErrnoException).code === "ENOENT";
|
|
950
|
+
}
|
|
951
|
+
try {
|
|
952
|
+
await execFileAsync("tmux", ["-S", socket, "has-session", "-t", session], { timeout: 5_000, maxBuffer: 8 * 1024 });
|
|
953
|
+
return false;
|
|
954
|
+
} catch (error) {
|
|
955
|
+
const details = error as NodeJS.ErrnoException & { stderr?: string };
|
|
956
|
+
if (String(details.code) !== "1") return false;
|
|
957
|
+
return /no server|can't find session|no such file|connection refused/iu.test(String(details.stderr ?? ""));
|
|
958
|
+
}
|
|
346
959
|
}
|
|
347
960
|
|
|
348
961
|
async function processStartTime(pid: number): Promise<string | undefined> {
|
|
@@ -379,14 +992,71 @@ function assertTaskId(taskId: string): void {
|
|
|
379
992
|
|
|
380
993
|
function assertWorker(worker: CwdLeaseWorker): void {
|
|
381
994
|
if (!["process-pipe", "jsonl", "pty", "tmux"].includes(worker.transport)
|
|
995
|
+
|| (worker.workerId !== undefined && !/^[0-9a-f-]{36}$/iu.test(worker.workerId))
|
|
382
996
|
|| (worker.pid !== undefined && (!Number.isSafeInteger(worker.pid) || worker.pid < 1))
|
|
383
997
|
|| (worker.startTime !== undefined && !/^\d+$/u.test(worker.startTime))
|
|
998
|
+
|| (worker.tmuxServerPid !== undefined && (!Number.isSafeInteger(worker.tmuxServerPid) || worker.tmuxServerPid < 1))
|
|
999
|
+
|| (worker.tmuxServerStartTime !== undefined && !/^\d+$/u.test(worker.tmuxServerStartTime))
|
|
384
1000
|
|| (worker.tmuxPaneId !== undefined && !/^%\d+$/u.test(worker.tmuxPaneId))
|
|
385
|
-
|| (worker.cgroupPath !== undefined && !worker.cgroupPath.startsWith("/"))
|
|
1001
|
+
|| (worker.cgroupPath !== undefined && !worker.cgroupPath.startsWith("/"))
|
|
1002
|
+
|| (worker.cgroupIdentity !== undefined && (!/^\d+$/u.test(worker.cgroupIdentity.device) || !/^\d+$/u.test(worker.cgroupIdentity.inode)))
|
|
1003
|
+
|| (worker.retainCgroupUntilLeaseRelease !== undefined && typeof worker.retainCgroupUntilLeaseRelease !== "boolean")) {
|
|
386
1004
|
throw new Error("invalid cwd lease worker identity");
|
|
387
1005
|
}
|
|
388
1006
|
}
|
|
389
1007
|
|
|
1008
|
+
function normalizeCleanup(value: unknown): CwdLeaseCleanup | undefined {
|
|
1009
|
+
if (value === undefined) return undefined;
|
|
1010
|
+
if (!value || typeof value !== "object") throw new Error("invalid cwd lease cleanup state");
|
|
1011
|
+
const cleanup = value as Partial<CwdLeaseCleanup>;
|
|
1012
|
+
if (cleanup.phase !== "prepared" && cleanup.phase !== "replacement"
|
|
1013
|
+
|| !cleanup.transport || !["process-pipe", "jsonl", "pty", "tmux"].includes(cleanup.transport)
|
|
1014
|
+
|| typeof cleanup.workerId !== "string" || !/^[0-9a-f-]{36}$/iu.test(cleanup.workerId)
|
|
1015
|
+
|| typeof cleanup.cgroupPath !== "string" || !isCgroupPath(cleanup.cgroupPath)
|
|
1016
|
+
|| !matchesGeneratedCgroupName(cleanup.cgroupPath, cleanup.transport, cleanup.workerId)
|
|
1017
|
+
|| !cleanup.cgroupIdentity || typeof cleanup.cgroupIdentity !== "object"
|
|
1018
|
+
|| typeof cleanup.cgroupIdentity.device !== "string" || !/^\d+$/u.test(cleanup.cgroupIdentity.device)
|
|
1019
|
+
|| typeof cleanup.cgroupIdentity.inode !== "string" || !/^\d+$/u.test(cleanup.cgroupIdentity.inode)
|
|
1020
|
+
|| (cleanup.cgroupCleaned !== undefined && cleanup.cgroupCleaned !== true)) {
|
|
1021
|
+
throw new Error("invalid cwd lease cleanup state");
|
|
1022
|
+
}
|
|
1023
|
+
if (cleanup.transport === "tmux"
|
|
1024
|
+
&& (typeof cleanup.sessionName !== "string" || cleanup.sessionName !== `pi-supervisor-${cleanup.workerId}`
|
|
1025
|
+
|| typeof cleanup.tmuxSocket !== "string" || !cleanup.tmuxSocket.startsWith("/")
|
|
1026
|
+
|| basename(resolve(cleanup.tmuxSocket)) !== `pi-cs-${cleanup.workerId}.sock`
|
|
1027
|
+
|| (cleanup.startupResource !== true
|
|
1028
|
+
&& (!Number.isSafeInteger(cleanup.tmuxServerPid) || cleanup.tmuxServerPid! < 1
|
|
1029
|
+
|| typeof cleanup.tmuxServerStartTime !== "string" || !/^\d+$/u.test(cleanup.tmuxServerStartTime))))) {
|
|
1030
|
+
throw new Error("invalid cwd lease cleanup state");
|
|
1031
|
+
}
|
|
1032
|
+
if (cleanup.transport !== "tmux" && (cleanup.sessionName !== undefined || cleanup.tmuxSocket !== undefined
|
|
1033
|
+
|| cleanup.tmuxServerPid !== undefined || cleanup.tmuxServerStartTime !== undefined || cleanup.socketMarkerIdentity !== undefined
|
|
1034
|
+
|| cleanup.startupResource !== undefined)) {
|
|
1035
|
+
throw new Error("invalid cwd lease cleanup state");
|
|
1036
|
+
}
|
|
1037
|
+
if (cleanup.startupResource !== undefined && cleanup.startupResource !== true) throw new Error("invalid cwd lease cleanup state");
|
|
1038
|
+
if (cleanup.socketMarkerIdentity !== undefined
|
|
1039
|
+
&& (typeof cleanup.socketMarkerIdentity !== "object"
|
|
1040
|
+
|| typeof cleanup.socketMarkerIdentity.device !== "string" || !/^\d+$/u.test(cleanup.socketMarkerIdentity.device)
|
|
1041
|
+
|| typeof cleanup.socketMarkerIdentity.inode !== "string" || !/^\d+$/u.test(cleanup.socketMarkerIdentity.inode))) {
|
|
1042
|
+
throw new Error("invalid cwd lease cleanup state");
|
|
1043
|
+
}
|
|
1044
|
+
return {
|
|
1045
|
+
phase: cleanup.phase,
|
|
1046
|
+
transport: cleanup.transport,
|
|
1047
|
+
workerId: cleanup.workerId,
|
|
1048
|
+
cgroupPath: cleanup.cgroupPath,
|
|
1049
|
+
cgroupIdentity: { device: cleanup.cgroupIdentity.device, inode: cleanup.cgroupIdentity.inode },
|
|
1050
|
+
...(cleanup.cgroupCleaned ? { cgroupCleaned: true } : {}),
|
|
1051
|
+
...(cleanup.sessionName ? { sessionName: cleanup.sessionName } : {}),
|
|
1052
|
+
...(cleanup.tmuxSocket ? { tmuxSocket: cleanup.tmuxSocket } : {}),
|
|
1053
|
+
...(cleanup.tmuxServerPid !== undefined ? { tmuxServerPid: cleanup.tmuxServerPid } : {}),
|
|
1054
|
+
...(cleanup.tmuxServerStartTime ? { tmuxServerStartTime: cleanup.tmuxServerStartTime } : {}),
|
|
1055
|
+
...(cleanup.socketMarkerIdentity ? { socketMarkerIdentity: { ...cleanup.socketMarkerIdentity } } : {}),
|
|
1056
|
+
...(cleanup.startupResource ? { startupResource: true } : {}),
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
|
|
390
1060
|
function normalizeLease(value: Partial<CwdLeaseRecord>): CwdLeaseRecord {
|
|
391
1061
|
if (value.version !== 1 || typeof value.leaseId !== "string" || !/^[0-9a-f-]{36}$/iu.test(value.leaseId)
|
|
392
1062
|
|| typeof value.taskId !== "string" || !/^[0-9a-f-]{36}$/iu.test(value.taskId)
|
|
@@ -399,6 +1069,12 @@ function normalizeLease(value: Partial<CwdLeaseRecord>): CwdLeaseRecord {
|
|
|
399
1069
|
}
|
|
400
1070
|
const worker = value.worker;
|
|
401
1071
|
if (worker && (typeof worker !== "object" || !["process-pipe", "jsonl", "pty", "tmux"].includes(worker.transport as string))) throw new Error("invalid cwd lease worker identity");
|
|
1072
|
+
const pendingStartup = value.pendingStartup;
|
|
1073
|
+
if (pendingStartup !== undefined
|
|
1074
|
+
&& (!pendingStartup || typeof pendingStartup !== "object" || !["process-pipe", "jsonl", "pty", "tmux"].includes(pendingStartup.transport))) {
|
|
1075
|
+
throw new Error("invalid cwd lease startup state");
|
|
1076
|
+
}
|
|
1077
|
+
const pendingCleanup = normalizeCleanup(value.pendingCleanup);
|
|
402
1078
|
const ownerPid = value.ownerPid!;
|
|
403
1079
|
return {
|
|
404
1080
|
version: 1,
|
|
@@ -411,17 +1087,28 @@ function normalizeLease(value: Partial<CwdLeaseRecord>): CwdLeaseRecord {
|
|
|
411
1087
|
updatedAt: value.updatedAt,
|
|
412
1088
|
worker: worker ? {
|
|
413
1089
|
transport: worker.transport!,
|
|
1090
|
+
workerId: typeof worker.workerId === "string" && /^[0-9a-f-]{36}$/iu.test(worker.workerId) ? worker.workerId : undefined,
|
|
414
1091
|
pid: Number.isSafeInteger(worker.pid) && worker.pid! > 0 ? worker.pid : undefined,
|
|
415
1092
|
startTime: typeof worker.startTime === "string" && /^\d+$/u.test(worker.startTime) ? worker.startTime : undefined,
|
|
416
1093
|
sessionName: typeof worker.sessionName === "string" ? worker.sessionName : undefined,
|
|
417
1094
|
tmuxSocket: typeof worker.tmuxSocket === "string" ? worker.tmuxSocket : undefined,
|
|
418
1095
|
ownership: worker.ownership === "owned" || worker.ownership === "adopted" ? worker.ownership : undefined,
|
|
419
1096
|
cgroupPath: typeof worker.cgroupPath === "string" ? worker.cgroupPath : undefined,
|
|
1097
|
+
cgroupIdentity: worker.cgroupIdentity && typeof worker.cgroupIdentity === "object"
|
|
1098
|
+
&& typeof worker.cgroupIdentity.device === "string" && /^\d+$/u.test(worker.cgroupIdentity.device)
|
|
1099
|
+
&& typeof worker.cgroupIdentity.inode === "string" && /^\d+$/u.test(worker.cgroupIdentity.inode)
|
|
1100
|
+
? { device: worker.cgroupIdentity.device, inode: worker.cgroupIdentity.inode }
|
|
1101
|
+
: undefined,
|
|
1102
|
+
retainCgroupUntilLeaseRelease: worker.retainCgroupUntilLeaseRelease === true ? true : undefined,
|
|
1103
|
+
tmuxServerPid: Number.isSafeInteger(worker.tmuxServerPid) && worker.tmuxServerPid! > 0 ? worker.tmuxServerPid : undefined,
|
|
1104
|
+
tmuxServerStartTime: typeof worker.tmuxServerStartTime === "string" && /^\d+$/u.test(worker.tmuxServerStartTime) ? worker.tmuxServerStartTime : undefined,
|
|
420
1105
|
tmuxTarget: typeof worker.tmuxTarget === "string" ? worker.tmuxTarget : undefined,
|
|
421
1106
|
tmuxPaneId: typeof worker.tmuxPaneId === "string" && /^%\d+$/u.test(worker.tmuxPaneId) ? worker.tmuxPaneId : undefined,
|
|
422
1107
|
paneStartTime: typeof worker.paneStartTime === "string" ? worker.paneStartTime : undefined,
|
|
423
1108
|
paneCommand: typeof worker.paneCommand === "string" ? worker.paneCommand : undefined,
|
|
424
1109
|
} : undefined,
|
|
1110
|
+
...(pendingStartup ? { pendingStartup: { transport: pendingStartup.transport } } : {}),
|
|
1111
|
+
pendingCleanup,
|
|
425
1112
|
};
|
|
426
1113
|
}
|
|
427
1114
|
|