pi-claude-supervisor 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -1
- package/README.cn.md +14 -6
- package/README.md +22 -10
- package/docs/architecture.md +30 -4
- package/docs/engineering-plan.md +54 -3
- package/docs/recovery-review.md +24 -0
- package/docs/stability-matrix-2.1.270.md +32 -0
- package/docs/testing.md +36 -7
- package/package.json +1 -1
- package/src/cwd-lease.ts +95 -6
- package/src/decision-session-store.ts +343 -37
- package/src/decision-worker.ts +41 -4
- package/src/index.ts +161 -82
- package/src/supervisor.ts +43 -13
- package/src/verifier.ts +5 -2
|
@@ -1,9 +1,18 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { chmod, lstat, mkdir, readdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
2
3
|
import { basename, dirname, join, resolve } from "node:path";
|
|
3
4
|
import { redactSensitive } from "./redaction.ts";
|
|
4
5
|
import { normalizeTaskSpec } from "./acceptance.ts";
|
|
5
6
|
import type { TaskSpec } from "./types.ts";
|
|
6
7
|
|
|
8
|
+
export type DecisionRecoveryState = "ready" | "starting" | "registered" | "recovered_idle" | "interrupted";
|
|
9
|
+
|
|
10
|
+
export interface DecisionRecoveryWorker {
|
|
11
|
+
id: string;
|
|
12
|
+
pid?: number;
|
|
13
|
+
startedAt: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
7
16
|
export interface DecisionSessionRecord {
|
|
8
17
|
version: 1;
|
|
9
18
|
taskId: string;
|
|
@@ -22,13 +31,26 @@ export interface DecisionSessionRecord {
|
|
|
22
31
|
repairRound?: number;
|
|
23
32
|
lastFindingSignature?: string;
|
|
24
33
|
state: "active" | "closed";
|
|
34
|
+
recoveryState: DecisionRecoveryState;
|
|
35
|
+
recoveryAttempt: number;
|
|
36
|
+
recoveryOwnerPid?: number;
|
|
37
|
+
recoveryOwnerStartTime?: string;
|
|
38
|
+
recoveryWorker?: DecisionRecoveryWorker;
|
|
25
39
|
updatedAt: string;
|
|
26
40
|
}
|
|
27
41
|
|
|
42
|
+
export type DecisionSessionRecordInput = Omit<DecisionSessionRecord, "version" | "updatedAt" | "recoveryState" | "recoveryAttempt" | "recoveryOwnerPid" | "recoveryOwnerStartTime"> & Partial<Pick<DecisionSessionRecord, "updatedAt" | "recoveryState" | "recoveryAttempt" | "recoveryOwnerPid" | "recoveryOwnerStartTime">>;
|
|
43
|
+
|
|
44
|
+
const LOCK_TIMEOUT_MS = 10_000;
|
|
45
|
+
const STALE_LOCK_MS = 5_000;
|
|
46
|
+
|
|
28
47
|
/**
|
|
29
48
|
* Small crash-tolerant registry for Decision Worker sessions.
|
|
30
49
|
* The Pi session JSONL remains the source of conversation history; this file
|
|
31
50
|
* only maps a supervisor task to that history and the restart parameters.
|
|
51
|
+
*
|
|
52
|
+
* Registry mutations use a process-bound lock and atomic record replacement so
|
|
53
|
+
* recovery state cannot be silently lost when two Pi processes race.
|
|
32
54
|
*/
|
|
33
55
|
export class DecisionSessionStore {
|
|
34
56
|
readonly #directory: string;
|
|
@@ -46,40 +68,143 @@ export class DecisionSessionStore {
|
|
|
46
68
|
return join(this.#directory, taskId);
|
|
47
69
|
}
|
|
48
70
|
|
|
49
|
-
async save(record:
|
|
71
|
+
async save(record: DecisionSessionRecordInput): Promise<void> {
|
|
50
72
|
assertTaskId(record.taskId);
|
|
51
73
|
const decisionSessionFile = resolve(record.decisionSessionFile);
|
|
52
74
|
await assertTaskDirectorySafe(this.#directory, record.taskId);
|
|
53
75
|
assertSessionPath(decisionSessionFile, this.#directory, record.taskId);
|
|
54
76
|
assertNoCredentialPath(decisionSessionFile);
|
|
55
77
|
assertNoCredentialPath(record.cwd);
|
|
56
|
-
|
|
57
|
-
const normalized: DecisionSessionRecord = {
|
|
58
|
-
...safeRecord,
|
|
59
|
-
version: 1,
|
|
60
|
-
updatedAt: safeRecord.updatedAt ?? new Date().toISOString(),
|
|
61
|
-
args: [...safeRecord.args],
|
|
62
|
-
decisionSessionFile,
|
|
63
|
-
};
|
|
64
|
-
await mkdir(this.#directory, { recursive: true, mode: 0o700 });
|
|
65
|
-
await chmod(this.#directory, 0o700);
|
|
66
|
-
const target = this.#recordPath(record.taskId);
|
|
67
|
-
const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
|
|
68
|
-
await writeJson(temporary, normalized);
|
|
69
|
-
await rename(temporary, target);
|
|
70
|
-
await chmod(target, 0o600);
|
|
78
|
+
await this.#withLock(() => this.#saveUnlocked({ ...record, decisionSessionFile }));
|
|
71
79
|
}
|
|
72
80
|
|
|
73
81
|
async close(taskId: string): Promise<void> {
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
82
|
+
assertTaskId(taskId);
|
|
83
|
+
await this.#withLock(async () => {
|
|
84
|
+
const record = await this.#loadUnlocked(taskId);
|
|
85
|
+
if (!record) return;
|
|
86
|
+
await this.#saveUnlocked({ ...record, state: "closed", updatedAt: new Date().toISOString() });
|
|
87
|
+
});
|
|
77
88
|
}
|
|
78
89
|
|
|
79
90
|
async update(taskId: string, patch: Partial<Pick<DecisionSessionRecord, "turn" | "repairRound" | "lastFindingSignature" | "updatedAt">>): Promise<void> {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
91
|
+
assertTaskId(taskId);
|
|
92
|
+
await this.#withLock(async () => {
|
|
93
|
+
const record = await this.#loadUnlocked(taskId);
|
|
94
|
+
if (!record || record.state !== "active") return;
|
|
95
|
+
await this.#saveUnlocked({ ...record, ...patch, updatedAt: patch.updatedAt ?? new Date().toISOString() });
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Claim an active record for one explicit recovery attempt. */
|
|
100
|
+
async beginRecovery(taskId: string): Promise<DecisionSessionRecord> {
|
|
101
|
+
assertTaskId(taskId);
|
|
102
|
+
return this.#withLock(async () => {
|
|
103
|
+
const record = await this.#loadUnlocked(taskId);
|
|
104
|
+
if (!record || record.state !== "active") throw new Error(`No active Decision Worker session: ${taskId}`);
|
|
105
|
+
if (record.recoveryState !== "ready" && record.recoveryState !== "interrupted") {
|
|
106
|
+
throw new Error(`Decision Worker recovery is already in progress or attached: ${taskId}`);
|
|
107
|
+
}
|
|
108
|
+
const next: DecisionSessionRecord = {
|
|
109
|
+
...record,
|
|
110
|
+
recoveryState: "starting",
|
|
111
|
+
recoveryAttempt: record.recoveryAttempt + 1,
|
|
112
|
+
recoveryOwnerPid: process.pid,
|
|
113
|
+
recoveryOwnerStartTime: await processStartTime(process.pid),
|
|
114
|
+
recoveryWorker: undefined,
|
|
115
|
+
updatedAt: new Date().toISOString(),
|
|
116
|
+
};
|
|
117
|
+
await this.#saveUnlocked(next);
|
|
118
|
+
return next;
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Persist the new Worker identity before exposing the recovered session. */
|
|
123
|
+
async recordRecoveryWorker(taskId: string, worker: DecisionRecoveryWorker): Promise<void> {
|
|
124
|
+
assertTaskId(taskId);
|
|
125
|
+
assertRecoveryWorker(worker);
|
|
126
|
+
await this.#withLock(async () => {
|
|
127
|
+
const record = await this.#loadUnlocked(taskId);
|
|
128
|
+
if (!record) throw new Error(`Decision Worker recovery record is missing: ${taskId}`);
|
|
129
|
+
if (record.state !== "active") throw new Error(`Decision Worker recovery record is closed: ${taskId}`);
|
|
130
|
+
if (record.recoveryState !== "starting") throw new Error(`Decision Worker recovery is not in startup state: ${taskId}`);
|
|
131
|
+
await this.#saveUnlocked({
|
|
132
|
+
...record,
|
|
133
|
+
recoveryState: "registered",
|
|
134
|
+
recoveryWorker: { ...worker },
|
|
135
|
+
updatedAt: new Date().toISOString(),
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Mark a registered recovery as an idle, human-controlled session. */
|
|
141
|
+
async markRecoveryIdle(taskId: string): Promise<void> {
|
|
142
|
+
assertTaskId(taskId);
|
|
143
|
+
await this.#withLock(async () => {
|
|
144
|
+
const record = await this.#loadUnlocked(taskId);
|
|
145
|
+
if (!record) throw new Error(`Decision Worker recovery record is missing: ${taskId}`);
|
|
146
|
+
if (record.state !== "active") throw new Error(`Decision Worker recovery record is closed: ${taskId}`);
|
|
147
|
+
if (record.recoveryState === "recovered_idle") return;
|
|
148
|
+
if (record.recoveryState !== "registered" && record.recoveryState !== "starting") {
|
|
149
|
+
throw new Error(`Decision Worker recovery cannot become idle from ${record.recoveryState}: ${taskId}`);
|
|
150
|
+
}
|
|
151
|
+
await this.#saveUnlocked({ ...record, recoveryState: "recovered_idle", updatedAt: new Date().toISOString() });
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Reconcile a stale attempt only after the cwd-lease takeover has proved the
|
|
157
|
+
* old supervisor/Worker boundary is gone. The owner identity check remains
|
|
158
|
+
* here as a second independent guard.
|
|
159
|
+
*/
|
|
160
|
+
async reconcileStaleRecovery(taskId: string): Promise<void> {
|
|
161
|
+
assertTaskId(taskId);
|
|
162
|
+
await this.#withLock(async () => {
|
|
163
|
+
const record = await this.#loadUnlocked(taskId);
|
|
164
|
+
if (!record || record.state !== "active") throw new Error(`Decision Worker recovery record is unavailable: ${taskId}`);
|
|
165
|
+
if (record.recoveryState === "ready" || record.recoveryState === "interrupted") return;
|
|
166
|
+
if (!record.recoveryOwnerPid || await processExists(record.recoveryOwnerPid)) {
|
|
167
|
+
throw new Error(`Decision Worker recovery owner is still live: ${taskId}`);
|
|
168
|
+
}
|
|
169
|
+
await this.#saveUnlocked({
|
|
170
|
+
...record,
|
|
171
|
+
recoveryState: "interrupted",
|
|
172
|
+
recoveryWorker: undefined,
|
|
173
|
+
updatedAt: new Date().toISOString(),
|
|
174
|
+
});
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Leave an active record visibly recoverable after an interrupted attempt. */
|
|
179
|
+
async markRecoveryInterrupted(taskId: string): Promise<void> {
|
|
180
|
+
assertTaskId(taskId);
|
|
181
|
+
await this.#withLock(async () => {
|
|
182
|
+
const record = await this.#loadUnlocked(taskId);
|
|
183
|
+
if (!record || record.state !== "active") return;
|
|
184
|
+
if (record.recoveryState === "interrupted") return;
|
|
185
|
+
await this.#saveUnlocked({
|
|
186
|
+
...record,
|
|
187
|
+
recoveryState: "interrupted",
|
|
188
|
+
recoveryWorker: undefined,
|
|
189
|
+
updatedAt: new Date().toISOString(),
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Reset a claimed record only when no Worker was spawned. */
|
|
195
|
+
async resetRecovery(taskId: string): Promise<void> {
|
|
196
|
+
assertTaskId(taskId);
|
|
197
|
+
await this.#withLock(async () => {
|
|
198
|
+
const record = await this.#loadUnlocked(taskId);
|
|
199
|
+
if (!record || record.state !== "active") return;
|
|
200
|
+
if (record.recoveryState === "ready") return;
|
|
201
|
+
await this.#saveUnlocked({
|
|
202
|
+
...record,
|
|
203
|
+
recoveryState: "ready",
|
|
204
|
+
recoveryWorker: undefined,
|
|
205
|
+
updatedAt: new Date().toISOString(),
|
|
206
|
+
});
|
|
207
|
+
});
|
|
83
208
|
}
|
|
84
209
|
|
|
85
210
|
async sessionFileExists(taskId: string): Promise<boolean> {
|
|
@@ -96,29 +221,25 @@ export class DecisionSessionStore {
|
|
|
96
221
|
|
|
97
222
|
async load(taskId: string): Promise<DecisionSessionRecord | undefined> {
|
|
98
223
|
assertTaskId(taskId);
|
|
99
|
-
|
|
100
|
-
const value = JSON.parse(await readFile(this.#recordPath(taskId), "utf8")) as Partial<DecisionSessionRecord>;
|
|
101
|
-
assertNoCredentialPath(typeof value.decisionSessionFile === "string" ? resolve(value.decisionSessionFile) : "");
|
|
102
|
-
return normalizeRecord(redactRecord(value), this.#directory, taskId);
|
|
103
|
-
} catch (error) {
|
|
104
|
-
if (error instanceof Error && /ENOENT/u.test(error.message)) return undefined;
|
|
105
|
-
throw error;
|
|
106
|
-
}
|
|
224
|
+
return this.#loadUnlocked(taskId);
|
|
107
225
|
}
|
|
108
226
|
|
|
109
227
|
async list(options: { activeOnly?: boolean } = {}): Promise<DecisionSessionRecord[]> {
|
|
110
228
|
try {
|
|
229
|
+
const directoryInfo = await lstat(this.#directory);
|
|
230
|
+
if (!directoryInfo.isDirectory() || directoryInfo.isSymbolicLink()) throw new Error("Decision Worker session registry is not a real directory");
|
|
111
231
|
const names = await readdir(this.#directory);
|
|
112
232
|
const records: DecisionSessionRecord[] = [];
|
|
113
233
|
for (const name of names.filter((item) => item.endsWith(".json"))) {
|
|
114
234
|
try {
|
|
115
|
-
const value = JSON.parse(await readFile(join(this.#directory, name), "utf8")) as Partial<DecisionSessionRecord>;
|
|
116
|
-
assertNoCredentialPath(typeof value.decisionSessionFile === "string" ? resolve(value.decisionSessionFile) : "");
|
|
117
235
|
const expectedTaskId = name.slice(0, -".json".length);
|
|
236
|
+
assertTaskId(expectedTaskId);
|
|
237
|
+
const value = JSON.parse(await readRecordFile(join(this.#directory, name))) as Partial<DecisionSessionRecord>;
|
|
238
|
+
assertNoCredentialPath(typeof value.decisionSessionFile === "string" ? resolve(value.decisionSessionFile) : "");
|
|
118
239
|
const record = normalizeRecord(redactRecord(value), this.#directory, expectedTaskId);
|
|
119
240
|
if (!options.activeOnly || record.state === "active") records.push(record);
|
|
120
241
|
} catch {
|
|
121
|
-
// A torn or manually edited registry record is not recoverable.
|
|
242
|
+
// A torn, misnamed, or manually edited registry record is not recoverable.
|
|
122
243
|
}
|
|
123
244
|
}
|
|
124
245
|
return records.sort((first, second) => second.updatedAt.localeCompare(first.updatedAt));
|
|
@@ -128,6 +249,82 @@ export class DecisionSessionStore {
|
|
|
128
249
|
}
|
|
129
250
|
}
|
|
130
251
|
|
|
252
|
+
async #loadUnlocked(taskId: string): Promise<DecisionSessionRecord | undefined> {
|
|
253
|
+
try {
|
|
254
|
+
const value = JSON.parse(await readRecordFile(this.#recordPath(taskId))) as Partial<DecisionSessionRecord>;
|
|
255
|
+
assertNoCredentialPath(typeof value.decisionSessionFile === "string" ? resolve(value.decisionSessionFile) : "");
|
|
256
|
+
return normalizeRecord(redactRecord(value), this.#directory, taskId);
|
|
257
|
+
} catch (error) {
|
|
258
|
+
if (error instanceof Error && /ENOENT/u.test(error.message)) return undefined;
|
|
259
|
+
throw error;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
async #saveUnlocked(record: DecisionSessionRecordInput | DecisionSessionRecord): Promise<void> {
|
|
264
|
+
assertTaskId(record.taskId);
|
|
265
|
+
const decisionSessionFile = resolve(record.decisionSessionFile);
|
|
266
|
+
await assertTaskDirectorySafe(this.#directory, record.taskId);
|
|
267
|
+
assertSessionPath(decisionSessionFile, this.#directory, record.taskId);
|
|
268
|
+
assertNoCredentialPath(decisionSessionFile);
|
|
269
|
+
assertNoCredentialPath(record.cwd);
|
|
270
|
+
assertRecoveryState(record.recoveryState);
|
|
271
|
+
assertRecoveryAttempt(record.recoveryAttempt);
|
|
272
|
+
assertRecoveryOwner(record.recoveryOwnerPid, record.recoveryOwnerStartTime);
|
|
273
|
+
if (record.recoveryWorker) assertRecoveryWorker(record.recoveryWorker);
|
|
274
|
+
const safeRecord = redactRecord(record);
|
|
275
|
+
const normalized: DecisionSessionRecord = {
|
|
276
|
+
...safeRecord,
|
|
277
|
+
version: 1,
|
|
278
|
+
recoveryState: safeRecord.recoveryState ?? "ready",
|
|
279
|
+
recoveryAttempt: safeRecord.recoveryAttempt ?? 0,
|
|
280
|
+
...(safeRecord.recoveryOwnerPid !== undefined ? { recoveryOwnerPid: safeRecord.recoveryOwnerPid } : {}),
|
|
281
|
+
...(safeRecord.recoveryOwnerStartTime !== undefined ? { recoveryOwnerStartTime: safeRecord.recoveryOwnerStartTime } : {}),
|
|
282
|
+
updatedAt: safeRecord.updatedAt ?? new Date().toISOString(),
|
|
283
|
+
args: [...safeRecord.args],
|
|
284
|
+
decisionSessionFile,
|
|
285
|
+
...(safeRecord.recoveryWorker ? { recoveryWorker: { ...safeRecord.recoveryWorker } } : {}),
|
|
286
|
+
} as DecisionSessionRecord;
|
|
287
|
+
await this.#ensureDirectory();
|
|
288
|
+
const target = this.#recordPath(record.taskId);
|
|
289
|
+
const temporary = `${target}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
|
290
|
+
await writeJson(temporary, normalized);
|
|
291
|
+
await rename(temporary, target);
|
|
292
|
+
await chmod(target, 0o600);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
async #withLock<T>(operation: () => Promise<T>): Promise<T> {
|
|
296
|
+
await this.#ensureDirectory();
|
|
297
|
+
const lockPath = join(this.#directory, ".lock");
|
|
298
|
+
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
299
|
+
while (true) {
|
|
300
|
+
try {
|
|
301
|
+
await mkdir(lockPath, { mode: 0o700 });
|
|
302
|
+
await writeFile(join(lockPath, "owner.json"), JSON.stringify({
|
|
303
|
+
pid: process.pid,
|
|
304
|
+
at: new Date().toISOString(),
|
|
305
|
+
}), { encoding: "utf8", mode: 0o600 });
|
|
306
|
+
break;
|
|
307
|
+
} catch (error) {
|
|
308
|
+
if (!(error instanceof Error) || !/EEXIST/u.test(error.message)) throw error;
|
|
309
|
+
if (await removeStaleLock(lockPath)) continue;
|
|
310
|
+
if (Date.now() >= deadline) throw new Error(`Decision Worker session registry lock timeout: ${lockPath}`);
|
|
311
|
+
await delay(25);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
try {
|
|
315
|
+
return await operation();
|
|
316
|
+
} finally {
|
|
317
|
+
await rm(lockPath, { recursive: true, force: true });
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
async #ensureDirectory(): Promise<void> {
|
|
322
|
+
await mkdir(this.#directory, { recursive: true, mode: 0o700 });
|
|
323
|
+
const info = await lstat(this.#directory);
|
|
324
|
+
if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("Decision Worker session registry is not a real directory");
|
|
325
|
+
await chmod(this.#directory, 0o700);
|
|
326
|
+
}
|
|
327
|
+
|
|
131
328
|
#recordPath(taskId: string): string {
|
|
132
329
|
assertTaskId(taskId);
|
|
133
330
|
return join(this.#directory, `${taskId}.json`);
|
|
@@ -135,7 +332,13 @@ export class DecisionSessionStore {
|
|
|
135
332
|
}
|
|
136
333
|
|
|
137
334
|
async function writeJson(path: string, value: unknown): Promise<void> {
|
|
138
|
-
await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
335
|
+
await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
async function readRecordFile(path: string): Promise<string> {
|
|
339
|
+
const info = await lstat(path);
|
|
340
|
+
if (!info.isFile() || info.isSymbolicLink()) throw new Error("Decision Worker session registry record is not a regular file");
|
|
341
|
+
return readFile(path, "utf8");
|
|
139
342
|
}
|
|
140
343
|
|
|
141
344
|
function normalizeRecord(value: Partial<DecisionSessionRecord>, directory: string, expectedTaskId?: string): DecisionSessionRecord {
|
|
@@ -146,15 +349,24 @@ function normalizeRecord(value: Partial<DecisionSessionRecord>, directory: strin
|
|
|
146
349
|
|| typeof value.decisionSessionFile !== "string" || (value.state !== "active" && value.state !== "closed")
|
|
147
350
|
|| typeof value.updatedAt !== "string" || !Number.isFinite(Date.parse(value.updatedAt))
|
|
148
351
|
|| !validLimit(value.maxTurns, 0) || !validLimit(value.deadlineMs, 0) || !validLimit(value.noOutputTimeoutMs, 0)
|
|
149
|
-
|| !validLimit(value.turn, 0) || !validLimit(value.repairRound, 0)
|
|
352
|
+
|| !validLimit(value.turn, 0) || !validLimit(value.repairRound, 0) || !validLimit(value.recoveryAttempt, 0)
|
|
353
|
+
|| (value.recoveryOwnerPid !== undefined && (!Number.isSafeInteger(value.recoveryOwnerPid) || value.recoveryOwnerPid < 1))
|
|
354
|
+
|| (value.recoveryOwnerStartTime !== undefined && (typeof value.recoveryOwnerStartTime !== "string" || !/^\d+$/u.test(value.recoveryOwnerStartTime)))
|
|
355
|
+
|| (value.recoveryState !== undefined && !isRecoveryState(value.recoveryState))
|
|
150
356
|
|| (value.lastFindingSignature !== undefined && (typeof value.lastFindingSignature !== "string" || value.lastFindingSignature.length > 128))
|
|
151
|
-
|| (value.startedAt !== undefined && (typeof value.startedAt !== "string" || !Number.isFinite(Date.parse(value.startedAt))))
|
|
357
|
+
|| (value.startedAt !== undefined && (typeof value.startedAt !== "string" || !Number.isFinite(Date.parse(value.startedAt))))
|
|
358
|
+
|| (value.recoveryWorker !== undefined && !isRecoveryWorker(value.recoveryWorker))) {
|
|
152
359
|
throw new Error("invalid Decision Worker session record");
|
|
153
360
|
}
|
|
154
361
|
const decisionSessionFile = resolve(value.decisionSessionFile);
|
|
155
362
|
assertSessionPath(decisionSessionFile, directory, value.taskId);
|
|
156
363
|
assertNoCredentialPath(decisionSessionFile);
|
|
157
364
|
assertNoCredentialPath(value.cwd);
|
|
365
|
+
const recoveryWorker = value.recoveryWorker && {
|
|
366
|
+
id: value.recoveryWorker.id,
|
|
367
|
+
...(value.recoveryWorker.pid !== undefined ? { pid: value.recoveryWorker.pid } : {}),
|
|
368
|
+
startedAt: value.recoveryWorker.startedAt,
|
|
369
|
+
};
|
|
158
370
|
return {
|
|
159
371
|
version: 1,
|
|
160
372
|
taskId: value.taskId,
|
|
@@ -173,6 +385,11 @@ function normalizeRecord(value: Partial<DecisionSessionRecord>, directory: strin
|
|
|
173
385
|
repairRound: value.repairRound ?? 0,
|
|
174
386
|
...(typeof value.lastFindingSignature === "string" ? { lastFindingSignature: value.lastFindingSignature } : {}),
|
|
175
387
|
state: value.state,
|
|
388
|
+
recoveryState: value.recoveryState ?? "ready",
|
|
389
|
+
recoveryAttempt: value.recoveryAttempt ?? 0,
|
|
390
|
+
...(value.recoveryOwnerPid !== undefined ? { recoveryOwnerPid: value.recoveryOwnerPid } : {}),
|
|
391
|
+
...(typeof value.recoveryOwnerStartTime === "string" ? { recoveryOwnerStartTime: value.recoveryOwnerStartTime } : {}),
|
|
392
|
+
...(recoveryWorker ? { recoveryWorker } : {}),
|
|
176
393
|
updatedAt: value.updatedAt,
|
|
177
394
|
};
|
|
178
395
|
}
|
|
@@ -207,6 +424,95 @@ function assertSessionPath(sessionFile: string, directory: string, taskId: strin
|
|
|
207
424
|
if (dirname(sessionFile) !== taskDirectory) throw new Error("Decision Worker session file must be a direct child of its task session directory");
|
|
208
425
|
}
|
|
209
426
|
|
|
427
|
+
function isRecoveryState(value: unknown): value is DecisionRecoveryState {
|
|
428
|
+
return value === "ready" || value === "starting" || value === "registered" || value === "recovered_idle" || value === "interrupted";
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
function assertRecoveryState(value: unknown): asserts value is DecisionRecoveryState | undefined {
|
|
432
|
+
if (value !== undefined && !isRecoveryState(value)) throw new Error("invalid Decision Worker recovery state");
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function isRecoveryWorker(value: unknown): value is DecisionRecoveryWorker {
|
|
436
|
+
if (!value || typeof value !== "object") return false;
|
|
437
|
+
const worker = value as Partial<DecisionRecoveryWorker>;
|
|
438
|
+
return typeof worker.id === "string" && worker.id.length > 0 && worker.id.length <= 256
|
|
439
|
+
&& (worker.pid === undefined || (Number.isSafeInteger(worker.pid) && worker.pid > 0))
|
|
440
|
+
&& typeof worker.startedAt === "string" && Number.isFinite(Date.parse(worker.startedAt));
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function assertRecoveryWorker(value: DecisionRecoveryWorker): void {
|
|
444
|
+
if (!isRecoveryWorker(value)) throw new Error("invalid Decision Worker recovery worker identity");
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
function assertRecoveryAttempt(value: unknown): asserts value is number | undefined {
|
|
448
|
+
if (value !== undefined && (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)) throw new Error("invalid Decision Worker recovery attempt");
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
function assertRecoveryOwner(pid: unknown, startTime: unknown): void {
|
|
452
|
+
if (pid !== undefined && (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid < 1)) throw new Error("invalid Decision Worker recovery owner pid");
|
|
453
|
+
if (startTime !== undefined && (typeof startTime !== "string" || !/^\d+$/u.test(startTime))) throw new Error("invalid Decision Worker recovery owner start time");
|
|
454
|
+
}
|
|
455
|
+
|
|
210
456
|
function validLimit(value: unknown, minimum: number): boolean {
|
|
211
457
|
return value === undefined || (typeof value === "number" && Number.isSafeInteger(value) && value >= minimum);
|
|
212
458
|
}
|
|
459
|
+
|
|
460
|
+
async function removeStaleLock(lockPath: string): Promise<boolean> {
|
|
461
|
+
try {
|
|
462
|
+
const info = await lstat(lockPath);
|
|
463
|
+
if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("Decision Worker session registry lock is not a real directory");
|
|
464
|
+
if (Date.now() - info.mtimeMs < STALE_LOCK_MS) return false;
|
|
465
|
+
const ownerPath = join(lockPath, "owner.json");
|
|
466
|
+
try {
|
|
467
|
+
const ownerInfo = await lstat(ownerPath);
|
|
468
|
+
if (!ownerInfo.isFile() || ownerInfo.isSymbolicLink()) throw new Error("Decision Worker session registry lock owner is not a regular file");
|
|
469
|
+
} catch (error) {
|
|
470
|
+
if (error instanceof Error && /ENOENT/u.test(error.message)) {
|
|
471
|
+
// An old/incomplete lock is reclaimable after the grace period.
|
|
472
|
+
} else {
|
|
473
|
+
throw error;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
let owner: { pid?: unknown } = {};
|
|
477
|
+
try { owner = JSON.parse(await readFile(ownerPath, "utf8")) as { pid?: unknown }; }
|
|
478
|
+
catch { /* an old/incomplete lock is reclaimable after the grace period */ }
|
|
479
|
+
if (typeof owner.pid === "number" && Number.isSafeInteger(owner.pid) && owner.pid > 0) {
|
|
480
|
+
try {
|
|
481
|
+
process.kill(owner.pid, 0);
|
|
482
|
+
return false;
|
|
483
|
+
} catch (error) {
|
|
484
|
+
if (error instanceof Error && /EPERM/u.test(error.message)) return false;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
await rm(lockPath, { recursive: true, force: true });
|
|
488
|
+
return true;
|
|
489
|
+
} catch (error) {
|
|
490
|
+
if (error instanceof Error && /ENOENT/u.test(error.message)) return true;
|
|
491
|
+
return false;
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
async function processExists(pid: number): Promise<boolean> {
|
|
496
|
+
try {
|
|
497
|
+
process.kill(pid, 0);
|
|
498
|
+
return true;
|
|
499
|
+
} catch (error) {
|
|
500
|
+
return error instanceof Error && /EPERM/u.test(error.message);
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
async function processStartTime(pid: number): Promise<string | undefined> {
|
|
505
|
+
try {
|
|
506
|
+
const statText = await readFile(`/proc/${pid}/stat`, "utf8");
|
|
507
|
+
const closeParen = statText.lastIndexOf(")");
|
|
508
|
+
const fields = closeParen >= 0 ? statText.slice(closeParen + 2).trim().split(/\s+/u) : [];
|
|
509
|
+
const startTime = fields[19];
|
|
510
|
+
return startTime && /^\d+$/u.test(startTime) ? startTime : undefined;
|
|
511
|
+
} catch {
|
|
512
|
+
return undefined;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
function delay(ms: number): Promise<void> {
|
|
517
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
518
|
+
}
|
package/src/decision-worker.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { constants as fsConstants } from "node:fs";
|
|
3
|
+
import { dirname, resolve } from "node:path";
|
|
4
|
+
import { lstat, open, rename, rm, writeFile, type FileHandle } from "node:fs/promises";
|
|
2
5
|
import { createAgentSession, DefaultResourceLoader, getAgentDir, SessionManager, type AgentSession } from "@earendil-works/pi-coding-agent";
|
|
3
6
|
import type { TaskSpec, WorkerEvent } from "./types.ts";
|
|
4
7
|
import { redactSensitive } from "./redaction.ts";
|
|
@@ -299,18 +302,52 @@ function sanitizeSessionHeader(manager: SessionManager): void {
|
|
|
299
302
|
}
|
|
300
303
|
|
|
301
304
|
async function sanitizeSessionFile(path: string): Promise<void> {
|
|
302
|
-
const
|
|
305
|
+
const securePath = resolve(path);
|
|
306
|
+
const contents = await readSessionFileSecure(securePath);
|
|
303
307
|
const sanitized = contents.split("\n").map((line) => {
|
|
304
308
|
if (!line.trim()) return line;
|
|
305
309
|
try { return JSON.stringify(redactSensitive(JSON.parse(line))); }
|
|
306
310
|
catch { return String(redactSensitive(line)); }
|
|
307
311
|
}).join("\n");
|
|
308
|
-
if (sanitized
|
|
312
|
+
if (sanitized === contents) return;
|
|
313
|
+
|
|
314
|
+
await assertSessionParentDirectory(securePath);
|
|
315
|
+
const temporary = `${securePath}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
|
|
316
|
+
try {
|
|
317
|
+
// Write beside the validated session and replace the name atomically. The
|
|
318
|
+
// temporary file is exclusive so a pre-created symlink cannot be followed.
|
|
319
|
+
await writeFile(temporary, sanitized, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
320
|
+
await assertSessionParentDirectory(securePath);
|
|
321
|
+
await rename(temporary, securePath);
|
|
322
|
+
} finally {
|
|
323
|
+
await rm(temporary, { force: true }).catch(() => {});
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async function readSessionFileSecure(path: string): Promise<string> {
|
|
328
|
+
const securePath = resolve(path);
|
|
329
|
+
await assertSessionParentDirectory(securePath);
|
|
330
|
+
if (typeof fsConstants.O_NOFOLLOW !== "number") throw new Error("secure Decision Worker session-file opening is unavailable");
|
|
331
|
+
let handle: FileHandle | undefined;
|
|
332
|
+
try {
|
|
333
|
+
handle = await open(securePath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
334
|
+
const info = await handle.stat();
|
|
335
|
+
if (!info.isFile() || info.size <= 0) throw new Error("Decision Worker session file is not a non-empty regular file");
|
|
336
|
+
return await handle.readFile("utf8");
|
|
337
|
+
} finally {
|
|
338
|
+
await handle?.close().catch(() => {});
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
async function assertSessionParentDirectory(path: string): Promise<void> {
|
|
343
|
+
const parent = dirname(path);
|
|
344
|
+
const info = await lstat(parent);
|
|
345
|
+
if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("Decision Worker session directory is not a real directory");
|
|
309
346
|
}
|
|
310
347
|
|
|
311
348
|
async function fileExists(path: string): Promise<boolean> {
|
|
312
349
|
try {
|
|
313
|
-
await
|
|
350
|
+
await readSessionFileSecure(path);
|
|
314
351
|
return true;
|
|
315
352
|
} catch (error) {
|
|
316
353
|
if (error instanceof Error && /ENOENT/u.test(error.message)) return false;
|