pi-async 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +188 -0
- package/README.md +93 -19
- package/docs/CONTRACT.md +301 -61
- package/extensions/index.ts +221 -38
- package/package.json +6 -3
- package/src/core/bus.ts +11 -3
- package/src/core/duration.ts +20 -0
- package/src/core/env.ts +134 -0
- package/src/core/notify.ts +82 -18
- package/src/core/output.ts +48 -2
- package/src/core/registry.ts +646 -95
- package/src/core/spawn.ts +95 -20
- package/src/core/task.ts +38 -8
- package/src/core/watch.ts +49 -13
- package/src/guard/settle-guard.ts +152 -46
- package/src/tools/index.ts +221 -27
- package/src/tui/panel.ts +375 -80
- package/src/tui/rail.ts +117 -33
- package/src/tui/status.ts +102 -25
- package/src/tui/width.ts +53 -0
package/src/core/registry.ts
CHANGED
|
@@ -1,19 +1,65 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { mkdirSync, readFileSync, readdirSync, renameSync,
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
import { killTree, spawnDetached } from "./spawn.ts";
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
2
|
+
import { closeSync, copyFileSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readdirSync, readSync, realpathSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { join, resolve, sep } from "node:path";
|
|
4
|
+
import { killTree, spawnDetached, streamPaths } from "./spawn.ts";
|
|
5
|
+
import { stripLogControls, truncateUtf8Bytes } from "./output.ts";
|
|
6
|
+
import { assertSafeEnvKeys, envNumber } from "./env.ts";
|
|
7
|
+
import { capName, deriveName, isTaskStatus, isTerminal, type FailureKind, type KillKind, type RunOptions, type SyncResult, type TaskMode, type TaskSnapshot, type TaskStatus, type WatchSpec, type WaitOutcome } from "./task.ts";
|
|
8
|
+
import { evalConditions, validateWatchSpec, WatchLoop, type WatchOutput } from "./watch.ts";
|
|
8
9
|
|
|
9
10
|
export interface RegistryEvents {
|
|
10
11
|
onFinalize(snapshot: TaskSnapshot): void;
|
|
11
12
|
onStateChange(snapshot: TaskSnapshot): void;
|
|
12
13
|
}
|
|
13
|
-
export interface RegistryConfig { dataDir: string; maxTasks?: number; maxOutputBytes?: number; }
|
|
14
|
+
export interface RegistryConfig { dataDir: string; cwd?: string; maxTasks?: number; maxOutputBytes?: number; retentionMs?: number; retentionCount?: number; }
|
|
15
|
+
export interface ListOptions { includeHistory?: boolean; status?: TaskStatus | "active"; name?: string; limit?: number; }
|
|
16
|
+
export type LogStream = "merged" | "stdout" | "stderr";
|
|
17
|
+
export interface LogSlice { text: string; truncated: boolean; fullPath: string; missing?: boolean; }
|
|
18
|
+
|
|
19
|
+
/** Registry fan-out event (REV-14/TUI refresh): `state` for every mutation, `finalize` once per terminal transition. */
|
|
20
|
+
export type RegistryEvent =
|
|
21
|
+
| { type: "state"; snapshot: TaskSnapshot }
|
|
22
|
+
| { type: "finalize"; snapshot: TaskSnapshot };
|
|
23
|
+
|
|
14
24
|
interface LiveTask { outputFile: string; exit?: Promise<{ code: number }>; timer?: ReturnType<typeof setTimeout>; delayTimer?: ReturnType<typeof setTimeout>; watcher?: ReturnType<typeof setInterval>; killKind?: KillKind; processExited?: boolean; }
|
|
15
25
|
interface Waiter { ids: Set<string>; resolve: () => void; }
|
|
16
26
|
const DEFAULT_MAX_OUTPUT = 20 * 1024 * 1024;
|
|
27
|
+
const DEFAULT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
|
|
28
|
+
const DEFAULT_RETENTION_COUNT = 100;
|
|
29
|
+
const MAX_LIST_LIMIT = 50;
|
|
30
|
+
const DEFAULT_RESEND_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
31
|
+
const DEFAULT_RESEND_MAX_ATTEMPTS = 2;
|
|
32
|
+
/** Default read window for `logs`; callers (tools/TUI) own the policy above this. */
|
|
33
|
+
const DEFAULT_LOG_BYTES = 50 * 1024;
|
|
34
|
+
/** Defensive ceiling for a single `logs` read: an accidental huge window is clamped, not honoured. */
|
|
35
|
+
const MAX_LOG_WINDOW = 8 * 1024 * 1024;
|
|
36
|
+
/** Artifacts owned by one task; migrated wholesale between data dirs. */
|
|
37
|
+
const ARTIFACT_SUFFIXES = [".json", ".output", ".stdout", ".stderr"] as const;
|
|
38
|
+
|
|
39
|
+
/** Read `length` bytes at `start`, tolerating short reads from a partially written file. */
|
|
40
|
+
function readExactly(handle: number, start: number, length: number): Buffer {
|
|
41
|
+
const buffer = Buffer.alloc(length);
|
|
42
|
+
let read = 0;
|
|
43
|
+
while (read < length) {
|
|
44
|
+
const got = readSync(handle, buffer, read, length - read, start + read);
|
|
45
|
+
if (got <= 0) break;
|
|
46
|
+
read += got;
|
|
47
|
+
}
|
|
48
|
+
return read === length ? buffer : buffer.subarray(0, read);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** UTF-8-safe decode of an already bounded window; mirrors `truncateUtf8Bytes` without the whole file. */
|
|
52
|
+
function truncateUtf8Window(buffer: Buffer, maxBytes: number, tail: boolean, truncated: boolean): string {
|
|
53
|
+
if (!truncated) return buffer.toString("utf8");
|
|
54
|
+
if (tail) {
|
|
55
|
+
let start = 0;
|
|
56
|
+
while (start < buffer.length && (buffer[start]! & 0xc0) === 0x80) start++;
|
|
57
|
+
return buffer.subarray(start).toString("utf8");
|
|
58
|
+
}
|
|
59
|
+
let end = Math.min(maxBytes, buffer.length);
|
|
60
|
+
while (end > 0 && (buffer[end]! & 0xc0) === 0x80) end--;
|
|
61
|
+
return buffer.subarray(0, end).toString("utf8");
|
|
62
|
+
}
|
|
17
63
|
|
|
18
64
|
export class TaskRegistry {
|
|
19
65
|
private readonly tasks = new Map<string, TaskSnapshot>();
|
|
@@ -21,28 +67,86 @@ export class TaskRegistry {
|
|
|
21
67
|
private readonly waiters = new Set<Waiter>();
|
|
22
68
|
private readonly notifyClaims = new Set<string>();
|
|
23
69
|
private readonly watchLoops = new Map<string, WatchLoop>();
|
|
70
|
+
private readonly listeners = new Set<(event: RegistryEvent) => void>();
|
|
24
71
|
private events: RegistryEvents = { onFinalize: () => undefined, onStateChange: () => undefined };
|
|
25
72
|
private readonly maxTasks: number;
|
|
26
73
|
private readonly maxOutputBytes: number;
|
|
74
|
+
private readonly retentionMs: number;
|
|
75
|
+
private readonly retentionCount: number;
|
|
76
|
+
private config: RegistryConfig;
|
|
27
77
|
private closed = false;
|
|
78
|
+
/** Set by `suspend()`: this generation is retired and must not settle adopted work (REV-17). */
|
|
79
|
+
private suspended = false;
|
|
80
|
+
private reloading = false;
|
|
28
81
|
private shutdownPromise?: Promise<void>;
|
|
82
|
+
private configured = false;
|
|
83
|
+
private dataDirReady = false;
|
|
29
84
|
|
|
30
|
-
private readonly config: RegistryConfig;
|
|
31
85
|
constructor(config: RegistryConfig) {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
this.
|
|
35
|
-
|
|
86
|
+
// NOTE (D-04): the constructor no longer creates the data dir — the factory must stay
|
|
87
|
+
// side-effect free. `persist()` and `configure()` create it on demand.
|
|
88
|
+
this.config = { ...config };
|
|
89
|
+
this.maxTasks = config.maxTasks ?? envNumber("PI_ASYNC_MAX_TASKS", 32, { integer: true, min: 1 });
|
|
90
|
+
this.maxOutputBytes = config.maxOutputBytes ?? envNumber("PI_ASYNC_MAX_OUTPUT_BYTES", DEFAULT_MAX_OUTPUT, { integer: true, min: 0 });
|
|
91
|
+
this.retentionMs = config.retentionMs ?? envNumber("PI_ASYNC_RETENTION_MS", DEFAULT_RETENTION_MS, { integer: true, min: 0 });
|
|
92
|
+
this.retentionCount = config.retentionCount ?? envNumber("PI_ASYNC_RETENTION_COUNT", DEFAULT_RETENTION_COUNT, { integer: true, min: 0 });
|
|
36
93
|
}
|
|
37
94
|
|
|
38
95
|
setEvents(events: RegistryEvents): void { this.events = events; }
|
|
39
96
|
|
|
97
|
+
/** Subscribe to registry state/finalize events; returns an unsubscribe function. */
|
|
98
|
+
subscribe(listener: (event: RegistryEvent) => void): () => void {
|
|
99
|
+
this.listeners.add(listener);
|
|
100
|
+
return () => { this.listeners.delete(listener); };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Session-scoped configuration (REV-13/REV-17). Idempotent.
|
|
105
|
+
*
|
|
106
|
+
* First call switches this registry to the session data dir + cwd and loads persisted
|
|
107
|
+
* tasks. Later calls only react to changes; a data-dir change migrates task artifacts
|
|
108
|
+
* instead of dropping them. Migration failures keep the old dir and never throw (D-05):
|
|
109
|
+
* this runs inside a `session_start` handler where a throw becomes extension noise.
|
|
110
|
+
*/
|
|
111
|
+
configure(opts: { dataDir?: string; cwd?: string }): void {
|
|
112
|
+
const first = !this.configured;
|
|
113
|
+
this.configured = true;
|
|
114
|
+
if (opts.cwd !== undefined && opts.cwd.length > 0) this.config = { ...this.config, cwd: opts.cwd };
|
|
115
|
+
const nextDataDir = opts.dataDir;
|
|
116
|
+
const changed = nextDataDir !== undefined && nextDataDir !== this.config.dataDir;
|
|
117
|
+
if (changed) {
|
|
118
|
+
if (this.tasks.size > 0) {
|
|
119
|
+
if (!this.migrateDataDir(nextDataDir)) {
|
|
120
|
+
this.ensureDataDirQuietly();
|
|
121
|
+
if (first) {
|
|
122
|
+
void this.reloadFromDisk();
|
|
123
|
+
this.reapOrphans();
|
|
124
|
+
}
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
} else {
|
|
128
|
+
this.switchDataDir(nextDataDir);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// D-04: the session data dir is created here or on first persist — never in the constructor.
|
|
132
|
+
this.ensureDataDirQuietly();
|
|
133
|
+
if (first || changed) {
|
|
134
|
+
// reloadFromDisk() has no await points, so it is fully applied when it returns; reaping
|
|
135
|
+
// keeps this generation from adopting snapshots whose processes are already gone (#44).
|
|
136
|
+
void this.reloadFromDisk();
|
|
137
|
+
this.reapOrphans();
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
40
141
|
submit(opts: RunOptions): TaskSnapshot {
|
|
41
142
|
if (opts.mode && opts.mode !== "background") throw new Error("submit only accepts background mode");
|
|
42
143
|
return this.createAndStart(opts, "background");
|
|
43
144
|
}
|
|
44
145
|
|
|
45
146
|
scheduleDelayed(opts: RunOptions, delayMs: number): TaskSnapshot {
|
|
147
|
+
// Fail fast: delayed tasks spawn later, so dangerous env overrides must be rejected at submit
|
|
148
|
+
// time exactly like background/sync (spawnDetached would otherwise only fail at due time).
|
|
149
|
+
assertSafeEnvKeys(opts.env, "safe");
|
|
46
150
|
this.assertAdmission(opts);
|
|
47
151
|
if (!Number.isFinite(delayMs) || delayMs < 0) throw new Error("delayMs must be a non-negative finite number");
|
|
48
152
|
this.validateRunWatch(opts, "delayed");
|
|
@@ -50,8 +154,9 @@ export class TaskRegistry {
|
|
|
50
154
|
const taskId = randomUUID();
|
|
51
155
|
const outputFile = join(this.config.dataDir, `${taskId}.output`);
|
|
52
156
|
const task: TaskSnapshot = {
|
|
53
|
-
taskId, name: opts.name
|
|
157
|
+
taskId, name: opts.name ? capName(opts.name) : deriveName(opts.command), command: opts.command, mode: "delayed", status: "pending",
|
|
54
158
|
createdAt, dueAt: createdAt + delayMs, taskTimeoutMs: opts.taskTimeoutMs, notified: false, outputConsumed: false,
|
|
159
|
+
cwd: opts.cwd ?? this.config.cwd ?? process.cwd(),
|
|
55
160
|
...(opts.watch ? { watch: opts.watch } : {}),
|
|
56
161
|
metadata: { notifyOnCompletion: opts.notifyOnCompletion ?? true, triggerOnCompletion: opts.triggerOnCompletion ?? true, outputFile, ...(opts.env ? { delayedEnv: opts.env } : {}) },
|
|
57
162
|
};
|
|
@@ -72,18 +177,41 @@ export class TaskRegistry {
|
|
|
72
177
|
let promoted = false;
|
|
73
178
|
try {
|
|
74
179
|
const outcome = await this.wait([task.taskId], timeoutMs, signal);
|
|
75
|
-
|
|
180
|
+
// Timeout / Esc: stop waiting. promote → background + promoted flag; abort → Esc semantics
|
|
181
|
+
// (keep process, bookkeeping mode becomes background so the receipt is not "sync+running").
|
|
182
|
+
if ("notReady" in outcome && !isTerminal(task.status)) {
|
|
76
183
|
task.mode = "background";
|
|
77
|
-
promoted = true;
|
|
184
|
+
if (onTimeout === "promote") promoted = true;
|
|
78
185
|
this.emitState(task);
|
|
79
186
|
}
|
|
80
187
|
} catch (error) {
|
|
81
188
|
if (!signal?.aborted) throw error;
|
|
189
|
+
if (!isTerminal(task.status) && task.mode === "sync") {
|
|
190
|
+
task.mode = "background";
|
|
191
|
+
this.emitState(task);
|
|
192
|
+
}
|
|
82
193
|
}
|
|
83
194
|
return this.syncResult(task, promoted);
|
|
84
195
|
}
|
|
85
196
|
|
|
86
|
-
|
|
197
|
+
/**
|
|
198
|
+
* Resolve a user-supplied task reference. Ids win; otherwise an exact (case-insensitive)
|
|
199
|
+
* name is required, and duplicate names are an error listing candidate ids (REV-15).
|
|
200
|
+
*/
|
|
201
|
+
resolveTaskId(idOrName: string): TaskSnapshot {
|
|
202
|
+
const direct = this.tasks.get(idOrName);
|
|
203
|
+
if (direct) return direct;
|
|
204
|
+
const needle = idOrName.trim().toLowerCase();
|
|
205
|
+
const matches = needle.length === 0 ? [] : [...this.tasks.values()].filter((task) => task.name.toLowerCase() === needle);
|
|
206
|
+
const [match] = matches;
|
|
207
|
+
if (match && matches.length === 1) return match;
|
|
208
|
+
if (matches.length > 1) {
|
|
209
|
+
throw new Error(`Ambiguous task name "${idOrName}": matches ${matches.map((task) => task.taskId).join(", ")}`);
|
|
210
|
+
}
|
|
211
|
+
throw new Error(`Unknown task or name: ${idOrName}`);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
attachWatch(taskId: string, spec: WatchSpec, deadline?: number): void {
|
|
87
215
|
const validation = validateWatchSpec(spec);
|
|
88
216
|
if (!validation.ok) throw new Error(`Invalid watch spec: ${validation.error}`);
|
|
89
217
|
const task = this.tasks.get(taskId);
|
|
@@ -91,8 +219,17 @@ export class TaskRegistry {
|
|
|
91
219
|
if (isTerminal(task.status)) throw new Error(`Cannot attach watch to terminal task ${taskId}`);
|
|
92
220
|
const existing = this.watchLoops.get(taskId);
|
|
93
221
|
existing?.stop();
|
|
222
|
+
const now = Date.now();
|
|
223
|
+
// Persist the absolute watch deadline so a reload can restore the remaining budget (D-03).
|
|
224
|
+
// Pending delayed tasks get their deadline when the task actually starts (startTask).
|
|
225
|
+
if (deadline !== undefined) task.watchDeadline = deadline;
|
|
226
|
+
else if (task.status === "running") task.watchDeadline = now + spec.timeoutMs;
|
|
227
|
+
else delete task.watchDeadline;
|
|
228
|
+
// The loop only understands relative timeouts; feed it the remaining budget without
|
|
229
|
+
// rewriting the user-visible spec (`task.watch.timeoutMs` stays the original value).
|
|
230
|
+
const loopSpec: WatchSpec = task.watchDeadline === undefined ? spec : { ...spec, timeoutMs: Math.max(1, task.watchDeadline - now) };
|
|
94
231
|
const loop = new WatchLoop({
|
|
95
|
-
taskId, spec,
|
|
232
|
+
taskId, spec: loopSpec,
|
|
96
233
|
readOutput: () => this.readWatchOutput(taskId),
|
|
97
234
|
getExitCode: () => this.tasks.get(taskId)?.exitCode,
|
|
98
235
|
onSuccess: (detail) => this.markWatchSuccess(taskId, detail),
|
|
@@ -100,23 +237,28 @@ export class TaskRegistry {
|
|
|
100
237
|
onBlindExhausted: () => {
|
|
101
238
|
const current = this.tasks.get(taskId);
|
|
102
239
|
if (current && !isTerminal(current.status)) {
|
|
103
|
-
|
|
240
|
+
current.failureKind = "blind_exhausted";
|
|
241
|
+
this.finalize(current, "failed", undefined, "watch_failure");
|
|
104
242
|
void this.kill(taskId, "watch_failure").catch((error: unknown) => console.error(`pi-async: blind-check kill failed for task ${taskId}`, error));
|
|
105
243
|
}
|
|
106
244
|
},
|
|
107
245
|
onWatchTimeout: () => {
|
|
108
246
|
const current = this.tasks.get(taskId);
|
|
109
247
|
if (current && !isTerminal(current.status)) {
|
|
110
|
-
|
|
248
|
+
current.failureKind = "watch_timeout";
|
|
249
|
+
this.finalize(current, "failed", undefined, "watch_failure");
|
|
111
250
|
void this.kill(taskId, "watch_failure").catch((error: unknown) => console.error(`pi-async: watch-timeout kill failed for task ${taskId}`, error));
|
|
112
251
|
}
|
|
113
252
|
},
|
|
114
253
|
});
|
|
115
254
|
this.watchLoops.set(taskId, loop);
|
|
255
|
+
this.persist(task);
|
|
116
256
|
if (task.status === "running") loop.start();
|
|
117
257
|
}
|
|
118
258
|
private assertAdmission(opts: RunOptions): void {
|
|
119
259
|
if (this.closed) throw new Error("Async task admission is closed");
|
|
260
|
+
// A retired generation must not start work: the next generation owns the session data dir.
|
|
261
|
+
if (this.suspended) throw new Error("Async task admission is closed (registry suspended)");
|
|
120
262
|
const active = this.activeCount();
|
|
121
263
|
if (active.running + active.pending >= this.maxTasks) throw new Error(`Maximum active async tasks reached (${this.maxTasks})`);
|
|
122
264
|
if (!opts.command.trim()) throw new Error("command must not be empty");
|
|
@@ -135,7 +277,7 @@ export class TaskRegistry {
|
|
|
135
277
|
const taskId = randomUUID();
|
|
136
278
|
const outputFile = join(this.config.dataDir, `${taskId}.output`);
|
|
137
279
|
const task: TaskSnapshot = {
|
|
138
|
-
taskId, name: opts.name
|
|
280
|
+
taskId, name: opts.name ? capName(opts.name) : deriveName(opts.command), command: opts.command, mode, status: "running",
|
|
139
281
|
createdAt: Date.now(), startedAt: Date.now(), taskTimeoutMs: opts.taskTimeoutMs, notified: false, outputConsumed: false,
|
|
140
282
|
...(opts.watch ? { watch: opts.watch } : {}),
|
|
141
283
|
metadata: { notifyOnCompletion: opts.notifyOnCompletion ?? true, triggerOnCompletion: opts.triggerOnCompletion ?? true, outputFile },
|
|
@@ -158,9 +300,19 @@ export class TaskRegistry {
|
|
|
158
300
|
if (live.delayTimer) { clearTimeout(live.delayTimer); live.delayTimer = undefined; }
|
|
159
301
|
task.status = "running";
|
|
160
302
|
task.startedAt = Date.now();
|
|
161
|
-
|
|
303
|
+
// REV-13: always run in the resolved session cwd, never the host process cwd.
|
|
304
|
+
task.cwd = opts.cwd ?? this.config.cwd ?? process.cwd();
|
|
305
|
+
if (task.watch) task.watchDeadline = Date.now() + task.watch.timeoutMs;
|
|
306
|
+
const handle = spawnDetached(task.command, {
|
|
307
|
+
outputFile: live.outputFile,
|
|
308
|
+
env: { ...opts.env, PI_ASYNC_INTERNAL_MAX_OUTPUT_BYTES: String(this.maxOutputBytes) },
|
|
309
|
+
cwd: task.cwd,
|
|
310
|
+
});
|
|
162
311
|
task.pid = handle.pid;
|
|
163
312
|
live.exit = handle.exit.then(({ code }) => {
|
|
313
|
+
// A retired generation no longer owns this process: the next generation adopted it via
|
|
314
|
+
// reloadFromDisk and its snapshot wins (no finalize, no persist, no emit).
|
|
315
|
+
if (this.suspended) return { code };
|
|
164
316
|
live.processExited = true;
|
|
165
317
|
if (live.timer) { clearTimeout(live.timer); live.timer = undefined; }
|
|
166
318
|
task.exitCode = code;
|
|
@@ -168,9 +320,9 @@ export class TaskRegistry {
|
|
|
168
320
|
if (task.metadata.capped || task.metadata.watchSuccess !== undefined) this.persist(task);
|
|
169
321
|
if (!isTerminal(task.status) && task.watch) {
|
|
170
322
|
try {
|
|
171
|
-
const
|
|
172
|
-
if (evalConditions(task.watch.failure_when, {
|
|
173
|
-
else if (evalConditions(task.watch.success_when, {
|
|
323
|
+
const streams = this.readWatchOutput(task.taskId);
|
|
324
|
+
if (evalConditions(task.watch.failure_when, { ...streams, exitCode: code })) this.finalize(task, "killed", undefined, "watch_failure");
|
|
325
|
+
else if (evalConditions(task.watch.success_when, { ...streams, exitCode: code })) {
|
|
174
326
|
task.metadata.watchSuccess = `success_when matched after process exit: ${JSON.stringify(task.watch.success_when)}`;
|
|
175
327
|
this.finalize(task, "completed", code);
|
|
176
328
|
}
|
|
@@ -183,21 +335,35 @@ export class TaskRegistry {
|
|
|
183
335
|
}
|
|
184
336
|
return { code };
|
|
185
337
|
}).catch(() => {
|
|
338
|
+
if (this.suspended) return { code: 1 };
|
|
186
339
|
live.processExited = true;
|
|
187
340
|
if (!isTerminal(task.status)) this.finalize(task, "failed", 1);
|
|
188
341
|
return { code: 1 };
|
|
189
342
|
});
|
|
343
|
+
if (opts.taskTimeoutMs !== undefined && opts.taskTimeoutMs > 0) {
|
|
344
|
+
// Absolute deadline so a reload can re-arm only the remaining time (D-03).
|
|
345
|
+
task.timeoutAt = task.timeoutAt ?? Date.now() + opts.taskTimeoutMs;
|
|
346
|
+
live.timer = this.armTaskTimeout(task, Math.max(1, task.timeoutAt - Date.now()));
|
|
347
|
+
}
|
|
190
348
|
this.persist(task);
|
|
191
|
-
|
|
349
|
+
this.emitState(task);
|
|
350
|
+
const watch = this.watchLoops.get(task.taskId);
|
|
351
|
+
if (watch) watch.start();
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Arm (or re-arm) the task-timeout kill for a running task. */
|
|
355
|
+
private armTaskTimeout(task: TaskSnapshot, remainingMs: number, opts: { unref?: boolean } = {}): ReturnType<typeof setTimeout> {
|
|
356
|
+
const live = this.live.get(task.taskId);
|
|
357
|
+
const timer = setTimeout(() => {
|
|
192
358
|
if (isTerminal(task.status)) {
|
|
193
|
-
if (!live.processExited && task.pid) void killTree(task.pid).catch((error: unknown) => console.error(`pi-async: timeout kill failed for completed task ${task.taskId}`, error));
|
|
359
|
+
if (live && !live.processExited && task.pid) void killTree(task.pid).catch((error: unknown) => console.error(`pi-async: timeout kill failed for completed task ${task.taskId}`, error));
|
|
194
360
|
return;
|
|
195
361
|
}
|
|
196
362
|
void this.kill(task.taskId, "timeout").catch((error: unknown) => console.error(`pi-async: timeout kill failed for task ${task.taskId}`, error));
|
|
197
|
-
},
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
363
|
+
}, remainingMs);
|
|
364
|
+
// A timeout restored from disk must not be the only thing keeping the host process alive.
|
|
365
|
+
if (opts.unref) timer.unref?.();
|
|
366
|
+
return timer;
|
|
201
367
|
}
|
|
202
368
|
|
|
203
369
|
private scheduleDelayedStart(task: TaskSnapshot, delayMs: number): void {
|
|
@@ -213,17 +379,27 @@ export class TaskRegistry {
|
|
|
213
379
|
triggerOnCompletion: task.metadata.triggerOnCompletion !== false,
|
|
214
380
|
...(rawEnv && typeof rawEnv === "object" ? { env: rawEnv as Record<string, string> } : {}),
|
|
215
381
|
...(task.watch ? { watch: task.watch } : {}),
|
|
382
|
+
...(task.cwd !== undefined ? { cwd: task.cwd } : {}),
|
|
216
383
|
};
|
|
217
384
|
try { this.startTask(task, opts); }
|
|
218
|
-
catch
|
|
385
|
+
catch (error) {
|
|
386
|
+
console.error(`pi-async: delayed task ${task.taskId} failed to start`, error);
|
|
387
|
+
task.metadata = { ...task.metadata, startError: error instanceof Error ? error.message : String(error) };
|
|
388
|
+
this.finalize(task, "failed", 1);
|
|
389
|
+
}
|
|
219
390
|
}, delayMs);
|
|
220
391
|
}
|
|
221
392
|
|
|
222
|
-
private readWatchOutput(taskId: string):
|
|
393
|
+
private readWatchOutput(taskId: string): WatchOutput {
|
|
223
394
|
const task = this.tasks.get(taskId);
|
|
224
395
|
if (!task) throw new Error(`Unknown task id: ${taskId}`);
|
|
225
|
-
const outputFile = String(task.metadata.outputFile ?? this.live.get(taskId)?.outputFile ?? join(this.config.dataDir, `${taskId}.output`));
|
|
226
|
-
|
|
396
|
+
const outputFile = this.assertInsideDataDir(String(task.metadata.outputFile ?? this.live.get(taskId)?.outputFile ?? join(this.config.dataDir, `${taskId}.output`)));
|
|
397
|
+
// Missing primary output must throw so WatchLoop blind_checks can fire (#37/#68 + CONTRACT).
|
|
398
|
+
// stripLogControls: ANSI + bare C0 for model/watch paths (#18); streams stay distinct (#4).
|
|
399
|
+
const output = stripLogControls(readFileSync(outputFile, "utf8"));
|
|
400
|
+
const { stdout, stderr } = streamPaths(outputFile);
|
|
401
|
+
const readSide = (file: string): string => existsSync(file) ? stripLogControls(readFileSync(file, "utf8")) : "";
|
|
402
|
+
return { output, stdout: readSide(stdout), stderr: readSide(stderr) };
|
|
227
403
|
}
|
|
228
404
|
|
|
229
405
|
private syncResult(task: TaskSnapshot, promoted: boolean): SyncResult {
|
|
@@ -231,7 +407,7 @@ export class TaskRegistry {
|
|
|
231
407
|
const lines = outputTail.split("\n");
|
|
232
408
|
if (lines.length > 200) outputTail = lines.slice(-200).join("\n");
|
|
233
409
|
const bytes = Buffer.from(outputTail, "utf8");
|
|
234
|
-
if (bytes.byteLength > 8192) outputTail =
|
|
410
|
+
if (bytes.byteLength > 8192) outputTail = truncateUtf8Bytes(bytes, 8192, true);
|
|
235
411
|
return {
|
|
236
412
|
taskId: task.taskId, name: task.name, status: task.status, mode: task.mode,
|
|
237
413
|
outputHint: `Use async_logs with taskId ${task.taskId} to inspect early output.`,
|
|
@@ -241,10 +417,40 @@ export class TaskRegistry {
|
|
|
241
417
|
|
|
242
418
|
markWatchSuccess(taskId: string, detail: string): void { const task = this.tasks.get(taskId); if (!task || isTerminal(task.status)) return; task.metadata.watchSuccess = detail; this.finalize(task, "completed", task.exitCode); }
|
|
243
419
|
get(taskId: string): TaskSnapshot | undefined { return this.tasks.get(taskId); }
|
|
244
|
-
list(opts?: { includeHistory?: boolean }): TaskSnapshot[] { const all = [...this.tasks.values()]; return opts?.includeHistory === false ? all.filter((task) => !isTerminal(task.status)) : all; }
|
|
245
420
|
|
|
246
|
-
|
|
247
|
-
|
|
421
|
+
/**
|
|
422
|
+
* List snapshots. `includeHistory`/`status`/`name` filter; `limit` clamps to 1–50 when given
|
|
423
|
+
* (the tool layer supplies the default so internal prune/reload callers stay unbounded).
|
|
424
|
+
* History-wide queries sort newest-first (D-08).
|
|
425
|
+
*/
|
|
426
|
+
list(opts: ListOptions = {}): TaskSnapshot[] {
|
|
427
|
+
let items = [...this.tasks.values()];
|
|
428
|
+
if (opts.includeHistory === false) items = items.filter((task) => !isTerminal(task.status));
|
|
429
|
+
if (opts.status !== undefined) {
|
|
430
|
+
const status = opts.status;
|
|
431
|
+
items = status === "active" ? items.filter((task) => !isTerminal(task.status)) : items.filter((task) => task.status === status);
|
|
432
|
+
}
|
|
433
|
+
if (opts.name !== undefined) {
|
|
434
|
+
const needle = opts.name.trim().toLowerCase();
|
|
435
|
+
items = items.filter((task) => task.name.toLowerCase().includes(needle));
|
|
436
|
+
}
|
|
437
|
+
if (opts.includeHistory === true || opts.status !== undefined) {
|
|
438
|
+
items = [...items].sort((a, b) => (b.finishedAt ?? b.createdAt) - (a.finishedAt ?? a.createdAt));
|
|
439
|
+
}
|
|
440
|
+
if (opts.limit !== undefined && Number.isFinite(opts.limit)) {
|
|
441
|
+
const limit = Math.min(MAX_LIST_LIMIT, Math.max(1, Math.trunc(opts.limit)));
|
|
442
|
+
items = items.slice(0, limit);
|
|
443
|
+
}
|
|
444
|
+
return items;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
wait(taskIds: string[] | string, timeoutMs: number, signal?: AbortSignal): Promise<WaitOutcome> {
|
|
448
|
+
// A bare string is one id; String is iterable so `[...new Set("uuid")]` would split into characters (#40).
|
|
449
|
+
const idList = typeof taskIds === "string" ? [taskIds] : taskIds;
|
|
450
|
+
if (!Array.isArray(idList) || idList.some((id) => typeof id !== "string")) {
|
|
451
|
+
return Promise.reject(new TypeError(`taskIds must be an array of strings (got ${typeof taskIds})`));
|
|
452
|
+
}
|
|
453
|
+
const uniqueIds = [...new Set(idList)];
|
|
248
454
|
const unknown = uniqueIds.filter((id) => !this.tasks.has(id));
|
|
249
455
|
if (unknown.length) return Promise.reject(new Error(`Unknown task id(s): ${unknown.join(", ")}`));
|
|
250
456
|
if (signal?.aborted) return Promise.reject(signal.reason ?? new Error("Wait aborted"));
|
|
@@ -260,7 +466,12 @@ export class TaskRegistry {
|
|
|
260
466
|
const timeout = setTimeout(() => {
|
|
261
467
|
if (settled) return;
|
|
262
468
|
settled = true; cleanup();
|
|
263
|
-
|
|
469
|
+
const still = pending();
|
|
470
|
+
const finishedIds = uniqueIds.filter((id) => !still.some((task) => task.taskId === id));
|
|
471
|
+
const notReady = still.map((task) => ({ taskId: task.taskId, status: task.status as "running" | "pending", elapsedMs: Date.now() - task.createdAt }));
|
|
472
|
+
// REV-7: no finished → pure { notReady }; mixed → { results, notReady }.
|
|
473
|
+
if (finishedIds.length === 0) resolve({ notReady });
|
|
474
|
+
else resolve({ results: finishedIds.map((id) => this.waitResult(this.tasks.get(id)!)), notReady });
|
|
264
475
|
}, Math.max(0, timeoutMs));
|
|
265
476
|
const abort = () => { if (settled) return; settled = true; cleanup(); reject(signal?.reason ?? new Error("Wait aborted")); };
|
|
266
477
|
const cleanup = () => { clearTimeout(timeout); signal?.removeEventListener("abort", abort); this.waiters.delete(waiter); };
|
|
@@ -269,15 +480,44 @@ export class TaskRegistry {
|
|
|
269
480
|
});
|
|
270
481
|
}
|
|
271
482
|
|
|
272
|
-
|
|
483
|
+
/**
|
|
484
|
+
* Read bounded task output. `stream` selects merged/stdout/stderr; a missing file returns
|
|
485
|
+
* `missing: true` instead of throwing, while a path outside the data dir throws (D-11).
|
|
486
|
+
*
|
|
487
|
+
* Only the requested window is read (open/fstat/read, tail by default), so a multi-GB output
|
|
488
|
+
* file costs one bounded read instead of a full-file load. The size policy belongs to the
|
|
489
|
+
* caller (`maxBytes` defaults to 50 KiB; tools cap `async_logs`, the TUI may ask for a larger
|
|
490
|
+
* window), while `MAX_LOG_WINDOW` keeps a mistaken huge request from allocating unboundedly.
|
|
491
|
+
*/
|
|
492
|
+
logs(taskId: string, opts: { maxBytes?: number; tail?: boolean; stream?: LogStream } = {}): LogSlice {
|
|
273
493
|
const task = this.tasks.get(taskId);
|
|
274
494
|
if (!task) throw new Error(`Unknown task id: ${taskId}`);
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
const
|
|
278
|
-
const
|
|
279
|
-
const
|
|
280
|
-
|
|
495
|
+
const mergedPath = String(task.metadata.outputFile ?? this.live.get(taskId)?.outputFile ?? join(this.config.dataDir, `${taskId}.output`));
|
|
496
|
+
const stream = opts.stream ?? "merged";
|
|
497
|
+
const { stdout, stderr } = streamPaths(mergedPath);
|
|
498
|
+
const requestPath = stream === "stdout" ? stdout : stream === "stderr" ? stderr : mergedPath;
|
|
499
|
+
const fullPath = this.assertInsideDataDir(requestPath);
|
|
500
|
+
const requestedBytes = opts.maxBytes ?? DEFAULT_LOG_BYTES;
|
|
501
|
+
const boundedBytes = Number.isFinite(requestedBytes) ? Math.max(0, Math.trunc(requestedBytes)) : DEFAULT_LOG_BYTES;
|
|
502
|
+
const maxBytes = Math.min(boundedBytes, MAX_LOG_WINDOW);
|
|
503
|
+
const tail = opts.tail !== false;
|
|
504
|
+
let handle: number | undefined;
|
|
505
|
+
try {
|
|
506
|
+
handle = openSync(fullPath, "r");
|
|
507
|
+
const { size } = fstatSync(handle);
|
|
508
|
+
const truncated = size > maxBytes;
|
|
509
|
+
if (maxBytes === 0) return { text: "", truncated, fullPath };
|
|
510
|
+
// Non-tail reads take one extra byte so the UTF-8 cut can inspect the excluded byte.
|
|
511
|
+
const start = tail ? Math.max(0, size - maxBytes) : 0;
|
|
512
|
+
const length = Math.min(size, tail ? maxBytes : maxBytes + (truncated ? 1 : 0));
|
|
513
|
+
const buffer = readExactly(handle, start, length);
|
|
514
|
+
return { text: stripLogControls(truncateUtf8Window(buffer, maxBytes, tail, truncated)), truncated, fullPath };
|
|
515
|
+
} catch (error) {
|
|
516
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return { text: "", truncated: false, fullPath, missing: true };
|
|
517
|
+
throw error;
|
|
518
|
+
} finally {
|
|
519
|
+
if (handle !== undefined) closeSync(handle);
|
|
520
|
+
}
|
|
281
521
|
}
|
|
282
522
|
|
|
283
523
|
async kill(taskId: string, kind: KillKind): Promise<void> {
|
|
@@ -294,12 +534,22 @@ export class TaskRegistry {
|
|
|
294
534
|
}
|
|
295
535
|
if (isTerminal(task.status) && live.processExited) return;
|
|
296
536
|
live.killKind = kind;
|
|
537
|
+
let confirmed = true;
|
|
297
538
|
if (task.pid && !live.processExited) {
|
|
298
539
|
const result = await killTree(task.pid);
|
|
299
|
-
|
|
540
|
+
confirmed = result.confirmed;
|
|
300
541
|
}
|
|
542
|
+
// Finalize first so async_kill can always return { taskId, status } even when the
|
|
543
|
+
// confirmation probe cannot prove the group is gone (#6).
|
|
301
544
|
if (!isTerminal(task.status)) this.finalize(task, "killed", undefined, kind);
|
|
302
|
-
else
|
|
545
|
+
else {
|
|
546
|
+
// D4 / leftover process: record kill without flipping the settled observation status (#33).
|
|
547
|
+
task.killKind = kind;
|
|
548
|
+
try { this.persist(task); } catch { /* dataDir may already be gone during teardown */ }
|
|
549
|
+
if (live.timer) { clearTimeout(live.timer); live.timer = undefined; }
|
|
550
|
+
}
|
|
551
|
+
// Finalize-first (#6): still surface confirmation failure after status is settled.
|
|
552
|
+
if (!confirmed) throw new Error("Descendant processes may have leaked");
|
|
303
553
|
}
|
|
304
554
|
cancel(taskId: string): boolean {
|
|
305
555
|
const task = this.tasks.get(taskId);
|
|
@@ -308,19 +558,136 @@ export class TaskRegistry {
|
|
|
308
558
|
return true;
|
|
309
559
|
}
|
|
310
560
|
consume(taskId: string): void { const task = this.tasks.get(taskId); if (!task) return; task.outputConsumed = true; this.persist(task); }
|
|
561
|
+
|
|
562
|
+
/** In-memory send claim only (D-01): `notified` is owned by `markNotified`. */
|
|
311
563
|
claimNotification(taskId: string): boolean {
|
|
312
564
|
const task = this.tasks.get(taskId);
|
|
313
565
|
if (!task || task.notified || task.outputConsumed || this.notifyClaims.has(taskId)) return false;
|
|
314
|
-
this.notifyClaims.add(taskId);
|
|
566
|
+
this.notifyClaims.add(taskId);
|
|
567
|
+
return true;
|
|
315
568
|
}
|
|
316
|
-
|
|
569
|
+
/** Record a delivery attempt before calling the host (no retries: attempts stay honest). */
|
|
570
|
+
noteNotifyAttempt(taskId: string): void {
|
|
571
|
+
const task = this.tasks.get(taskId);
|
|
572
|
+
if (!task) return;
|
|
573
|
+
task.notifyAttempts = (task.notifyAttempts ?? 0) + 1;
|
|
574
|
+
this.persist(task);
|
|
575
|
+
}
|
|
576
|
+
/** Host accepted the message; the claim is kept for the rest of the generation. */
|
|
577
|
+
markNotified(taskId: string): void {
|
|
578
|
+
const task = this.tasks.get(taskId);
|
|
579
|
+
if (!task) return;
|
|
580
|
+
task.notified = true;
|
|
581
|
+
this.persist(task);
|
|
582
|
+
}
|
|
583
|
+
/**
|
|
584
|
+
* Release the in-memory claim after a failed delivery. `notified` is untouched: it is only
|
|
585
|
+
* ever set by `markNotified`, so a failed send leaves it false and `resendUnnotified` can retry.
|
|
586
|
+
*/
|
|
587
|
+
releaseNotification(taskId: string): void {
|
|
588
|
+
this.notifyClaims.delete(taskId);
|
|
589
|
+
}
|
|
590
|
+
|
|
317
591
|
activeCount(): { running: number; pending: number } { let running = 0; let pending = 0; for (const task of this.tasks.values()) { if (task.status === "running") running++; else if (task.status === "pending") pending++; } return { running, pending }; }
|
|
318
592
|
hasActiveWork(): boolean { const count = this.activeCount(); return count.running + count.pending > 0; }
|
|
593
|
+
|
|
594
|
+
/** Liveness probe for `async_kill` (D-07): no pid (or an exited child) → false. */
|
|
595
|
+
isProcessAlive(taskId: string): boolean {
|
|
596
|
+
const task = this.tasks.get(taskId);
|
|
597
|
+
if (!task || task.pid === undefined) return false;
|
|
598
|
+
const live = this.live.get(taskId);
|
|
599
|
+
if (live?.processExited) return false;
|
|
600
|
+
return this.pidAlive(task.pid);
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* Stop every timer/watcher and persist, but never kill (REV-17 `/reload`):
|
|
605
|
+
* the next generation adopts the surviving processes via `reloadFromDisk`.
|
|
606
|
+
*/
|
|
607
|
+
async suspend(): Promise<void> {
|
|
608
|
+
// Retire first, so a child exit landing during teardown is already ignored.
|
|
609
|
+
this.suspended = true;
|
|
610
|
+
for (const task of this.tasks.values()) {
|
|
611
|
+
const live = this.live.get(task.taskId);
|
|
612
|
+
if (live?.delayTimer) { clearTimeout(live.delayTimer); live.delayTimer = undefined; }
|
|
613
|
+
if (live?.timer) { clearTimeout(live.timer); live.timer = undefined; }
|
|
614
|
+
if (live?.watcher) { clearInterval(live.watcher); live.watcher = undefined; }
|
|
615
|
+
this.watchLoops.get(task.taskId)?.stop();
|
|
616
|
+
this.watchLoops.delete(task.taskId);
|
|
617
|
+
try { this.persist(task); } catch (error) { console.error(`pi-async: suspend could not persist task ${task.taskId}`, error); }
|
|
618
|
+
}
|
|
619
|
+
// Pending waiters belong to the retired generation; never wake them with stale results.
|
|
620
|
+
this.waiters.clear();
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
/**
|
|
624
|
+
* Re-send completion notifications the host never accepted (REV-14 `session_start` backfill).
|
|
625
|
+
* Only the existing `onFinalize` hook is replayed (D-02) — the registry owns no notifier.
|
|
626
|
+
*/
|
|
627
|
+
async resendUnnotified(opts: { maxAgeMs?: number; maxAttempts?: number } = {}): Promise<number> {
|
|
628
|
+
const maxAgeMs = opts.maxAgeMs ?? DEFAULT_RESEND_MAX_AGE_MS;
|
|
629
|
+
const maxAttempts = opts.maxAttempts ?? DEFAULT_RESEND_MAX_ATTEMPTS;
|
|
630
|
+
const now = Date.now();
|
|
631
|
+
let resent = 0;
|
|
632
|
+
for (const task of [...this.tasks.values()]) {
|
|
633
|
+
if (!isTerminal(task.status)) continue;
|
|
634
|
+
if (task.notified === true || task.outputConsumed === true) continue;
|
|
635
|
+
if (task.metadata.notifyOnCompletion === false) continue;
|
|
636
|
+
if ((task.notifyAttempts ?? 0) >= maxAttempts) continue;
|
|
637
|
+
if (now - (task.finishedAt ?? task.createdAt) > maxAgeMs) continue;
|
|
638
|
+
try {
|
|
639
|
+
this.events.onFinalize(task);
|
|
640
|
+
this.emit({ type: "finalize", snapshot: task });
|
|
641
|
+
resent++;
|
|
642
|
+
} catch (error) {
|
|
643
|
+
console.error(`pi-async: resend of completion notification failed for ${task.taskId}`, error);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
return resent;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Mark orphaned non-terminal tasks `failed/lost` (REV-16 startup hygiene):
|
|
651
|
+
* a running task whose process group is gone is dead; pending delayed tasks are kept.
|
|
652
|
+
*/
|
|
653
|
+
reapOrphans(now: number = Date.now()): number {
|
|
654
|
+
let reaped = 0;
|
|
655
|
+
for (const task of [...this.tasks.values()]) {
|
|
656
|
+
if (isTerminal(task.status)) continue;
|
|
657
|
+
if (task.status === "pending") {
|
|
658
|
+
// Pending delayed tasks hold no OS resources — their due timer is the source of truth…
|
|
659
|
+
// …unless the due time itself fell out of the retention window (PLAN §2.4).
|
|
660
|
+
if (task.dueAt === undefined || now - task.dueAt <= this.retentionMs) continue;
|
|
661
|
+
task.failureKind = "lost";
|
|
662
|
+
this.finalize(task, "failed");
|
|
663
|
+
reaped++;
|
|
664
|
+
continue;
|
|
665
|
+
}
|
|
666
|
+
const pid = task.pid;
|
|
667
|
+
if (pid !== undefined && this.pidAlive(pid)) continue;
|
|
668
|
+
task.failureKind = "lost";
|
|
669
|
+
this.finalize(task, "failed");
|
|
670
|
+
reaped++;
|
|
671
|
+
}
|
|
672
|
+
return reaped;
|
|
673
|
+
}
|
|
674
|
+
|
|
319
675
|
shutdown(): Promise<void> {
|
|
320
676
|
if (this.shutdownPromise) return this.shutdownPromise;
|
|
321
677
|
this.closed = true;
|
|
322
678
|
this.shutdownPromise = (async () => {
|
|
679
|
+
// Pending delayed tasks hold no OS resources. Clear their timers but leave snapshots
|
|
680
|
+
// as pending so same-process /reload can restore them (CONTRACT reload / #20).
|
|
681
|
+
for (const task of this.tasks.values()) {
|
|
682
|
+
if (task.status !== "pending") continue;
|
|
683
|
+
const live = this.live.get(task.taskId);
|
|
684
|
+
if (live?.delayTimer) { clearTimeout(live.delayTimer); live.delayTimer = undefined; }
|
|
685
|
+
const watch = this.watchLoops.get(task.taskId);
|
|
686
|
+
watch?.stop();
|
|
687
|
+
this.watchLoops.delete(task.taskId);
|
|
688
|
+
}
|
|
323
689
|
const ids = [...this.tasks.values()].filter((task) => {
|
|
690
|
+
if (task.status === "pending") return false;
|
|
324
691
|
const live = this.live.get(task.taskId);
|
|
325
692
|
return !isTerminal(task.status) || (task.pid !== undefined && (live ? !live.processExited : this.pidAlive(task.pid)));
|
|
326
693
|
}).map((task) => task.taskId);
|
|
@@ -337,64 +704,168 @@ export class TaskRegistry {
|
|
|
337
704
|
})();
|
|
338
705
|
return this.shutdownPromise;
|
|
339
706
|
}
|
|
707
|
+
|
|
340
708
|
async reloadFromDisk(): Promise<void> {
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
if (snapshot.
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
if (!
|
|
365
|
-
if (live.
|
|
366
|
-
|
|
367
|
-
|
|
709
|
+
this.reloading = true;
|
|
710
|
+
try {
|
|
711
|
+
let files: string[] = [];
|
|
712
|
+
try { files = readdirSync(this.config.dataDir); } catch { /* data dir not created yet */ }
|
|
713
|
+
for (const file of files.filter((name) => name.endsWith(".json"))) {
|
|
714
|
+
try {
|
|
715
|
+
const snapshot = JSON.parse(readFileSync(join(this.config.dataDir, file), "utf8")) as TaskSnapshot;
|
|
716
|
+
// #34: skip unknown status values from disk.
|
|
717
|
+
if (!snapshot.taskId || !isTaskStatus(snapshot.status) || this.tasks.has(snapshot.taskId)) continue;
|
|
718
|
+
this.tasks.set(snapshot.taskId, snapshot);
|
|
719
|
+
if (snapshot.status === "pending" && snapshot.mode === "delayed") {
|
|
720
|
+
const outputFile = String(snapshot.metadata.outputFile ?? join(this.config.dataDir, `${snapshot.taskId}.output`));
|
|
721
|
+
this.live.set(snapshot.taskId, { outputFile });
|
|
722
|
+
if (snapshot.watch) this.attachWatch(snapshot.taskId, snapshot.watch);
|
|
723
|
+
const dueAt = snapshot.dueAt ?? Date.now();
|
|
724
|
+
this.scheduleDelayedStart(snapshot, Math.max(0, dueAt - Date.now()));
|
|
725
|
+
continue;
|
|
726
|
+
}
|
|
727
|
+
if ((snapshot.status === "running" || snapshot.metadata.watchSuccess !== undefined) && snapshot.pid && this.pidAlive(snapshot.pid)) {
|
|
728
|
+
const outputFile = String(snapshot.metadata.outputFile ?? join(this.config.dataDir, `${snapshot.taskId}.output`));
|
|
729
|
+
const live: LiveTask = { outputFile };
|
|
730
|
+
this.live.set(snapshot.taskId, live);
|
|
731
|
+
live.watcher = setInterval(() => {
|
|
732
|
+
if (snapshot.pid && !this.pidAlive(snapshot.pid)) {
|
|
733
|
+
if (live.watcher) clearInterval(live.watcher);
|
|
734
|
+
live.processExited = true;
|
|
735
|
+
if (live.timer) { clearTimeout(live.timer); live.timer = undefined; }
|
|
736
|
+
snapshot.metadata.capped = this.isOutputCapped(snapshot);
|
|
737
|
+
if (!isTerminal(snapshot.status)) {
|
|
738
|
+
if (live.killKind) this.finalize(snapshot, "killed", undefined, live.killKind);
|
|
739
|
+
else if (snapshot.metadata.capped) this.finalize(snapshot, "failed", undefined, "output_cap");
|
|
740
|
+
else { snapshot.failureKind = "lost"; this.finalize(snapshot, "failed"); }
|
|
741
|
+
}
|
|
368
742
|
}
|
|
743
|
+
}, 100);
|
|
744
|
+
live.watcher.unref?.();
|
|
745
|
+
// D-03: re-arm only the remaining task-timeout budget instead of a fresh full window.
|
|
746
|
+
if (snapshot.status === "running" && snapshot.taskTimeoutMs !== undefined && snapshot.taskTimeoutMs > 0) {
|
|
747
|
+
const anchor = snapshot.timeoutAt ?? (snapshot.startedAt ?? snapshot.createdAt) + snapshot.taskTimeoutMs;
|
|
748
|
+
live.timer = this.armTaskTimeout(snapshot, Math.max(1, anchor - Date.now()), { unref: true });
|
|
369
749
|
}
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
if (snapshot.status === "running"
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
750
|
+
// D-03: restore the watch window from the persisted absolute deadline.
|
|
751
|
+
if (snapshot.status === "running" && snapshot.watch) this.attachWatch(snapshot.taskId, snapshot.watch, snapshot.watchDeadline);
|
|
752
|
+
} else if (snapshot.status === "running") {
|
|
753
|
+
// Orphaned on disk: settle it through the same path as `reapOrphans()` so the
|
|
754
|
+
// snapshot also records `processAlive = false` (D-07) and emits one finalize.
|
|
755
|
+
snapshot.failureKind = "lost";
|
|
756
|
+
this.finalize(snapshot, "failed");
|
|
757
|
+
}
|
|
758
|
+
} catch { /* Ignore incomplete/corrupt metadata files; atomic writer leaves prior version intact. */ }
|
|
759
|
+
}
|
|
760
|
+
} finally {
|
|
761
|
+
this.reloading = false;
|
|
762
|
+
this.prune();
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
/**
|
|
767
|
+
* Drop terminal-task artifacts that fall outside the retention window.
|
|
768
|
+
*
|
|
769
|
+
* Policy (per dataDir): keep the union of (1) the newest `retentionCount`
|
|
770
|
+
* terminal tasks by finishedAt/createdAt and (2) any terminal task still
|
|
771
|
+
* younger than `retentionMs`. Everything else terminal is deleted
|
|
772
|
+
* (.json + .output/.stdout/.stderr via metadata.outputFile) and removed from memory.
|
|
773
|
+
* Non-terminal tasks are never touched. Missing files are ignored.
|
|
774
|
+
*/
|
|
775
|
+
prune(now = Date.now()): { pruned: number } {
|
|
776
|
+
const terminal = this.list({ includeHistory: true })
|
|
777
|
+
.filter((task) => isTerminal(task.status))
|
|
778
|
+
.sort((a, b) => (b.finishedAt ?? b.createdAt) - (a.finishedAt ?? a.createdAt));
|
|
779
|
+
let pruned = 0;
|
|
780
|
+
for (let i = 0; i < terminal.length; i++) {
|
|
781
|
+
const task = terminal[i]!;
|
|
782
|
+
const ageAnchor = task.finishedAt ?? task.createdAt;
|
|
783
|
+
const withinCount = i < this.retentionCount;
|
|
784
|
+
const withinAge = now - ageAnchor < this.retentionMs;
|
|
785
|
+
if (withinCount || withinAge) continue;
|
|
786
|
+
this.removeArtifacts(task);
|
|
787
|
+
pruned++;
|
|
788
|
+
}
|
|
789
|
+
return { pruned };
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
private removeArtifacts(task: TaskSnapshot): void {
|
|
793
|
+
const outputFile = String(task.metadata.outputFile ?? join(this.config.dataDir, `${task.taskId}.output`));
|
|
794
|
+
const { stdout, stderr } = streamPaths(outputFile);
|
|
795
|
+
// D-11: never delete outside the data dir (skip silently so prune cannot crash).
|
|
796
|
+
for (const file of [outputFile, stdout, stderr, join(this.config.dataDir, `${task.taskId}.json`)]) {
|
|
797
|
+
if (!this.isInsideDataDir(file)) continue;
|
|
798
|
+
try { rmSync(file, { force: true }); } catch { /* force already ignores ENOENT; swallow other rare races */ }
|
|
377
799
|
}
|
|
800
|
+
const live = this.live.get(task.taskId);
|
|
801
|
+
if (live?.timer) clearTimeout(live.timer);
|
|
802
|
+
if (live?.delayTimer) clearTimeout(live.delayTimer);
|
|
803
|
+
if (live?.watcher) clearInterval(live.watcher);
|
|
804
|
+
this.live.delete(task.taskId);
|
|
805
|
+
this.watchLoops.get(task.taskId)?.stop();
|
|
806
|
+
this.watchLoops.delete(task.taskId);
|
|
807
|
+
this.tasks.delete(task.taskId);
|
|
808
|
+
this.notifyClaims.delete(task.taskId);
|
|
378
809
|
}
|
|
379
810
|
|
|
380
|
-
private waitResult(task: TaskSnapshot): { taskId: string; status: TaskSnapshot["status"]; exitCode?: number; outputTail: string } {
|
|
811
|
+
private waitResult(task: TaskSnapshot): { taskId: string; status: TaskSnapshot["status"]; exitCode?: number; outputTail: string; killKind?: KillKind; failureKind?: FailureKind } {
|
|
381
812
|
this.consume(task.taskId);
|
|
382
|
-
return {
|
|
813
|
+
return {
|
|
814
|
+
taskId: task.taskId,
|
|
815
|
+
status: task.status,
|
|
816
|
+
...(task.exitCode === undefined ? {} : { exitCode: task.exitCode }),
|
|
817
|
+
outputTail: this.logs(task.taskId, { maxBytes: 8192, tail: true }).text,
|
|
818
|
+
...(task.killKind ? { killKind: task.killKind } : {}),
|
|
819
|
+
...(task.failureKind ? { failureKind: task.failureKind } : {}),
|
|
820
|
+
};
|
|
383
821
|
}
|
|
384
822
|
private finalize(task: TaskSnapshot, status: TaskSnapshot["status"], code?: number, killKind?: KillKind): void {
|
|
385
823
|
if (isTerminal(task.status)) return;
|
|
386
824
|
task.status = status; task.exitCode = code; task.killKind = killKind; task.finishedAt = Date.now();
|
|
825
|
+
// D-07: a terminal snapshot never claims a live process.
|
|
826
|
+
task.processAlive = false;
|
|
387
827
|
const live = this.live.get(task.taskId);
|
|
388
828
|
if (live?.delayTimer) { clearTimeout(live.delayTimer); live.delayTimer = undefined; }
|
|
389
829
|
const watch = this.watchLoops.get(task.taskId); watch?.stop(); this.watchLoops.delete(task.taskId);
|
|
390
830
|
if (live?.watcher) clearInterval(live.watcher);
|
|
391
831
|
// D4: watch success settles the observed task but leaves the process under its task timeout.
|
|
392
832
|
if (live?.timer && !(task.metadata.watchSuccess !== undefined && !live.processExited)) { clearTimeout(live.timer); live.timer = undefined; }
|
|
393
|
-
|
|
833
|
+
// #15: a waiter already owns this result — set the suppression latch before onFinalize
|
|
834
|
+
// can claimNotification / sendCompletionNotification.
|
|
835
|
+
if ([...this.waiters].some((waiter) => waiter.ids.has(task.taskId))) {
|
|
836
|
+
task.outputConsumed = true;
|
|
837
|
+
}
|
|
838
|
+
this.emitState(task);
|
|
839
|
+
this.finalizeEvent(task);
|
|
394
840
|
for (const waiter of [...this.waiters]) if (waiter.ids.has(task.taskId)) waiter.resolve();
|
|
841
|
+
if (!this.reloading) this.prune();
|
|
842
|
+
}
|
|
843
|
+
private finalizeEvent(task: TaskSnapshot): void {
|
|
844
|
+
this.events.onFinalize(task);
|
|
845
|
+
this.emit({ type: "finalize", snapshot: task });
|
|
846
|
+
}
|
|
847
|
+
private emitState(task: TaskSnapshot): void {
|
|
848
|
+
this.persist(task);
|
|
849
|
+
this.events.onStateChange(task);
|
|
850
|
+
this.emit({ type: "state", snapshot: task });
|
|
851
|
+
}
|
|
852
|
+
private emit(event: RegistryEvent): void {
|
|
853
|
+
for (const listener of [...this.listeners]) {
|
|
854
|
+
try { listener(event); } catch (error) { console.error("pi-async: registry event listener failed", error); }
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
private ensureDataDir(): void {
|
|
859
|
+
if (this.dataDirReady) return;
|
|
860
|
+
mkdirSync(this.config.dataDir, { recursive: true });
|
|
861
|
+
this.dataDirReady = true;
|
|
862
|
+
}
|
|
863
|
+
/** `configure()` is contractually non-throwing (D-05); a bad data dir surfaces on persist. */
|
|
864
|
+
private ensureDataDirQuietly(): void {
|
|
865
|
+
try { this.ensureDataDir(); } catch (error) { console.error(`pi-async: cannot create data dir ${this.config.dataDir}`, error); }
|
|
395
866
|
}
|
|
396
|
-
private emitState(task: TaskSnapshot): void { this.persist(task); this.events.onStateChange(task); }
|
|
397
867
|
private persist(task: TaskSnapshot): void {
|
|
868
|
+
this.ensureDataDir();
|
|
398
869
|
const file = join(this.config.dataDir, `${task.taskId}.json`); const temp = `${file}.${process.pid}.tmp`;
|
|
399
870
|
writeFileSync(temp, JSON.stringify(task), { mode: 0o600 }); renameSync(temp, file);
|
|
400
871
|
}
|
|
@@ -404,13 +875,93 @@ export class TaskRegistry {
|
|
|
404
875
|
return persisted.metadata.capped === true;
|
|
405
876
|
} catch { return task.metadata.capped === true; }
|
|
406
877
|
}
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
878
|
+
|
|
879
|
+
private switchDataDir(nextDataDir: string): void {
|
|
880
|
+
this.config = { ...this.config, dataDir: nextDataDir };
|
|
881
|
+
this.dataDirReady = false;
|
|
410
882
|
}
|
|
411
|
-
}
|
|
412
883
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
884
|
+
/**
|
|
885
|
+
* Move every task artifact from the current data dir to `nextDataDir` (copy + rename),
|
|
886
|
+
* then repoint snapshots. On failure nothing is lost: the old dir stays authoritative
|
|
887
|
+
* and the error is reported, never thrown (D-05).
|
|
888
|
+
*/
|
|
889
|
+
private migrateDataDir(nextDataDir: string): boolean {
|
|
890
|
+
const oldDir = this.config.dataDir;
|
|
891
|
+
let names: string[] = [];
|
|
892
|
+
try {
|
|
893
|
+
names = readdirSync(oldDir).filter((name) => ARTIFACT_SUFFIXES.some((suffix) => name.endsWith(suffix)));
|
|
894
|
+
} catch { /* Old dir missing: nothing to migrate. */ }
|
|
895
|
+
const copied: string[] = [];
|
|
896
|
+
try {
|
|
897
|
+
mkdirSync(nextDataDir, { recursive: true });
|
|
898
|
+
for (const name of names) {
|
|
899
|
+
const target = join(nextDataDir, name);
|
|
900
|
+
const temp = `${target}.${process.pid}.tmp`;
|
|
901
|
+
copyFileSync(join(oldDir, name), temp);
|
|
902
|
+
renameSync(temp, target);
|
|
903
|
+
copied.push(target);
|
|
904
|
+
}
|
|
905
|
+
} catch (error) {
|
|
906
|
+
for (const file of copied) { try { rmSync(file, { force: true }); } catch { /* best-effort rollback */ } }
|
|
907
|
+
console.error(`pi-async: data dir migration failed; keeping ${oldDir}`, error);
|
|
908
|
+
return false;
|
|
909
|
+
}
|
|
910
|
+
this.switchDataDir(nextDataDir);
|
|
911
|
+
this.ensureDataDir();
|
|
912
|
+
for (const task of this.tasks.values()) {
|
|
913
|
+
const outputFile = join(nextDataDir, `${task.taskId}.output`);
|
|
914
|
+
task.metadata = { ...task.metadata, outputFile };
|
|
915
|
+
const live = this.live.get(task.taskId);
|
|
916
|
+
if (live) live.outputFile = outputFile;
|
|
917
|
+
try { this.persist(task); } catch (error) { console.error(`pi-async: could not rewrite task ${task.taskId} after data dir migration`, error); }
|
|
918
|
+
}
|
|
919
|
+
for (const name of names) { try { rmSync(join(oldDir, name), { force: true }); } catch { /* best-effort cleanup */ } }
|
|
920
|
+
return true;
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
/** Absolute data dir (realpath when it exists) for boundary checks. */
|
|
924
|
+
private resolvedDataDir(): string {
|
|
925
|
+
try { return realpathSync(this.config.dataDir); } catch { return resolve(this.config.dataDir); }
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
private isInsideDataDir(target: string): boolean {
|
|
929
|
+
const root = this.resolvedDataDir();
|
|
930
|
+
let resolvedTarget: string;
|
|
931
|
+
try { resolvedTarget = realpathSync(target); } catch { resolvedTarget = resolve(target); }
|
|
932
|
+
return resolvedTarget === root || resolvedTarget.startsWith(root + sep);
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
/** D-11: reject reads of `metadata.outputFile` outside the data dir. */
|
|
936
|
+
private assertInsideDataDir(target: string): string {
|
|
937
|
+
const root = this.resolvedDataDir();
|
|
938
|
+
let resolvedTarget: string;
|
|
939
|
+
try { resolvedTarget = realpathSync(target); } catch { resolvedTarget = resolve(target); }
|
|
940
|
+
if (resolvedTarget !== root && !resolvedTarget.startsWith(root + sep)) {
|
|
941
|
+
throw new Error(`Refusing to read task output outside the async data directory: ${target}`);
|
|
942
|
+
}
|
|
943
|
+
return resolvedTarget;
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
private pidAlive(pid: number): boolean {
|
|
947
|
+
// Prefer /proc so a zombie-only group is not treated as alive (#6 / reload lost detection).
|
|
948
|
+
try {
|
|
949
|
+
for (const entry of readdirSync("/proc")) {
|
|
950
|
+
if (!/^\d+$/.test(entry)) continue;
|
|
951
|
+
try {
|
|
952
|
+
const stat = readFileSync(join("/proc", entry, "stat"), "utf8");
|
|
953
|
+
const closeParen = stat.lastIndexOf(") ");
|
|
954
|
+
if (closeParen < 0) continue;
|
|
955
|
+
const rest = stat.slice(closeParen + 2).split(" ");
|
|
956
|
+
const state = rest[0];
|
|
957
|
+
const pgrp = Number(rest[2]);
|
|
958
|
+
if (pgrp === pid && state !== "Z") return true;
|
|
959
|
+
} catch { /* raced with exit */ }
|
|
960
|
+
}
|
|
961
|
+
return false;
|
|
962
|
+
} catch {
|
|
963
|
+
try { process.kill(-pid, 0); return true; }
|
|
964
|
+
catch (error) { return (error as NodeJS.ErrnoException).code === "EPERM"; }
|
|
965
|
+
}
|
|
966
|
+
}
|
|
416
967
|
}
|