pi-simple-background-tasks 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/FORK-CHANGES.md +66 -0
- package/LICENSE +15 -0
- package/README.md +100 -2
- package/package.json +59 -4
- package/src/core/common.ts +914 -0
- package/src/core/config.ts +83 -0
- package/src/core/durable-fs.ts +545 -0
- package/src/core/extension-api.ts +70 -0
- package/src/core/lazy-module.ts +55 -0
- package/src/core/registry.ts +2426 -0
- package/src/core/reload-shell-owner.ts +1512 -0
- package/src/core/shell-policy.ts +180 -0
- package/src/core/task-cleanup.ts +146 -0
- package/src/core/task-durable.ts +65 -0
- package/src/core/windows-taskkill.ts +263 -0
- package/src/extension.ts +928 -0
- package/src/ui/background-tasks-manager.ts +731 -0
|
@@ -0,0 +1,2426 @@
|
|
|
1
|
+
import { spawn as nodeSpawn, type SpawnOptions } from 'node:child_process';
|
|
2
|
+
import { randomBytes } from 'node:crypto';
|
|
3
|
+
import { createWriteStream, existsSync } from 'node:fs';
|
|
4
|
+
// biome-ignore lint/correctness/noUnusedImports: rm is used in task-file cleanup (Promise.allSettled), which biome's import heuristic misses
|
|
5
|
+
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { formatSize } from '@earendil-works/pi-coding-agent';
|
|
8
|
+
import {
|
|
9
|
+
type BgLogsDetails,
|
|
10
|
+
type BgTask,
|
|
11
|
+
type BgTaskSnapshot,
|
|
12
|
+
boundedRead,
|
|
13
|
+
deriveTaskNameFromCommand,
|
|
14
|
+
escapeXml,
|
|
15
|
+
formatDuration,
|
|
16
|
+
type KillKind,
|
|
17
|
+
normalizeTaskName,
|
|
18
|
+
type ReloadableShellExecutionV1,
|
|
19
|
+
type ReloadShellActivationClaimV1,
|
|
20
|
+
type ReloadShellActivationLeaseV1,
|
|
21
|
+
type ReloadShellHostAdapterV1,
|
|
22
|
+
type ReloadShellIdentityV1,
|
|
23
|
+
type ReloadShellOwnerHubV1,
|
|
24
|
+
ReloadSurvivalError,
|
|
25
|
+
type ResolvedShellPolicy,
|
|
26
|
+
resolveShellPolicy,
|
|
27
|
+
type StartTaskOptions,
|
|
28
|
+
sanitizePathSegment,
|
|
29
|
+
shellInvocationForPolicy,
|
|
30
|
+
shellPolicySnapshot,
|
|
31
|
+
snapshot,
|
|
32
|
+
type TaskStatus,
|
|
33
|
+
type TerminalPublicationAbandonReason,
|
|
34
|
+
taskDisplayName,
|
|
35
|
+
} from './common.js';
|
|
36
|
+
import { defaultTasksDir } from './config.js';
|
|
37
|
+
import { BackgroundTaskExtensionServiceClosedError } from './extension-api.js';
|
|
38
|
+
import {
|
|
39
|
+
createReloadableShellExecutionV1,
|
|
40
|
+
RELOAD_SHELL_OWNER_PROTOCOL,
|
|
41
|
+
} from './reload-shell-owner.js';
|
|
42
|
+
import {
|
|
43
|
+
type CleanupScope,
|
|
44
|
+
type CleanupSummary,
|
|
45
|
+
cleanupFinishedTasks as cleanupTasksOnDisk,
|
|
46
|
+
} from './task-cleanup.js';
|
|
47
|
+
import { closeAndFsyncOutputStream, writeJsonAtomic } from './task-durable.js';
|
|
48
|
+
import {
|
|
49
|
+
runWindowsTaskkill,
|
|
50
|
+
type TaskkillOutcome,
|
|
51
|
+
type WindowsKillPhase,
|
|
52
|
+
type WindowsTaskkillOptions,
|
|
53
|
+
} from './windows-taskkill.js';
|
|
54
|
+
|
|
55
|
+
export const MAX_OUTPUT_BYTES = Number(process.env.PI_BG_MAX_OUTPUT_BYTES ?? 20 * 1024 * 1024);
|
|
56
|
+
export const KILL_GRACE_MS = 3000;
|
|
57
|
+
export const STOP_WAIT_MS = KILL_GRACE_MS + 1500;
|
|
58
|
+
export const MAX_RECENT_TASKS = 100;
|
|
59
|
+
export const TERMINAL_PUBLICATION_MAX_ATTEMPTS = 3;
|
|
60
|
+
export const TERMINAL_PUBLICATION_RETRY_MS = 100;
|
|
61
|
+
export const TASK_ADMISSION_TIMEOUT_MS = 30_000;
|
|
62
|
+
const TERMINAL_PUBLICATION_DIAGNOSTIC_CHARS = 500;
|
|
63
|
+
|
|
64
|
+
export type TerminalPublicationClosureReason = Extract<
|
|
65
|
+
TerminalPublicationAbandonReason,
|
|
66
|
+
'registry_shutdown' | 'publisher_closed'
|
|
67
|
+
>;
|
|
68
|
+
|
|
69
|
+
type TerminalPublicationGateOutcome =
|
|
70
|
+
| { readonly kind: 'released' }
|
|
71
|
+
| { readonly kind: 'rejected'; readonly error: unknown }
|
|
72
|
+
| { readonly kind: 'closed'; readonly reason: TerminalPublicationAbandonReason };
|
|
73
|
+
|
|
74
|
+
interface TerminalPublicationAbandonSignal {
|
|
75
|
+
readonly promise: Promise<TerminalPublicationAbandonReason>;
|
|
76
|
+
readonly resolve: (reason: TerminalPublicationAbandonReason) => void;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export class BackgroundTaskAdmissionClosedError extends Error {
|
|
80
|
+
readonly code = 'pi_background_tasks_admission_closed';
|
|
81
|
+
|
|
82
|
+
constructor(kind: string) {
|
|
83
|
+
super(`Cannot start ${kind} after background task admissions have closed`);
|
|
84
|
+
this.name = 'BackgroundTaskAdmissionClosedError';
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export class BackgroundTaskAdmissionTimeoutError extends Error {
|
|
89
|
+
readonly code = 'pi_background_tasks_admission_timeout';
|
|
90
|
+
|
|
91
|
+
constructor(kind: string, timeoutMs: number) {
|
|
92
|
+
super(`Timed out while preparing ${kind} after ${String(timeoutMs)}ms`);
|
|
93
|
+
this.name = 'BackgroundTaskAdmissionTimeoutError';
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
interface TaskAdmission {
|
|
98
|
+
readonly kind: string;
|
|
99
|
+
readonly controller: AbortController;
|
|
100
|
+
readonly deadlineAt: number;
|
|
101
|
+
readonly timeoutMs: number;
|
|
102
|
+
timeoutHandle: NodeJS.Timeout | undefined;
|
|
103
|
+
released: boolean;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export interface BackgroundTaskContext {
|
|
107
|
+
cwd: string;
|
|
108
|
+
sessionId?: string;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
interface OutputEventSource {
|
|
112
|
+
on(event: 'data', listener: (data: Buffer | string) => void): unknown;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
interface ChildStdin {
|
|
116
|
+
write(data: Buffer, callback: (error?: Error | null) => void): boolean;
|
|
117
|
+
end(callback?: () => void): unknown;
|
|
118
|
+
once(event: 'error', listener: (error: Error) => void): unknown;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export interface BackgroundTaskChildProcess {
|
|
122
|
+
pid?: number | undefined;
|
|
123
|
+
stdin?: ChildStdin | null | undefined;
|
|
124
|
+
stdout?: OutputEventSource | null | undefined;
|
|
125
|
+
stderr?: OutputEventSource | null | undefined;
|
|
126
|
+
kill(signal?: NodeJS.Signals): boolean;
|
|
127
|
+
on(event: 'error', listener: (error: Error) => void): unknown;
|
|
128
|
+
on(
|
|
129
|
+
event: 'close',
|
|
130
|
+
listener: (code: number | null, signal: NodeJS.Signals | null) => void,
|
|
131
|
+
): unknown;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export type BackgroundTaskSpawn = (
|
|
135
|
+
command: string,
|
|
136
|
+
args: string[],
|
|
137
|
+
options: SpawnOptions,
|
|
138
|
+
) => BackgroundTaskChildProcess;
|
|
139
|
+
|
|
140
|
+
type KillProcessFn = (pid: number, signal?: NodeJS.Signals | number) => boolean;
|
|
141
|
+
type KillTreeFn = (
|
|
142
|
+
pid: number,
|
|
143
|
+
phase: WindowsKillPhase,
|
|
144
|
+
signal?: AbortSignal,
|
|
145
|
+
) => Promise<TaskkillOutcome>;
|
|
146
|
+
|
|
147
|
+
interface PosixProcessGroupKillState {
|
|
148
|
+
readonly groupId: number;
|
|
149
|
+
readonly completion: Promise<void>;
|
|
150
|
+
readonly resolveCompletion: () => void;
|
|
151
|
+
readonly deadlineAt: number;
|
|
152
|
+
forceAttempted: boolean;
|
|
153
|
+
settled: boolean;
|
|
154
|
+
failure?: Error | undefined;
|
|
155
|
+
lastProbeError?: Error | undefined;
|
|
156
|
+
verificationTimer?: NodeJS.Timeout | undefined;
|
|
157
|
+
failureListeners?: Array<(error: Error) => void> | undefined;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
interface WindowsKillState {
|
|
161
|
+
softController?: AbortController | undefined;
|
|
162
|
+
softPromise?: Promise<void> | undefined;
|
|
163
|
+
forcePromise?: Promise<void> | undefined;
|
|
164
|
+
forceFailure?: Error | undefined;
|
|
165
|
+
forceFailureListeners?: Array<(error: Error) => void> | undefined;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export interface CompletionNotificationMessage {
|
|
169
|
+
customType: 'background-task-notification';
|
|
170
|
+
content: string;
|
|
171
|
+
display: true;
|
|
172
|
+
details: BgTaskSnapshot;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export interface CompletionNotificationOptions {
|
|
176
|
+
deliverAs: 'followUp';
|
|
177
|
+
triggerTurn: boolean;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
export type CompletionNotificationSender = (
|
|
181
|
+
message: CompletionNotificationMessage,
|
|
182
|
+
options: CompletionNotificationOptions,
|
|
183
|
+
) => void;
|
|
184
|
+
|
|
185
|
+
export interface BackgroundTaskRegistryOptions {
|
|
186
|
+
sendCompletionNotification: CompletionNotificationSender;
|
|
187
|
+
publishTerminal?: (task: BgTaskSnapshot) => void;
|
|
188
|
+
spawn?: BackgroundTaskSpawn;
|
|
189
|
+
killProcess?: KillProcessFn;
|
|
190
|
+
killTree?: KillTreeFn;
|
|
191
|
+
platform?: NodeJS.Platform;
|
|
192
|
+
env?: NodeJS.ProcessEnv;
|
|
193
|
+
shellPolicy?: ResolvedShellPolicy;
|
|
194
|
+
tasksDir?: string | undefined;
|
|
195
|
+
makeTaskId?: () => string;
|
|
196
|
+
now?: () => number;
|
|
197
|
+
maxOutputBytes?: number;
|
|
198
|
+
maxRecentTasks?: number;
|
|
199
|
+
killGraceMs?: number;
|
|
200
|
+
stopWaitMs?: number;
|
|
201
|
+
taskAdmissionTimeoutMs?: number;
|
|
202
|
+
logger?: Pick<Console, 'error'>;
|
|
203
|
+
reloadShellOwner?: ReloadShellOwnerHubV1;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
interface RuntimeDir {
|
|
207
|
+
abs: string;
|
|
208
|
+
display: string;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function defaultTaskId(): string {
|
|
212
|
+
return `b${randomBytes(4).toString('hex')}`;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function noopOnChange(): void {}
|
|
216
|
+
|
|
217
|
+
export class BackgroundTaskRegistry {
|
|
218
|
+
private readonly tasks = new Map<string, BgTask>();
|
|
219
|
+
private runtimeDir: RuntimeDir | undefined;
|
|
220
|
+
private readonly tasksDir: string;
|
|
221
|
+
private shuttingDown = false;
|
|
222
|
+
private taskAdmissionsClosed = false;
|
|
223
|
+
private readonly activeTaskAdmissions = new Set<TaskAdmission>();
|
|
224
|
+
private readonly taskAdmissionDrainWaiters = new Set<() => void>();
|
|
225
|
+
private terminalPublicationClosed = false;
|
|
226
|
+
private terminalPublicationCloseReason: TerminalPublicationClosureReason | undefined;
|
|
227
|
+
private readonly terminalPublicationClosedSignal: Promise<TerminalPublicationClosureReason>;
|
|
228
|
+
private resolveTerminalPublicationClosedSignal: (
|
|
229
|
+
reason: TerminalPublicationClosureReason,
|
|
230
|
+
) => void = () => {};
|
|
231
|
+
private readonly spawn: BackgroundTaskSpawn;
|
|
232
|
+
private readonly killProcess: KillProcessFn;
|
|
233
|
+
private readonly killTree: KillTreeFn;
|
|
234
|
+
private readonly platform: NodeJS.Platform;
|
|
235
|
+
private readonly env: NodeJS.ProcessEnv;
|
|
236
|
+
private shellPolicy: ResolvedShellPolicy | undefined;
|
|
237
|
+
private readonly shellPolicyEnv: NodeJS.ProcessEnv | undefined;
|
|
238
|
+
private readonly makeTaskIdFn: () => string;
|
|
239
|
+
private readonly now: () => number;
|
|
240
|
+
private readonly maxOutputBytes: number;
|
|
241
|
+
private readonly maxRecentTasks: number;
|
|
242
|
+
private readonly killGraceMs: number;
|
|
243
|
+
private readonly stopWaitMs: number;
|
|
244
|
+
private readonly taskAdmissionTimeoutMs: number;
|
|
245
|
+
private readonly logger: Pick<Console, 'error'>;
|
|
246
|
+
private readonly sendCompletionNotification: CompletionNotificationSender;
|
|
247
|
+
private readonly publishTerminalSnapshot: (task: BgTaskSnapshot) => void;
|
|
248
|
+
private readonly posixProcessGroupKillStates = new WeakMap<BgTask, PosixProcessGroupKillState>();
|
|
249
|
+
private readonly windowsKillStates = new WeakMap<BgTask, WindowsKillState>();
|
|
250
|
+
private readonly terminalPublicationAbandonSignals = new WeakMap<
|
|
251
|
+
BgTask,
|
|
252
|
+
TerminalPublicationAbandonSignal
|
|
253
|
+
>();
|
|
254
|
+
private readonly reloadShellOwner: ReloadShellOwnerHubV1 | undefined;
|
|
255
|
+
private reloadShellLease: ReloadShellActivationLeaseV1 | undefined;
|
|
256
|
+
private reloadShellIdentity: ReloadShellIdentityV1 | undefined;
|
|
257
|
+
|
|
258
|
+
constructor(options: BackgroundTaskRegistryOptions) {
|
|
259
|
+
this.terminalPublicationClosedSignal = new Promise((resolve) => {
|
|
260
|
+
this.resolveTerminalPublicationClosedSignal = resolve;
|
|
261
|
+
});
|
|
262
|
+
this.spawn =
|
|
263
|
+
options.spawn ?? ((command, args, spawnOptions) => nodeSpawn(command, args, spawnOptions));
|
|
264
|
+
this.killProcess = options.killProcess ?? process.kill.bind(process);
|
|
265
|
+
this.platform = options.platform ?? process.platform;
|
|
266
|
+
this.env = options.env ?? process.env;
|
|
267
|
+
this.shellPolicy = options.shellPolicy;
|
|
268
|
+
this.tasksDir = options.tasksDir ?? defaultTasksDir();
|
|
269
|
+
this.shellPolicyEnv = options.shellPolicy === undefined ? { ...this.env } : undefined;
|
|
270
|
+
const taskkillEnv = this.env;
|
|
271
|
+
this.killTree =
|
|
272
|
+
options.killTree ??
|
|
273
|
+
((pid, phase, signal) => {
|
|
274
|
+
const taskkillOptions: WindowsTaskkillOptions =
|
|
275
|
+
signal === undefined ? { env: taskkillEnv } : { env: taskkillEnv, signal };
|
|
276
|
+
return runWindowsTaskkill(pid, phase, taskkillOptions);
|
|
277
|
+
});
|
|
278
|
+
this.makeTaskIdFn = options.makeTaskId ?? defaultTaskId;
|
|
279
|
+
this.now = options.now ?? Date.now;
|
|
280
|
+
this.maxOutputBytes = options.maxOutputBytes ?? MAX_OUTPUT_BYTES;
|
|
281
|
+
this.maxRecentTasks = options.maxRecentTasks ?? MAX_RECENT_TASKS;
|
|
282
|
+
this.killGraceMs = options.killGraceMs ?? KILL_GRACE_MS;
|
|
283
|
+
this.stopWaitMs = options.stopWaitMs ?? STOP_WAIT_MS;
|
|
284
|
+
this.taskAdmissionTimeoutMs = BackgroundTaskRegistry.positiveTimeout(
|
|
285
|
+
options.taskAdmissionTimeoutMs,
|
|
286
|
+
TASK_ADMISSION_TIMEOUT_MS,
|
|
287
|
+
'taskAdmissionTimeoutMs',
|
|
288
|
+
);
|
|
289
|
+
this.logger = options.logger ?? console;
|
|
290
|
+
this.sendCompletionNotification = options.sendCompletionNotification;
|
|
291
|
+
this.publishTerminalSnapshot = options.publishTerminal ?? noopOnChange;
|
|
292
|
+
this.reloadShellOwner = options.reloadShellOwner;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
isShuttingDown(): boolean {
|
|
296
|
+
return this.shuttingDown;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
private resolvedShellPolicy(): ResolvedShellPolicy {
|
|
300
|
+
const existing = this.shellPolicy;
|
|
301
|
+
if (existing !== undefined) return existing;
|
|
302
|
+
const resolved = resolveShellPolicy(
|
|
303
|
+
this.platform,
|
|
304
|
+
this.shellPolicyEnv ?? this.env,
|
|
305
|
+
process.cwd(),
|
|
306
|
+
);
|
|
307
|
+
this.shellPolicy = resolved;
|
|
308
|
+
return resolved;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
private static positiveTimeout(
|
|
312
|
+
value: number | undefined,
|
|
313
|
+
fallback: number,
|
|
314
|
+
label: string,
|
|
315
|
+
): number {
|
|
316
|
+
const candidate = value ?? fallback;
|
|
317
|
+
if (!Number.isFinite(candidate) || candidate <= 0) {
|
|
318
|
+
throw new Error(`${label} must be a positive finite number`);
|
|
319
|
+
}
|
|
320
|
+
return Math.max(1, Math.floor(candidate));
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
private beginTaskAdmission(kind: string): TaskAdmission {
|
|
324
|
+
this.assertTaskAdmissionOpen(kind);
|
|
325
|
+
const controller = new AbortController();
|
|
326
|
+
const admission: TaskAdmission = {
|
|
327
|
+
kind,
|
|
328
|
+
controller,
|
|
329
|
+
deadlineAt: Date.now() + this.taskAdmissionTimeoutMs,
|
|
330
|
+
timeoutMs: this.taskAdmissionTimeoutMs,
|
|
331
|
+
timeoutHandle: undefined,
|
|
332
|
+
released: false,
|
|
333
|
+
};
|
|
334
|
+
admission.timeoutHandle = setTimeout(() => {
|
|
335
|
+
if (admission.released || admission.controller.signal.aborted) return;
|
|
336
|
+
admission.controller.abort(
|
|
337
|
+
new BackgroundTaskAdmissionTimeoutError(admission.kind, admission.timeoutMs),
|
|
338
|
+
);
|
|
339
|
+
}, admission.timeoutMs);
|
|
340
|
+
this.activeTaskAdmissions.add(admission);
|
|
341
|
+
return admission;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
private releaseTaskAdmission(admission: TaskAdmission): void {
|
|
345
|
+
if (admission.released) return;
|
|
346
|
+
admission.released = true;
|
|
347
|
+
if (admission.timeoutHandle !== undefined) {
|
|
348
|
+
clearTimeout(admission.timeoutHandle);
|
|
349
|
+
admission.timeoutHandle = undefined;
|
|
350
|
+
}
|
|
351
|
+
this.activeTaskAdmissions.delete(admission);
|
|
352
|
+
if (this.activeTaskAdmissions.size !== 0) return;
|
|
353
|
+
for (const resolve of this.taskAdmissionDrainWaiters) resolve();
|
|
354
|
+
this.taskAdmissionDrainWaiters.clear();
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
private taskAdmissionError(admission: TaskAdmission): Error {
|
|
358
|
+
const reason = admission.controller.signal.reason;
|
|
359
|
+
if (reason instanceof Error) return reason;
|
|
360
|
+
if (this.shuttingDown || this.taskAdmissionsClosed) {
|
|
361
|
+
return new BackgroundTaskAdmissionClosedError(admission.kind);
|
|
362
|
+
}
|
|
363
|
+
return new BackgroundTaskAdmissionTimeoutError(admission.kind, admission.timeoutMs);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
private surfacedTaskAdmissionError(admission: TaskAdmission, ...details: unknown[]): Error {
|
|
367
|
+
const primary = this.taskAdmissionError(admission);
|
|
368
|
+
const meaningful = details.filter((detail) => {
|
|
369
|
+
if (detail === undefined || detail === primary) return false;
|
|
370
|
+
if (typeof detail !== 'object' || detail === null) return true;
|
|
371
|
+
return (
|
|
372
|
+
Reflect.get(detail, 'name') !== 'AbortError' &&
|
|
373
|
+
Reflect.get(detail, 'code') !== Reflect.get(primary, 'code')
|
|
374
|
+
);
|
|
375
|
+
});
|
|
376
|
+
if (meaningful.length === 0) return primary;
|
|
377
|
+
return new AggregateError(
|
|
378
|
+
[primary, ...meaningful],
|
|
379
|
+
`${primary.message}; admission cancellation or cleanup reported additional failures: ${meaningful.map(BackgroundTaskRegistry.errorMessage).join('; ')}`,
|
|
380
|
+
);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
private assertTaskAdmissionOpen(kind: string, admission?: TaskAdmission): void {
|
|
384
|
+
if (admission?.controller.signal.aborted === true) throw this.taskAdmissionError(admission);
|
|
385
|
+
if (this.shuttingDown || this.taskAdmissionsClosed) {
|
|
386
|
+
throw new BackgroundTaskAdmissionClosedError(kind);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
private async awaitTaskAdmissionBoundary<T>(
|
|
391
|
+
promise: Promise<T>,
|
|
392
|
+
admission: TaskAdmission,
|
|
393
|
+
): Promise<T> {
|
|
394
|
+
try {
|
|
395
|
+
const value = await promise;
|
|
396
|
+
this.assertTaskAdmissionOpen(admission.kind, admission);
|
|
397
|
+
return value;
|
|
398
|
+
} catch (error) {
|
|
399
|
+
if (admission.controller.signal.aborted && typeof error === 'object' && error !== null) {
|
|
400
|
+
const isPlainAbort = Reflect.get(error, 'name') === 'AbortError';
|
|
401
|
+
const isCleanDurableCancellation =
|
|
402
|
+
Reflect.get(error, 'code') === 'durable_file_cancelled' &&
|
|
403
|
+
Reflect.get(error, 'renameCompleted') !== true &&
|
|
404
|
+
Array.isArray(Reflect.get(error, 'cleanupFailures')) &&
|
|
405
|
+
(Reflect.get(error, 'cleanupFailures') as unknown[]).length === 0;
|
|
406
|
+
if (isPlainAbort || isCleanDurableCancellation) {
|
|
407
|
+
throw this.taskAdmissionError(admission);
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
throw error;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
closeTaskAdmissions(): void {
|
|
415
|
+
if (this.taskAdmissionsClosed) return;
|
|
416
|
+
this.taskAdmissionsClosed = true;
|
|
417
|
+
for (const admission of this.activeTaskAdmissions) {
|
|
418
|
+
if (!admission.controller.signal.aborted) {
|
|
419
|
+
admission.controller.abort(new BackgroundTaskAdmissionClosedError(admission.kind));
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
waitForTaskAdmissions(): Promise<void> {
|
|
425
|
+
if (this.activeTaskAdmissions.size === 0) return Promise.resolve();
|
|
426
|
+
return new Promise((resolve) => {
|
|
427
|
+
this.taskAdmissionDrainWaiters.add(resolve);
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
setShuttingDown(value: boolean): void {
|
|
432
|
+
if (value) {
|
|
433
|
+
this.shuttingDown = true;
|
|
434
|
+
this.closeTaskAdmissions();
|
|
435
|
+
this.closeTerminalPublication('registry_shutdown');
|
|
436
|
+
return;
|
|
437
|
+
}
|
|
438
|
+
// Publication and admission closure belong to one extension activation and
|
|
439
|
+
// are one-way. Pi session replacement creates a fresh registry; an old
|
|
440
|
+
// registry must not be reopened by a late lifecycle continuation.
|
|
441
|
+
if (!this.terminalPublicationClosed && !this.taskAdmissionsClosed) this.shuttingDown = false;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
closeTerminalPublication(reason: TerminalPublicationClosureReason): void {
|
|
445
|
+
if (!this.terminalPublicationClosed) {
|
|
446
|
+
this.terminalPublicationClosed = true;
|
|
447
|
+
this.terminalPublicationCloseReason = reason;
|
|
448
|
+
this.resolveTerminalPublicationClosedSignal(reason);
|
|
449
|
+
}
|
|
450
|
+
const effectiveReason = this.terminalPublicationCloseReason ?? reason;
|
|
451
|
+
for (const task of this.tasks.values()) {
|
|
452
|
+
if (task.terminalPublicationState === 'pending' && task.terminalEmitInFlight === true) {
|
|
453
|
+
// A synchronous listener can close the service while emit() is still on
|
|
454
|
+
// the stack. Dispose queued work now, but let the emitter's return/throw
|
|
455
|
+
// settle the in-flight attempt exactly once.
|
|
456
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
457
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
458
|
+
task.terminalPublishRetryHandle = undefined;
|
|
459
|
+
}
|
|
460
|
+
task.terminalPublicationGate = undefined;
|
|
461
|
+
continue;
|
|
462
|
+
}
|
|
463
|
+
const shouldLog =
|
|
464
|
+
task.terminalPublicationState === 'pending' &&
|
|
465
|
+
task.status !== 'running' &&
|
|
466
|
+
(task.terminalPublishAttempts > 0 ||
|
|
467
|
+
task.terminalPublishRetryHandle !== undefined ||
|
|
468
|
+
task.terminalPublicationGate !== undefined);
|
|
469
|
+
this.abandonTerminalPublication(task, effectiveReason, undefined, shouldLog);
|
|
470
|
+
}
|
|
471
|
+
this.pruneOldTasks();
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
allTasks(): BgTask[] {
|
|
475
|
+
return [...this.tasks.values()];
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
snapshot(task: BgTask): BgTaskSnapshot {
|
|
479
|
+
return snapshot(task);
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// Delete the on-disk files of finished tasks in the given scope. Running
|
|
483
|
+
// tasks are left untouched and reported. Current-run tasks are also dropped
|
|
484
|
+
// from the in-memory registry so /jobs stays consistent.
|
|
485
|
+
async cleanupFinishedTasks(scope: CleanupScope, taskId?: string): Promise<CleanupSummary> {
|
|
486
|
+
const summary = await cleanupTasksOnDisk({
|
|
487
|
+
tasksDir: this.tasksDir,
|
|
488
|
+
sessionRunDir: this.runtimeDir?.abs,
|
|
489
|
+
scope,
|
|
490
|
+
taskId,
|
|
491
|
+
});
|
|
492
|
+
for (const deleted of summary.deleted) {
|
|
493
|
+
this.tasks.delete(deleted.id);
|
|
494
|
+
}
|
|
495
|
+
return summary;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
hasCurrentReloadLease(): boolean {
|
|
499
|
+
const lease = this.reloadShellLease;
|
|
500
|
+
return lease !== undefined && this.reloadShellOwner?.isCurrentLease(lease) === true;
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
async stageReloadActivation(
|
|
504
|
+
claim: ReloadShellActivationClaimV1,
|
|
505
|
+
): Promise<ReloadShellHostAdapterV1> {
|
|
506
|
+
if (claim.protocol !== RELOAD_SHELL_OWNER_PROTOCOL) {
|
|
507
|
+
throw new ReloadSurvivalError(
|
|
508
|
+
'pi_bg_reload_owner_protocol_incompatible',
|
|
509
|
+
'activation claim does not use the supported reload shell owner protocol',
|
|
510
|
+
);
|
|
511
|
+
}
|
|
512
|
+
const staged: ReloadableShellExecutionV1[] = [];
|
|
513
|
+
const ids = new Set<string>();
|
|
514
|
+
for (const execution of claim.executions) {
|
|
515
|
+
if (
|
|
516
|
+
execution.protocol !== RELOAD_SHELL_OWNER_PROTOCOL ||
|
|
517
|
+
execution.task.reloadExecution !== execution ||
|
|
518
|
+
execution.task.surviveReload !== true
|
|
519
|
+
) {
|
|
520
|
+
throw new ReloadSurvivalError(
|
|
521
|
+
'pi_bg_reload_owner_protocol_incompatible',
|
|
522
|
+
'activation claim contains an incompatible reload shell execution',
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
if (ids.has(execution.task.id) || this.tasks.has(execution.task.id)) {
|
|
526
|
+
throw new ReloadSurvivalError(
|
|
527
|
+
'pi_bg_reload_owner_activation_conflict',
|
|
528
|
+
`claimed task id ${execution.task.id} conflicts with the fresh registry`,
|
|
529
|
+
);
|
|
530
|
+
}
|
|
531
|
+
ids.add(execution.task.id);
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
try {
|
|
535
|
+
for (const execution of claim.executions) {
|
|
536
|
+
const task = execution.task;
|
|
537
|
+
task.reloadHostDeliveryInFlight = false;
|
|
538
|
+
task.reloadHostDeliverySettled = false;
|
|
539
|
+
task.reloadHostNotificationSettled = false;
|
|
540
|
+
execution.updateLeaseAudit(claim.generation, (task.reloadSurvival?.handoffCount ?? 0) + 1);
|
|
541
|
+
this.tasks.set(task.id, task);
|
|
542
|
+
staged.push(execution);
|
|
543
|
+
}
|
|
544
|
+
await Promise.all(staged.map(async (execution) => this.writeMetadata(execution.task)));
|
|
545
|
+
} catch (error) {
|
|
546
|
+
for (const execution of staged) this.tasks.delete(execution.task.id);
|
|
547
|
+
throw error;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
let boundLease: ReloadShellActivationLeaseV1 | undefined;
|
|
551
|
+
return {
|
|
552
|
+
activationNonce: claim.activationNonce,
|
|
553
|
+
onBound: (lease) => {
|
|
554
|
+
if (
|
|
555
|
+
lease.activationNonce !== claim.activationNonce ||
|
|
556
|
+
lease.generation !== claim.generation ||
|
|
557
|
+
lease.identityKey !== claim.identityKey
|
|
558
|
+
) {
|
|
559
|
+
throw new ReloadSurvivalError(
|
|
560
|
+
'pi_bg_reload_owner_stale_claim',
|
|
561
|
+
'committed lease does not match its staged activation claim',
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
boundLease = lease;
|
|
565
|
+
this.reloadShellLease = lease;
|
|
566
|
+
this.reloadShellIdentity = claim.identity;
|
|
567
|
+
},
|
|
568
|
+
onChanged: (execution) => {
|
|
569
|
+
const lease = boundLease;
|
|
570
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
571
|
+
},
|
|
572
|
+
onTerminal: (execution) => {
|
|
573
|
+
const lease = boundLease;
|
|
574
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
575
|
+
void this.deliverReloadTerminal(execution, lease);
|
|
576
|
+
},
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
abortReloadActivation(claim: ReloadShellActivationClaimV1): void {
|
|
581
|
+
for (const execution of claim.executions) {
|
|
582
|
+
const task = execution.task;
|
|
583
|
+
if (this.tasks.get(task.id) !== task) continue;
|
|
584
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
585
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
586
|
+
task.terminalPublishRetryHandle = undefined;
|
|
587
|
+
}
|
|
588
|
+
task.terminalPublicationGate = undefined;
|
|
589
|
+
task.terminalPublishInFlight = false;
|
|
590
|
+
this.tasks.delete(task.id);
|
|
591
|
+
}
|
|
592
|
+
if (this.reloadShellLease?.activationNonce === claim.activationNonce) {
|
|
593
|
+
this.reloadShellLease = undefined;
|
|
594
|
+
this.reloadShellIdentity = undefined;
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
prepareReloadHandoff(lease: ReloadShellActivationLeaseV1): readonly BgTask[] {
|
|
599
|
+
if (this.reloadShellOwner === undefined || this.reloadShellLease !== lease) {
|
|
600
|
+
throw new ReloadSurvivalError(
|
|
601
|
+
'pi_bg_reload_owner_stale_claim',
|
|
602
|
+
'registry does not own the requested reload activation lease',
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
const executions = this.reloadShellOwner.beginReloadHandoff(lease);
|
|
606
|
+
const tasks: BgTask[] = [];
|
|
607
|
+
for (const execution of executions) {
|
|
608
|
+
const task = execution.task;
|
|
609
|
+
if (this.tasks.get(task.id) !== task) {
|
|
610
|
+
throw new ReloadSurvivalError(
|
|
611
|
+
'pi_bg_reload_owner_stale_claim',
|
|
612
|
+
`registry no longer owns survivor ${task.id}`,
|
|
613
|
+
);
|
|
614
|
+
}
|
|
615
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
616
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
617
|
+
task.terminalPublishRetryHandle = undefined;
|
|
618
|
+
}
|
|
619
|
+
task.terminalPublicationGate = undefined;
|
|
620
|
+
task.terminalPublishInFlight = false;
|
|
621
|
+
task.reloadHostDeliveryInFlight = false;
|
|
622
|
+
task.reloadHostDeliverySettled = false;
|
|
623
|
+
task.reloadHostNotificationSettled = false;
|
|
624
|
+
this.tasks.delete(task.id);
|
|
625
|
+
tasks.push(task);
|
|
626
|
+
}
|
|
627
|
+
this.reloadShellLease = undefined;
|
|
628
|
+
this.reloadShellIdentity = undefined;
|
|
629
|
+
return Object.freeze(tasks);
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
async waitForReloadHostSettlement(timeoutMs = this.stopWaitMs): Promise<void> {
|
|
633
|
+
const deadline = Date.now() + timeoutMs;
|
|
634
|
+
while (true) {
|
|
635
|
+
const unsettled = [...this.tasks.values()].filter(
|
|
636
|
+
(task) =>
|
|
637
|
+
task.reloadExecution?.phase === 'terminal' &&
|
|
638
|
+
(task.reloadHostNotificationSettled !== true ||
|
|
639
|
+
task.terminalPublicationState === 'pending'),
|
|
640
|
+
);
|
|
641
|
+
if (unsettled.length === 0) return;
|
|
642
|
+
const remaining = deadline - Date.now();
|
|
643
|
+
if (remaining <= 0) {
|
|
644
|
+
throw new Error(
|
|
645
|
+
`Timed out waiting for reload shell host settlement: ${unsettled.map((task) => task.id).join(', ')}`,
|
|
646
|
+
);
|
|
647
|
+
}
|
|
648
|
+
await new Promise<void>((resolve) => setTimeout(resolve, Math.min(10, remaining)));
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
releaseReloadActivation(lease: ReloadShellActivationLeaseV1): void {
|
|
653
|
+
if (this.reloadShellOwner === undefined) return;
|
|
654
|
+
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) return;
|
|
655
|
+
this.reloadShellOwner.releaseActivation(lease);
|
|
656
|
+
this.reloadShellLease = undefined;
|
|
657
|
+
this.reloadShellIdentity = undefined;
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
currentReloadLease(): ReloadShellActivationLeaseV1 | undefined {
|
|
661
|
+
return this.hasCurrentReloadLease() ? this.reloadShellLease : undefined;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
private ownsReloadExecution(
|
|
665
|
+
execution: ReloadableShellExecutionV1,
|
|
666
|
+
lease: ReloadShellActivationLeaseV1 | undefined,
|
|
667
|
+
): lease is ReloadShellActivationLeaseV1 {
|
|
668
|
+
return (
|
|
669
|
+
lease !== undefined &&
|
|
670
|
+
this.reloadShellLease === lease &&
|
|
671
|
+
this.reloadShellOwner?.isCurrentLease(lease) === true &&
|
|
672
|
+
this.tasks.get(execution.task.id) === execution.task &&
|
|
673
|
+
execution.task.reloadExecution === execution
|
|
674
|
+
);
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
async ensureRuntimeDir(ctx: BackgroundTaskContext): Promise<RuntimeDir> {
|
|
678
|
+
if (this.runtimeDir) return this.runtimeDir;
|
|
679
|
+
const sessionId = sanitizePathSegment(ctx.sessionId ?? `session-${String(process.pid)}`);
|
|
680
|
+
const runId = `${sessionId}-${String(process.pid)}`;
|
|
681
|
+
const runtimeDirAbs = join(this.tasksDir, runId);
|
|
682
|
+
await mkdir(runtimeDirAbs, { recursive: true });
|
|
683
|
+
this.runtimeDir = { abs: runtimeDirAbs, display: runtimeDirAbs };
|
|
684
|
+
return this.runtimeDir;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
private async destroyTaskStream(task: BgTask): Promise<void> {
|
|
688
|
+
const stream = task.stream;
|
|
689
|
+
if (stream === undefined || stream.closed) return;
|
|
690
|
+
await new Promise<void>((resolve) => {
|
|
691
|
+
const closed = () => {
|
|
692
|
+
stream.off('close', closed);
|
|
693
|
+
resolve();
|
|
694
|
+
};
|
|
695
|
+
stream.once('close', closed);
|
|
696
|
+
if (!stream.destroyed) stream.destroy();
|
|
697
|
+
if (stream.closed) closed();
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
private async discardUnspawnedTask(task: BgTask, paths: readonly string[]): Promise<void> {
|
|
702
|
+
this.tasks.delete(task.id);
|
|
703
|
+
task.finalized = true;
|
|
704
|
+
task.status = 'failed';
|
|
705
|
+
if (task.timeoutHandle !== undefined) clearTimeout(task.timeoutHandle);
|
|
706
|
+
if (task.killEscalationTimer !== undefined) clearTimeout(task.killEscalationTimer);
|
|
707
|
+
await this.destroyTaskStream(task);
|
|
708
|
+
const removals = await Promise.allSettled(paths.map((path) => rm(path, { force: true })));
|
|
709
|
+
const failures: Error[] = [];
|
|
710
|
+
for (let index = 0; index < removals.length; index++) {
|
|
711
|
+
const result = removals[index];
|
|
712
|
+
if (result?.status !== 'rejected') continue;
|
|
713
|
+
const path = paths[index] ?? '<unknown admission artifact>';
|
|
714
|
+
failures.push(
|
|
715
|
+
new Error(
|
|
716
|
+
`Failed to remove interrupted admission artifact ${path}: ${BackgroundTaskRegistry.errorMessage(result.reason)}`,
|
|
717
|
+
),
|
|
718
|
+
);
|
|
719
|
+
}
|
|
720
|
+
if (failures.length > 0) {
|
|
721
|
+
throw new AggregateError(failures, 'Interrupted task admission artifact cleanup failed');
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
private captureSpawnedChild(task: BgTask, child: BackgroundTaskChildProcess): void {
|
|
726
|
+
task.child = child;
|
|
727
|
+
task.pid = child.pid;
|
|
728
|
+
if (
|
|
729
|
+
this.platform !== 'win32' &&
|
|
730
|
+
child.pid !== undefined &&
|
|
731
|
+
Number.isSafeInteger(child.pid) &&
|
|
732
|
+
child.pid > 0
|
|
733
|
+
) {
|
|
734
|
+
// Capture detached-group ownership once, directly from this spawn. Never
|
|
735
|
+
// reconstruct signal authority from mutable task metadata or a later PID.
|
|
736
|
+
task.ownedPosixProcessGroupId = child.pid;
|
|
737
|
+
}
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
private bindOwnedTaskToAdmission(task: BgTask, admission: TaskAdmission): () => void {
|
|
741
|
+
const cancelOwnedTask = (): void => {
|
|
742
|
+
this.stopOwnedTaskAfterAdmissionCancellation(task, this.taskAdmissionError(admission));
|
|
743
|
+
};
|
|
744
|
+
admission.controller.signal.addEventListener('abort', cancelOwnedTask, { once: true });
|
|
745
|
+
if (admission.controller.signal.aborted) cancelOwnedTask();
|
|
746
|
+
return () => {
|
|
747
|
+
admission.controller.signal.removeEventListener('abort', cancelOwnedTask);
|
|
748
|
+
};
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
private stopOwnedTaskAfterAdmissionCancellation(task: BgTask, error: Error): void {
|
|
752
|
+
if (task.status !== 'running') return;
|
|
753
|
+
task.killKind = error instanceof BackgroundTaskAdmissionTimeoutError ? 'timeout' : 'shutdown';
|
|
754
|
+
task.error = error.message;
|
|
755
|
+
try {
|
|
756
|
+
this.requestKill(task, 'SIGTERM');
|
|
757
|
+
} catch (killError) {
|
|
758
|
+
this.logger.error(
|
|
759
|
+
`[background-tasks] failed to stop ${task.id} after admission cancellation:`,
|
|
760
|
+
killError,
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
async startTask(
|
|
766
|
+
ctx: BackgroundTaskContext,
|
|
767
|
+
command: string,
|
|
768
|
+
options: StartTaskOptions = {},
|
|
769
|
+
): Promise<BgTask> {
|
|
770
|
+
const hasSurvival = Object.hasOwn(options, 'surviveReload');
|
|
771
|
+
if (hasSurvival && typeof options.surviveReload !== 'boolean') {
|
|
772
|
+
throw new ReloadSurvivalError(
|
|
773
|
+
'pi_bg_survive_reload_invalid',
|
|
774
|
+
'surviveReload must be a boolean when present',
|
|
775
|
+
);
|
|
776
|
+
}
|
|
777
|
+
const surviveReload = options.surviveReload === true;
|
|
778
|
+
const reloadLease = surviveReload ? this.currentReloadLease() : undefined;
|
|
779
|
+
if (surviveReload && reloadLease === undefined) {
|
|
780
|
+
throw new ReloadSurvivalError(
|
|
781
|
+
'pi_bg_reload_owner_unavailable',
|
|
782
|
+
'no successfully bound same-process reload owner activation is available',
|
|
783
|
+
);
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
const admission = this.beginTaskAdmission('a background task');
|
|
787
|
+
try {
|
|
788
|
+
if (surviveReload && reloadLease !== undefined) {
|
|
789
|
+
return await this.startReloadableTaskAdmitted(
|
|
790
|
+
ctx,
|
|
791
|
+
command,
|
|
792
|
+
options,
|
|
793
|
+
admission,
|
|
794
|
+
reloadLease,
|
|
795
|
+
);
|
|
796
|
+
}
|
|
797
|
+
return await this.startTaskAdmitted(ctx, command, options, admission);
|
|
798
|
+
} finally {
|
|
799
|
+
this.releaseTaskAdmission(admission);
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
private async startReloadableTaskAdmitted(
|
|
804
|
+
ctx: BackgroundTaskContext,
|
|
805
|
+
command: string,
|
|
806
|
+
options: StartTaskOptions,
|
|
807
|
+
admission: TaskAdmission,
|
|
808
|
+
lease: ReloadShellActivationLeaseV1,
|
|
809
|
+
): Promise<BgTask> {
|
|
810
|
+
const normalizedCommand = command.trim();
|
|
811
|
+
if (!normalizedCommand) throw new Error('Background command is empty');
|
|
812
|
+
if (
|
|
813
|
+
this.reloadShellOwner === undefined ||
|
|
814
|
+
this.reloadShellIdentity === undefined ||
|
|
815
|
+
this.reloadShellLease !== lease ||
|
|
816
|
+
!this.reloadShellOwner.isCurrentLease(lease)
|
|
817
|
+
) {
|
|
818
|
+
throw new ReloadSurvivalError(
|
|
819
|
+
'pi_bg_reload_owner_unavailable',
|
|
820
|
+
'reload owner activation became unavailable before launch',
|
|
821
|
+
);
|
|
822
|
+
}
|
|
823
|
+
if (ctx.sessionId !== this.reloadShellIdentity.sessionId) {
|
|
824
|
+
throw new ReloadSurvivalError(
|
|
825
|
+
'pi_bg_reload_owner_stale_claim',
|
|
826
|
+
'launch context session id does not match the bound reload owner identity',
|
|
827
|
+
);
|
|
828
|
+
}
|
|
829
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
830
|
+
|
|
831
|
+
const shellPolicy = this.resolvedShellPolicy();
|
|
832
|
+
const invocation = shellInvocationForPolicy(normalizedCommand, shellPolicy);
|
|
833
|
+
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
834
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
835
|
+
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) {
|
|
836
|
+
throw new ReloadSurvivalError(
|
|
837
|
+
'pi_bg_reload_owner_stale_claim',
|
|
838
|
+
'reload owner activation changed during task preflight',
|
|
839
|
+
);
|
|
840
|
+
}
|
|
841
|
+
|
|
842
|
+
const id = this.makeTaskIdFn();
|
|
843
|
+
const outputAbsPath = join(dir.abs, `${id}.output`);
|
|
844
|
+
const metadataAbsPath = join(dir.abs, `${id}.json`);
|
|
845
|
+
const outputPath = join(dir.display, `${id}.output`);
|
|
846
|
+
const timeoutSeconds =
|
|
847
|
+
typeof options.timeoutSeconds === 'number' &&
|
|
848
|
+
Number.isFinite(options.timeoutSeconds) &&
|
|
849
|
+
options.timeoutSeconds > 0
|
|
850
|
+
? Math.floor(options.timeoutSeconds)
|
|
851
|
+
: undefined;
|
|
852
|
+
const taskName =
|
|
853
|
+
normalizeTaskName(options.name) ??
|
|
854
|
+
normalizeTaskName(options.description) ??
|
|
855
|
+
deriveTaskNameFromCommand(normalizedCommand);
|
|
856
|
+
const trimmedDescription = options.description?.trim();
|
|
857
|
+
const description =
|
|
858
|
+
trimmedDescription && trimmedDescription.length > 0 ? trimmedDescription : undefined;
|
|
859
|
+
const task: BgTask = {
|
|
860
|
+
id,
|
|
861
|
+
name: taskName,
|
|
862
|
+
command: normalizedCommand,
|
|
863
|
+
description,
|
|
864
|
+
status: 'running',
|
|
865
|
+
outputPath,
|
|
866
|
+
outputAbsPath,
|
|
867
|
+
metadataAbsPath,
|
|
868
|
+
cwd: ctx.cwd,
|
|
869
|
+
startTime: this.now(),
|
|
870
|
+
exitCode: undefined,
|
|
871
|
+
pid: undefined,
|
|
872
|
+
bytesWritten: 0,
|
|
873
|
+
surviveReload: true,
|
|
874
|
+
notified: false,
|
|
875
|
+
notifyOnCompletion: options.notifyOnCompletion ?? true,
|
|
876
|
+
triggerOnCompletion: options.triggerOnCompletion ?? false,
|
|
877
|
+
timeoutSeconds,
|
|
878
|
+
terminalPublished: false,
|
|
879
|
+
terminalPublicationState: 'pending',
|
|
880
|
+
terminalPublishAttempts: 0,
|
|
881
|
+
terminalPublicationGate: options.terminalPublicationGate,
|
|
882
|
+
shellPolicy: shellPolicySnapshot(shellPolicy),
|
|
883
|
+
waiters: [],
|
|
884
|
+
};
|
|
885
|
+
const launchNonce = randomBytes(16).toString('hex');
|
|
886
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
887
|
+
this.tasks.set(id, task);
|
|
888
|
+
|
|
889
|
+
let execution: ReloadableShellExecutionV1 | undefined;
|
|
890
|
+
let registered = false;
|
|
891
|
+
let committed = false;
|
|
892
|
+
let abortListener: (() => void) | undefined;
|
|
893
|
+
try {
|
|
894
|
+
execution = createReloadableShellExecutionV1({
|
|
895
|
+
task,
|
|
896
|
+
identity: this.reloadShellIdentity,
|
|
897
|
+
lease,
|
|
898
|
+
launchNonce,
|
|
899
|
+
invocation,
|
|
900
|
+
spawn: this.spawn,
|
|
901
|
+
killProcess: this.killProcess,
|
|
902
|
+
killTree: this.killTree,
|
|
903
|
+
platform: this.platform,
|
|
904
|
+
env: this.env,
|
|
905
|
+
maxOutputBytes: this.maxOutputBytes,
|
|
906
|
+
killGraceMs: this.killGraceMs,
|
|
907
|
+
stopWaitMs: this.stopWaitMs,
|
|
908
|
+
now: this.now,
|
|
909
|
+
logger: this.logger,
|
|
910
|
+
});
|
|
911
|
+
this.reloadShellOwner.registerExecution(lease, execution);
|
|
912
|
+
registered = true;
|
|
913
|
+
abortListener = () => {
|
|
914
|
+
if (execution === undefined) return;
|
|
915
|
+
const error = this.taskAdmissionError(admission);
|
|
916
|
+
execution.failAdmission(error);
|
|
917
|
+
const kind: KillKind =
|
|
918
|
+
error instanceof BackgroundTaskAdmissionTimeoutError ? 'timeout' : 'shutdown';
|
|
919
|
+
void execution.requestStop(kind, error.message).catch((stopError: unknown) => {
|
|
920
|
+
this.logger.error(
|
|
921
|
+
`[background-tasks] failed to stop reloadable task ${task.id} after admission cancellation:`,
|
|
922
|
+
stopError,
|
|
923
|
+
);
|
|
924
|
+
});
|
|
925
|
+
};
|
|
926
|
+
admission.controller.signal.addEventListener('abort', abortListener, { once: true });
|
|
927
|
+
if (admission.controller.signal.aborted) abortListener();
|
|
928
|
+
|
|
929
|
+
await this.awaitTaskAdmissionBoundary(
|
|
930
|
+
execution.commitInitialMetadata(admission.controller.signal),
|
|
931
|
+
admission,
|
|
932
|
+
);
|
|
933
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
934
|
+
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) {
|
|
935
|
+
throw new ReloadSurvivalError(
|
|
936
|
+
'pi_bg_reload_owner_stale_claim',
|
|
937
|
+
'reload owner activation changed before admission commit',
|
|
938
|
+
);
|
|
939
|
+
}
|
|
940
|
+
this.reloadShellOwner.markAdmissionCommitted(lease, execution);
|
|
941
|
+
committed = true;
|
|
942
|
+
return task;
|
|
943
|
+
} catch (error) {
|
|
944
|
+
const primary = admission.controller.signal.aborted
|
|
945
|
+
? this.taskAdmissionError(admission)
|
|
946
|
+
: error instanceof Error
|
|
947
|
+
? error
|
|
948
|
+
: new Error(String(error));
|
|
949
|
+
execution?.failAdmission(primary);
|
|
950
|
+
let cleanupError: unknown;
|
|
951
|
+
if (execution !== undefined) {
|
|
952
|
+
try {
|
|
953
|
+
if (task.status === 'running') {
|
|
954
|
+
await execution.requestStop(
|
|
955
|
+
admission.controller.signal.aborted &&
|
|
956
|
+
primary instanceof BackgroundTaskAdmissionTimeoutError
|
|
957
|
+
? 'timeout'
|
|
958
|
+
: 'shutdown',
|
|
959
|
+
primary.message,
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
} catch (stopError) {
|
|
963
|
+
cleanupError = stopError;
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
if (registered && !committed && execution !== undefined) {
|
|
967
|
+
if (this.reloadShellOwner.isCurrentLease(lease)) {
|
|
968
|
+
this.reloadShellOwner.releaseExecution(lease, execution);
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
this.tasks.delete(task.id);
|
|
972
|
+
if (execution === undefined) {
|
|
973
|
+
const removals = await Promise.allSettled([
|
|
974
|
+
rm(outputAbsPath, { force: true }),
|
|
975
|
+
rm(metadataAbsPath, { force: true }),
|
|
976
|
+
]);
|
|
977
|
+
const removalFailure = removals.find((result) => result.status === 'rejected');
|
|
978
|
+
if (removalFailure?.status === 'rejected') cleanupError = removalFailure.reason;
|
|
979
|
+
}
|
|
980
|
+
if (admission.controller.signal.aborted) {
|
|
981
|
+
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
982
|
+
}
|
|
983
|
+
if (cleanupError !== undefined) {
|
|
984
|
+
throw new AggregateError(
|
|
985
|
+
[primary, cleanupError],
|
|
986
|
+
`Failed to start reloadable background task and cleanup also failed: ${BackgroundTaskRegistry.errorMessage(cleanupError)}`,
|
|
987
|
+
);
|
|
988
|
+
}
|
|
989
|
+
throw new Error(`Failed to start background task: ${primary.message}`);
|
|
990
|
+
} finally {
|
|
991
|
+
if (abortListener !== undefined) {
|
|
992
|
+
admission.controller.signal.removeEventListener('abort', abortListener);
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
private async startTaskAdmitted(
|
|
998
|
+
ctx: BackgroundTaskContext,
|
|
999
|
+
command: string,
|
|
1000
|
+
options: StartTaskOptions,
|
|
1001
|
+
admission: TaskAdmission,
|
|
1002
|
+
): Promise<BgTask> {
|
|
1003
|
+
const normalizedCommand = command.trim();
|
|
1004
|
+
if (!normalizedCommand) throw new Error('Background command is empty');
|
|
1005
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1006
|
+
|
|
1007
|
+
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
1008
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1009
|
+
const id = this.makeTaskIdFn();
|
|
1010
|
+
const outputAbsPath = join(dir.abs, `${id}.output`);
|
|
1011
|
+
const metadataAbsPath = join(dir.abs, `${id}.json`);
|
|
1012
|
+
const outputPath = join(dir.display, `${id}.output`);
|
|
1013
|
+
const shellPolicy = this.resolvedShellPolicy();
|
|
1014
|
+
const invocation = shellInvocationForPolicy(normalizedCommand, shellPolicy);
|
|
1015
|
+
const timeoutSeconds =
|
|
1016
|
+
typeof options.timeoutSeconds === 'number' &&
|
|
1017
|
+
Number.isFinite(options.timeoutSeconds) &&
|
|
1018
|
+
options.timeoutSeconds > 0
|
|
1019
|
+
? Math.floor(options.timeoutSeconds)
|
|
1020
|
+
: undefined;
|
|
1021
|
+
const taskName =
|
|
1022
|
+
normalizeTaskName(options.name) ??
|
|
1023
|
+
normalizeTaskName(options.description) ??
|
|
1024
|
+
deriveTaskNameFromCommand(normalizedCommand);
|
|
1025
|
+
const trimmedDescription = options.description?.trim();
|
|
1026
|
+
const description =
|
|
1027
|
+
trimmedDescription && trimmedDescription.length > 0 ? trimmedDescription : undefined;
|
|
1028
|
+
|
|
1029
|
+
const task: BgTask = {
|
|
1030
|
+
id,
|
|
1031
|
+
name: taskName,
|
|
1032
|
+
command: normalizedCommand,
|
|
1033
|
+
description,
|
|
1034
|
+
status: 'running',
|
|
1035
|
+
outputPath,
|
|
1036
|
+
outputAbsPath,
|
|
1037
|
+
metadataAbsPath,
|
|
1038
|
+
cwd: ctx.cwd,
|
|
1039
|
+
startTime: this.now(),
|
|
1040
|
+
exitCode: undefined,
|
|
1041
|
+
pid: undefined,
|
|
1042
|
+
bytesWritten: 0,
|
|
1043
|
+
surviveReload: false,
|
|
1044
|
+
notified: false,
|
|
1045
|
+
notifyOnCompletion: options.notifyOnCompletion ?? true,
|
|
1046
|
+
triggerOnCompletion: options.triggerOnCompletion ?? false,
|
|
1047
|
+
timeoutSeconds,
|
|
1048
|
+
terminalPublished: false,
|
|
1049
|
+
terminalPublicationState: 'pending',
|
|
1050
|
+
terminalPublishAttempts: 0,
|
|
1051
|
+
terminalPublicationGate: options.terminalPublicationGate,
|
|
1052
|
+
shellPolicy: shellPolicySnapshot(shellPolicy),
|
|
1053
|
+
waiters: [],
|
|
1054
|
+
};
|
|
1055
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1056
|
+
this.tasks.set(id, task);
|
|
1057
|
+
|
|
1058
|
+
const stream = createWriteStream(outputAbsPath, { flags: 'a', encoding: 'utf8' });
|
|
1059
|
+
task.stream = stream;
|
|
1060
|
+
stream.on('error', (error) => {
|
|
1061
|
+
task.error = `Output file write failed: ${error.message}`;
|
|
1062
|
+
if (task.status === 'running') {
|
|
1063
|
+
task.killKind = 'output_cap';
|
|
1064
|
+
try {
|
|
1065
|
+
this.requestKill(task, 'SIGTERM');
|
|
1066
|
+
} catch (killError) {
|
|
1067
|
+
void this.finalizeTask(
|
|
1068
|
+
task,
|
|
1069
|
+
'failed',
|
|
1070
|
+
null,
|
|
1071
|
+
undefined,
|
|
1072
|
+
`${task.error}; kill failed: ${killError instanceof Error ? killError.message : String(killError)}`,
|
|
1073
|
+
);
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
});
|
|
1077
|
+
|
|
1078
|
+
let unbindAdmissionCancellation = (): void => undefined;
|
|
1079
|
+
try {
|
|
1080
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1081
|
+
const child = this.spawn(invocation.shell, invocation.args, {
|
|
1082
|
+
cwd: ctx.cwd,
|
|
1083
|
+
detached: this.platform !== 'win32',
|
|
1084
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
1085
|
+
env: this.env,
|
|
1086
|
+
windowsHide: true,
|
|
1087
|
+
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
|
|
1088
|
+
});
|
|
1089
|
+
|
|
1090
|
+
this.captureSpawnedChild(task, child);
|
|
1091
|
+
|
|
1092
|
+
child.stdout?.on('data', (data) => {
|
|
1093
|
+
this.appendChildOutput(task, data);
|
|
1094
|
+
});
|
|
1095
|
+
child.stderr?.on('data', (data) => {
|
|
1096
|
+
this.appendChildOutput(task, data);
|
|
1097
|
+
});
|
|
1098
|
+
|
|
1099
|
+
child.on('error', (error) => {
|
|
1100
|
+
this.writeNotice(task, `\n[background task spawn error: ${error.message}]\n`);
|
|
1101
|
+
void this.finalizeTask(task, 'failed', null, undefined, error.message);
|
|
1102
|
+
});
|
|
1103
|
+
|
|
1104
|
+
child.on('close', (code, signalName) => {
|
|
1105
|
+
let status: TaskStatus;
|
|
1106
|
+
let error: string | undefined;
|
|
1107
|
+
if (task.killKind === 'user' || task.killKind === 'shutdown') {
|
|
1108
|
+
status = 'killed';
|
|
1109
|
+
} else if (task.killKind === 'timeout') {
|
|
1110
|
+
status = 'failed';
|
|
1111
|
+
error = task.error ?? `Timed out after ${String(timeoutSeconds)}s`;
|
|
1112
|
+
} else if (task.killKind === 'output_cap') {
|
|
1113
|
+
status = 'failed';
|
|
1114
|
+
error = task.error ?? `Output exceeded cap of ${formatSize(this.maxOutputBytes)}`;
|
|
1115
|
+
} else if ((code ?? 0) === 0) {
|
|
1116
|
+
status = 'completed';
|
|
1117
|
+
} else {
|
|
1118
|
+
status = 'failed';
|
|
1119
|
+
const exitCode = code === null ? 'null' : String(code);
|
|
1120
|
+
error = `Exited with code ${exitCode}${signalName ? ` (${signalName})` : ''}`;
|
|
1121
|
+
}
|
|
1122
|
+
void this.finalizeTask(task, status, code, signalName, error);
|
|
1123
|
+
});
|
|
1124
|
+
|
|
1125
|
+
if (timeoutSeconds !== undefined) {
|
|
1126
|
+
task.timeoutHandle = setTimeout(() => {
|
|
1127
|
+
if (task.status !== 'running') return;
|
|
1128
|
+
task.killKind = 'timeout';
|
|
1129
|
+
task.error = `Timed out after ${String(timeoutSeconds)}s`;
|
|
1130
|
+
this.writeNotice(task, `\n[background task timeout: ${task.error}]\n`);
|
|
1131
|
+
try {
|
|
1132
|
+
this.requestKill(task, 'SIGTERM');
|
|
1133
|
+
} catch (error) {
|
|
1134
|
+
void this.finalizeTask(
|
|
1135
|
+
task,
|
|
1136
|
+
'failed',
|
|
1137
|
+
null,
|
|
1138
|
+
undefined,
|
|
1139
|
+
`${task.error}; kill failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
}, timeoutSeconds * 1000);
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
unbindAdmissionCancellation = this.bindOwnedTaskToAdmission(task, admission);
|
|
1146
|
+
await this.awaitTaskAdmissionBoundary(
|
|
1147
|
+
this.writeMetadata(task, admission.controller.signal),
|
|
1148
|
+
admission,
|
|
1149
|
+
);
|
|
1150
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1151
|
+
return task;
|
|
1152
|
+
} catch (error) {
|
|
1153
|
+
if (admission.controller.signal.aborted) {
|
|
1154
|
+
const admissionError = this.taskAdmissionError(admission);
|
|
1155
|
+
let cleanupError: unknown;
|
|
1156
|
+
if (task.child === undefined) {
|
|
1157
|
+
try {
|
|
1158
|
+
await this.discardUnspawnedTask(task, [outputAbsPath, metadataAbsPath]);
|
|
1159
|
+
} catch (cleanupFailure) {
|
|
1160
|
+
cleanupError = cleanupFailure;
|
|
1161
|
+
}
|
|
1162
|
+
} else {
|
|
1163
|
+
this.stopOwnedTaskAfterAdmissionCancellation(task, admissionError);
|
|
1164
|
+
}
|
|
1165
|
+
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
1166
|
+
}
|
|
1167
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1168
|
+
this.writeNotice(task, `\n[background task spawn exception: ${message}]\n`);
|
|
1169
|
+
await this.finalizeTask(task, 'failed', null, undefined, message);
|
|
1170
|
+
throw new Error(`Failed to start background task: ${message}`);
|
|
1171
|
+
} finally {
|
|
1172
|
+
unbindAdmissionCancellation();
|
|
1173
|
+
}
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
resolveTask(idOrPrefix: string): BgTask {
|
|
1177
|
+
const id = idOrPrefix.trim();
|
|
1178
|
+
if (!id) throw new Error('Task ID is required');
|
|
1179
|
+
const exact = this.tasks.get(id);
|
|
1180
|
+
if (exact) return exact;
|
|
1181
|
+
const matches = [...this.tasks.values()].filter((task) => task.id.startsWith(id));
|
|
1182
|
+
const onlyMatch = matches[0];
|
|
1183
|
+
if (matches.length === 1 && onlyMatch) return onlyMatch;
|
|
1184
|
+
if (matches.length > 1)
|
|
1185
|
+
throw new Error(
|
|
1186
|
+
`Ambiguous task ID prefix "${id}": ${matches.map((task) => task.id).join(', ')}`,
|
|
1187
|
+
);
|
|
1188
|
+
throw new Error(`Unknown background task ID: ${id}`);
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1191
|
+
async stopTask(task: BgTask, kind: KillKind, reason?: string): Promise<BgTask> {
|
|
1192
|
+
if (task.status !== 'running') {
|
|
1193
|
+
throw new Error(`Task ${task.id} is ${task.status}, not running`);
|
|
1194
|
+
}
|
|
1195
|
+
if (task.reloadExecution !== undefined) {
|
|
1196
|
+
return task.reloadExecution.requestStop(kind, reason);
|
|
1197
|
+
}
|
|
1198
|
+
const stopWaitMs = this.stopWaitMs;
|
|
1199
|
+
if (
|
|
1200
|
+
this.platform !== 'win32' &&
|
|
1201
|
+
task.posixProcessGroupSignalAuthorityReleased === true &&
|
|
1202
|
+
this.posixProcessGroupKillStates.get(task) === undefined
|
|
1203
|
+
) {
|
|
1204
|
+
const finalized = await this.waitForEnd(task, stopWaitMs);
|
|
1205
|
+
if (!finalized) {
|
|
1206
|
+
throw new Error(
|
|
1207
|
+
`Task ${task.id} did not finish terminalization within ${formatDuration(stopWaitMs)} after its process group signal authority was released`,
|
|
1208
|
+
);
|
|
1209
|
+
}
|
|
1210
|
+
return task;
|
|
1211
|
+
}
|
|
1212
|
+
task.killKind = kind;
|
|
1213
|
+
if (reason) task.error = reason;
|
|
1214
|
+
this.requestKill(task, 'SIGTERM');
|
|
1215
|
+
const stopped =
|
|
1216
|
+
this.platform === 'win32'
|
|
1217
|
+
? await this.waitForEndOrWindowsForceFailure(task, stopWaitMs)
|
|
1218
|
+
: await this.waitForEndOrPosixForceFailure(task, stopWaitMs);
|
|
1219
|
+
const posixForceFailure = this.posixProcessGroupKillStates.get(task)?.failure;
|
|
1220
|
+
if (posixForceFailure !== undefined) throw posixForceFailure;
|
|
1221
|
+
const windowsForceFailure = this.windowsKillStates.get(task)?.forceFailure;
|
|
1222
|
+
if (windowsForceFailure !== undefined) throw windowsForceFailure;
|
|
1223
|
+
if (!stopped) {
|
|
1224
|
+
throw new Error(
|
|
1225
|
+
`Task ${task.id} did not exit within ${formatDuration(stopWaitMs)} after cancellation`,
|
|
1226
|
+
);
|
|
1227
|
+
}
|
|
1228
|
+
return task;
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
async stopAllRunning(
|
|
1232
|
+
kind: KillKind,
|
|
1233
|
+
reason?: string,
|
|
1234
|
+
): Promise<{ stopped: number; failures: string[] }> {
|
|
1235
|
+
const running = this.allTasks().filter((task) => task.status === 'running');
|
|
1236
|
+
const failures: string[] = [];
|
|
1237
|
+
let stopped = 0;
|
|
1238
|
+
await Promise.all(
|
|
1239
|
+
running.map(async (task) => {
|
|
1240
|
+
try {
|
|
1241
|
+
await this.stopTask(task, kind, reason);
|
|
1242
|
+
stopped++;
|
|
1243
|
+
} catch (error) {
|
|
1244
|
+
failures.push(
|
|
1245
|
+
`${taskDisplayName(task)} (${task.id}): ${error instanceof Error ? error.message : String(error)}`,
|
|
1246
|
+
);
|
|
1247
|
+
}
|
|
1248
|
+
}),
|
|
1249
|
+
);
|
|
1250
|
+
return { stopped, failures };
|
|
1251
|
+
}
|
|
1252
|
+
|
|
1253
|
+
async getTaskLogs(
|
|
1254
|
+
task: BgTask,
|
|
1255
|
+
maxBytes: number,
|
|
1256
|
+
tail: boolean,
|
|
1257
|
+
): Promise<{ text: string; details: BgLogsDetails }> {
|
|
1258
|
+
if (!existsSync(task.outputAbsPath)) {
|
|
1259
|
+
throw new Error(`Output file does not exist for ${task.id}: ${task.outputPath}`);
|
|
1260
|
+
}
|
|
1261
|
+
const read = await boundedRead(task.outputAbsPath, maxBytes, tail);
|
|
1262
|
+
const direction = tail ? 'tail' : 'head';
|
|
1263
|
+
let text = read.content.length > 0 ? read.content : '(no output yet)';
|
|
1264
|
+
if (read.truncated) {
|
|
1265
|
+
const omitted = read.totalBytes - read.bytesRead;
|
|
1266
|
+
const notice = `\n\n[Showing ${direction} ${formatSize(read.bytesRead)} of ${formatSize(read.totalBytes)}; ${formatSize(omitted)} omitted. Full output: ${task.outputPath}]`;
|
|
1267
|
+
text = tail ? `${notice}\n\n${text}` : `${text}${notice}`;
|
|
1268
|
+
} else {
|
|
1269
|
+
text += `\n\n[Full output: ${task.outputPath}]`;
|
|
1270
|
+
}
|
|
1271
|
+
return {
|
|
1272
|
+
text,
|
|
1273
|
+
details: {
|
|
1274
|
+
task: snapshot(task),
|
|
1275
|
+
path: task.outputPath,
|
|
1276
|
+
bytesRead: read.bytesRead,
|
|
1277
|
+
truncated: read.truncated,
|
|
1278
|
+
tail,
|
|
1279
|
+
},
|
|
1280
|
+
};
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
private async writeMetadata(task: BgTask, signal?: AbortSignal): Promise<void> {
|
|
1284
|
+
await this.writeMetadataSnapshot(task, snapshot(task), signal);
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
private async writeMetadataSnapshot(
|
|
1288
|
+
task: BgTask,
|
|
1289
|
+
value: BgTaskSnapshot,
|
|
1290
|
+
signal?: AbortSignal,
|
|
1291
|
+
): Promise<void> {
|
|
1292
|
+
const write = async () => {
|
|
1293
|
+
await writeJsonAtomic(task.metadataAbsPath, value, signal);
|
|
1294
|
+
};
|
|
1295
|
+
const previous = task.metadataWriteChain ?? Promise.resolve();
|
|
1296
|
+
const next = previous.then(write, write);
|
|
1297
|
+
task.metadataWriteChain = next.catch(() => undefined);
|
|
1298
|
+
await next;
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
/** Cap-enforcing sink for all persisted task output; terminates the task once the byte cap is exceeded. */
|
|
1302
|
+
private writeToStream(task: BgTask, buffer: Buffer): void {
|
|
1303
|
+
if (!task.stream || task.stream.destroyed) return;
|
|
1304
|
+
if (buffer.length === 0) return;
|
|
1305
|
+
|
|
1306
|
+
const nextBytes = task.bytesWritten + buffer.length;
|
|
1307
|
+
if (nextBytes <= this.maxOutputBytes) {
|
|
1308
|
+
task.stream.write(buffer);
|
|
1309
|
+
task.bytesWritten = nextBytes;
|
|
1310
|
+
return;
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
const remaining = Math.max(0, this.maxOutputBytes - task.bytesWritten);
|
|
1314
|
+
if (remaining > 0) {
|
|
1315
|
+
task.stream.write(buffer.subarray(0, remaining));
|
|
1316
|
+
task.bytesWritten += remaining;
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
if (!task.capExceeded) {
|
|
1320
|
+
task.capExceeded = true;
|
|
1321
|
+
task.error = `Output exceeded cap of ${formatSize(this.maxOutputBytes)}; terminating task`;
|
|
1322
|
+
const notice = `\n\n[background task error: ${task.error}]\n`;
|
|
1323
|
+
task.stream.write(notice);
|
|
1324
|
+
task.bytesWritten += Buffer.byteLength(notice, 'utf8');
|
|
1325
|
+
task.killKind = 'output_cap';
|
|
1326
|
+
try {
|
|
1327
|
+
this.requestKill(task, 'SIGTERM');
|
|
1328
|
+
} catch (error) {
|
|
1329
|
+
task.error = `${task.error}; kill failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
1330
|
+
void this.finalizeTask(task, 'failed', null, undefined, task.error);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
/** Persist an internally generated notice (spawn/timeout/cap diagnostics) verbatim. */
|
|
1336
|
+
private writeNotice(task: BgTask, text: string): void {
|
|
1337
|
+
if (!text) return;
|
|
1338
|
+
this.writeToStream(task, Buffer.from(text, 'utf8'));
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
private appendChildOutput(task: BgTask, data: Buffer | string): void {
|
|
1342
|
+
if (!task.stream || task.stream.destroyed) return;
|
|
1343
|
+
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf8');
|
|
1344
|
+
if (buffer.length === 0) return;
|
|
1345
|
+
this.writeToStream(task, buffer);
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
private beginPosixProcessGroupKill(task: BgTask, armGrace: boolean): PosixProcessGroupKillState {
|
|
1349
|
+
const existing = this.posixProcessGroupKillStates.get(task);
|
|
1350
|
+
if (existing !== undefined) return existing;
|
|
1351
|
+
if (task.posixProcessGroupSignalAuthorityReleased === true) {
|
|
1352
|
+
throw new Error(`Task ${task.id} has released its POSIX process-group signal authority`);
|
|
1353
|
+
}
|
|
1354
|
+
const groupId = task.ownedPosixProcessGroupId;
|
|
1355
|
+
if (groupId === undefined) {
|
|
1356
|
+
throw new Error(`Task ${task.id} has no owned POSIX process group`);
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
let resolveCompletion = (): void => undefined;
|
|
1360
|
+
const completion = new Promise<void>((resolve) => {
|
|
1361
|
+
resolveCompletion = resolve;
|
|
1362
|
+
});
|
|
1363
|
+
// Finish ownership slightly before stopTask's waiter so a force/proof
|
|
1364
|
+
// failure is observed as that specific loud error instead of a generic
|
|
1365
|
+
// cancellation timeout.
|
|
1366
|
+
const reserveMs = Math.min(25, Math.max(1, Math.floor(this.stopWaitMs / 4)));
|
|
1367
|
+
const ownershipMs = Math.max(1, this.stopWaitMs - reserveMs);
|
|
1368
|
+
const state: PosixProcessGroupKillState = {
|
|
1369
|
+
groupId,
|
|
1370
|
+
completion,
|
|
1371
|
+
resolveCompletion,
|
|
1372
|
+
deadlineAt: Date.now() + ownershipMs,
|
|
1373
|
+
forceAttempted: false,
|
|
1374
|
+
settled: false,
|
|
1375
|
+
};
|
|
1376
|
+
this.posixProcessGroupKillStates.set(task, state);
|
|
1377
|
+
|
|
1378
|
+
if (armGrace) {
|
|
1379
|
+
// Publish the sole force owner before TERM. An injected signal can emit
|
|
1380
|
+
// close reentrantly; that close must see and await this exact state.
|
|
1381
|
+
task.killEscalationTimer = setTimeout(
|
|
1382
|
+
() => {
|
|
1383
|
+
task.killEscalationTimer = undefined;
|
|
1384
|
+
this.forceOwnedPosixProcessGroup(task, state);
|
|
1385
|
+
},
|
|
1386
|
+
Math.min(this.killGraceMs, ownershipMs),
|
|
1387
|
+
);
|
|
1388
|
+
// Unlike ordinary housekeeping timers, this owner stays referenced: a
|
|
1389
|
+
// departed leader must not let the host exit and strand its owned group.
|
|
1390
|
+
}
|
|
1391
|
+
return state;
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
private finishPosixProcessGroupKill(
|
|
1395
|
+
task: BgTask,
|
|
1396
|
+
state: PosixProcessGroupKillState,
|
|
1397
|
+
releaseOwnership: boolean,
|
|
1398
|
+
): void {
|
|
1399
|
+
if (state.settled) return;
|
|
1400
|
+
state.settled = true;
|
|
1401
|
+
this.clearKillEscalationTimer(task);
|
|
1402
|
+
if (state.verificationTimer !== undefined) {
|
|
1403
|
+
clearTimeout(state.verificationTimer);
|
|
1404
|
+
state.verificationTimer = undefined;
|
|
1405
|
+
}
|
|
1406
|
+
task.posixProcessGroupSignalAuthorityReleased = true;
|
|
1407
|
+
if (releaseOwnership && task.ownedPosixProcessGroupId === state.groupId) {
|
|
1408
|
+
delete task.ownedPosixProcessGroupId;
|
|
1409
|
+
}
|
|
1410
|
+
state.resolveCompletion();
|
|
1411
|
+
const failure = state.failure;
|
|
1412
|
+
const listeners = state.failureListeners;
|
|
1413
|
+
if (listeners !== undefined) {
|
|
1414
|
+
delete state.failureListeners;
|
|
1415
|
+
if (failure !== undefined) {
|
|
1416
|
+
for (const listener of listeners) listener(failure);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
private observeOwnedPosixProcessGroupGone(
|
|
1422
|
+
task: BgTask,
|
|
1423
|
+
state: PosixProcessGroupKillState,
|
|
1424
|
+
): boolean {
|
|
1425
|
+
if (state.settled) return state.failure === undefined;
|
|
1426
|
+
try {
|
|
1427
|
+
const exists = this.killProcess(-state.groupId, 0);
|
|
1428
|
+
state.lastProbeError = exists
|
|
1429
|
+
? undefined
|
|
1430
|
+
: new Error(`process-group probe for ${String(state.groupId)} returned false`);
|
|
1431
|
+
return false;
|
|
1432
|
+
} catch (error) {
|
|
1433
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
1434
|
+
this.finishPosixProcessGroupKill(task, state, true);
|
|
1435
|
+
return true;
|
|
1436
|
+
}
|
|
1437
|
+
state.lastProbeError =
|
|
1438
|
+
error instanceof Error ? error : new Error(BackgroundTaskRegistry.errorMessage(error));
|
|
1439
|
+
return false;
|
|
1440
|
+
}
|
|
1441
|
+
}
|
|
1442
|
+
|
|
1443
|
+
private recordPosixProcessGroupForceFailure(
|
|
1444
|
+
task: BgTask,
|
|
1445
|
+
state: PosixProcessGroupKillState,
|
|
1446
|
+
error: Error,
|
|
1447
|
+
): void {
|
|
1448
|
+
if (state.settled) return;
|
|
1449
|
+
state.failure = error;
|
|
1450
|
+
this.finishPosixProcessGroupKill(task, state, false);
|
|
1451
|
+
task.error = BackgroundTaskRegistry.appendTaskError(task.error, error.message);
|
|
1452
|
+
this.writeNotice(task, `\n[background task POSIX termination: ${error.message}]\n`);
|
|
1453
|
+
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1454
|
+
this.logger.error(
|
|
1455
|
+
`[background-tasks] failed to write POSIX process-group failure metadata for ${task.id}:`,
|
|
1456
|
+
metadataError,
|
|
1457
|
+
);
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
private schedulePosixProcessGroupVerification(
|
|
1462
|
+
task: BgTask,
|
|
1463
|
+
state: PosixProcessGroupKillState,
|
|
1464
|
+
): void {
|
|
1465
|
+
if (state.settled) return;
|
|
1466
|
+
const remainingMs = state.deadlineAt - Date.now();
|
|
1467
|
+
if (remainingMs <= 0) {
|
|
1468
|
+
const probeDetail =
|
|
1469
|
+
state.lastProbeError === undefined
|
|
1470
|
+
? ''
|
|
1471
|
+
: `; last group probe failed: ${state.lastProbeError.message}`;
|
|
1472
|
+
this.recordPosixProcessGroupForceFailure(
|
|
1473
|
+
task,
|
|
1474
|
+
state,
|
|
1475
|
+
new Error(
|
|
1476
|
+
`POSIX process group ${String(state.groupId)} remained present after SIGKILL${probeDetail}. Descendant processes may have leaked.`,
|
|
1477
|
+
),
|
|
1478
|
+
);
|
|
1479
|
+
return;
|
|
1480
|
+
}
|
|
1481
|
+
state.verificationTimer = setTimeout(
|
|
1482
|
+
() => {
|
|
1483
|
+
state.verificationTimer = undefined;
|
|
1484
|
+
if (this.observeOwnedPosixProcessGroupGone(task, state)) return;
|
|
1485
|
+
this.schedulePosixProcessGroupVerification(task, state);
|
|
1486
|
+
},
|
|
1487
|
+
Math.min(10, remainingMs),
|
|
1488
|
+
);
|
|
1489
|
+
}
|
|
1490
|
+
|
|
1491
|
+
private forceOwnedPosixProcessGroup(task: BgTask, state: PosixProcessGroupKillState): void {
|
|
1492
|
+
if (state.settled || state.forceAttempted) return;
|
|
1493
|
+
// Latch before either probe or signal: both are injected boundaries that can
|
|
1494
|
+
// reentrantly emit root close, and no continuation may launch a second KILL.
|
|
1495
|
+
state.forceAttempted = true;
|
|
1496
|
+
this.clearKillEscalationTimer(task);
|
|
1497
|
+
if (this.observeOwnedPosixProcessGroupGone(task, state)) return;
|
|
1498
|
+
|
|
1499
|
+
try {
|
|
1500
|
+
const forced = this.killProcess(-state.groupId, 'SIGKILL');
|
|
1501
|
+
if (!forced) {
|
|
1502
|
+
this.recordPosixProcessGroupForceFailure(
|
|
1503
|
+
task,
|
|
1504
|
+
state,
|
|
1505
|
+
new Error(
|
|
1506
|
+
`POSIX process-group SIGKILL returned false for task ${task.id} group ${String(state.groupId)}. Descendant processes may have leaked.`,
|
|
1507
|
+
),
|
|
1508
|
+
);
|
|
1509
|
+
return;
|
|
1510
|
+
}
|
|
1511
|
+
} catch (error) {
|
|
1512
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
1513
|
+
this.finishPosixProcessGroupKill(task, state, true);
|
|
1514
|
+
return;
|
|
1515
|
+
}
|
|
1516
|
+
this.recordPosixProcessGroupForceFailure(
|
|
1517
|
+
task,
|
|
1518
|
+
state,
|
|
1519
|
+
new Error(
|
|
1520
|
+
`POSIX process-group SIGKILL failed for task ${task.id} group ${String(state.groupId)}: ${BackgroundTaskRegistry.errorMessage(error)}. Descendant processes may have leaked.`,
|
|
1521
|
+
),
|
|
1522
|
+
);
|
|
1523
|
+
return;
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
if (this.observeOwnedPosixProcessGroupGone(task, state)) return;
|
|
1527
|
+
this.schedulePosixProcessGroupVerification(task, state);
|
|
1528
|
+
}
|
|
1529
|
+
|
|
1530
|
+
private requestPosixKill(task: BgTask, signal: NodeJS.Signals): void {
|
|
1531
|
+
const state = this.beginPosixProcessGroupKill(task, signal !== 'SIGKILL');
|
|
1532
|
+
if (signal === 'SIGKILL') {
|
|
1533
|
+
task.killSignalSent = true;
|
|
1534
|
+
this.forceOwnedPosixProcessGroup(task, state);
|
|
1535
|
+
return;
|
|
1536
|
+
}
|
|
1537
|
+
if (task.killSignalSent) return;
|
|
1538
|
+
// Publish de-duplication before the signal boundary for the same reason the
|
|
1539
|
+
// state/timer is published above: close may be emitted synchronously.
|
|
1540
|
+
task.killSignalSent = true;
|
|
1541
|
+
|
|
1542
|
+
const errors: string[] = [];
|
|
1543
|
+
let killed = false;
|
|
1544
|
+
try {
|
|
1545
|
+
killed = this.killProcess(-state.groupId, signal);
|
|
1546
|
+
if (!killed) errors.push(`process group ${signal} returned false`);
|
|
1547
|
+
} catch (error) {
|
|
1548
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
1549
|
+
this.finishPosixProcessGroupKill(task, state, true);
|
|
1550
|
+
} else {
|
|
1551
|
+
errors.push(`process group kill failed: ${BackgroundTaskRegistry.errorMessage(error)}`);
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
if (!killed) {
|
|
1556
|
+
try {
|
|
1557
|
+
killed = task.child?.kill(signal) === true;
|
|
1558
|
+
if (!killed) errors.push(`child ${signal} returned false`);
|
|
1559
|
+
} catch (error) {
|
|
1560
|
+
errors.push(`child kill failed: ${BackgroundTaskRegistry.errorMessage(error)}`);
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
if (!killed) {
|
|
1565
|
+
// If TERM reached neither target, waiting out grace has no benefit. Keep
|
|
1566
|
+
// the same owner but attempt its one force phase immediately.
|
|
1567
|
+
if (!state.settled) this.forceOwnedPosixProcessGroup(task, state);
|
|
1568
|
+
throw new Error(`Could not kill task ${task.id}: ${errors.join('; ')}`);
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
|
|
1572
|
+
private async awaitPosixProcessGroupBeforeTerminal(task: BgTask): Promise<Error | undefined> {
|
|
1573
|
+
if (this.platform === 'win32') return undefined;
|
|
1574
|
+
const state = this.posixProcessGroupKillStates.get(task);
|
|
1575
|
+
if (state === undefined) {
|
|
1576
|
+
// No tree stop won the race before direct-child finalization. Release
|
|
1577
|
+
// signal authority synchronously so a concurrent late stop waits for this
|
|
1578
|
+
// terminalization instead of targeting a potentially reused group id.
|
|
1579
|
+
task.posixProcessGroupSignalAuthorityReleased = true;
|
|
1580
|
+
delete task.ownedPosixProcessGroupId;
|
|
1581
|
+
return undefined;
|
|
1582
|
+
}
|
|
1583
|
+
this.observeOwnedPosixProcessGroupGone(task, state);
|
|
1584
|
+
await state.completion;
|
|
1585
|
+
return state.failure;
|
|
1586
|
+
}
|
|
1587
|
+
|
|
1588
|
+
private waitForEndOrPosixForceFailure(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
1589
|
+
const state = this.posixProcessGroupKillStates.get(task);
|
|
1590
|
+
if (state === undefined) return this.waitForEnd(task, timeoutMs);
|
|
1591
|
+
if (state.failure !== undefined) return Promise.reject(state.failure);
|
|
1592
|
+
if (task.status !== 'running') return Promise.resolve(true);
|
|
1593
|
+
return new Promise((resolve, reject) => {
|
|
1594
|
+
const cleanup = () => {
|
|
1595
|
+
clearTimeout(timeout);
|
|
1596
|
+
const waiterIndex = task.waiters.indexOf(done);
|
|
1597
|
+
if (waiterIndex >= 0) task.waiters.splice(waiterIndex, 1);
|
|
1598
|
+
const listeners = state.failureListeners;
|
|
1599
|
+
if (listeners !== undefined) {
|
|
1600
|
+
const listenerIndex = listeners.indexOf(failed);
|
|
1601
|
+
if (listenerIndex >= 0) listeners.splice(listenerIndex, 1);
|
|
1602
|
+
if (listeners.length === 0) delete state.failureListeners;
|
|
1603
|
+
}
|
|
1604
|
+
};
|
|
1605
|
+
const timeout = setTimeout(() => {
|
|
1606
|
+
cleanup();
|
|
1607
|
+
resolve(false);
|
|
1608
|
+
}, timeoutMs);
|
|
1609
|
+
const done = () => {
|
|
1610
|
+
cleanup();
|
|
1611
|
+
resolve(true);
|
|
1612
|
+
};
|
|
1613
|
+
const failed = (error: Error) => {
|
|
1614
|
+
cleanup();
|
|
1615
|
+
reject(error);
|
|
1616
|
+
};
|
|
1617
|
+
task.waiters.push(done);
|
|
1618
|
+
if (state.failureListeners === undefined) state.failureListeners = [];
|
|
1619
|
+
state.failureListeners.push(failed);
|
|
1620
|
+
});
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
private getWindowsKillState(task: BgTask): WindowsKillState {
|
|
1624
|
+
let state = this.windowsKillStates.get(task);
|
|
1625
|
+
if (state === undefined) {
|
|
1626
|
+
state = {};
|
|
1627
|
+
this.windowsKillStates.set(task, state);
|
|
1628
|
+
}
|
|
1629
|
+
return state;
|
|
1630
|
+
}
|
|
1631
|
+
|
|
1632
|
+
private static errorMessage(error: unknown): string {
|
|
1633
|
+
return error instanceof Error ? error.message : String(error);
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
private static appendTaskError(existing: string | undefined, next: string): string {
|
|
1637
|
+
if (existing === undefined || existing.length === 0) return next;
|
|
1638
|
+
if (existing.includes(next)) return existing;
|
|
1639
|
+
return `${existing}; ${next}`;
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
private static describeTaskkillOutcome(outcome: TaskkillOutcome): string {
|
|
1643
|
+
const exitCode = outcome.exitCode === null ? 'null' : String(outcome.exitCode);
|
|
1644
|
+
const signal = outcome.signal === null ? 'null' : outcome.signal;
|
|
1645
|
+
const stdout = outcome.stdout.length > 0 ? ` stdout=${JSON.stringify(outcome.stdout)}` : '';
|
|
1646
|
+
const stderr = outcome.stderr.length > 0 ? ` stderr=${JSON.stringify(outcome.stderr)}` : '';
|
|
1647
|
+
const stdoutTruncated = outcome.stdoutTruncated ? ' stdout_truncated=true' : '';
|
|
1648
|
+
const stderrTruncated = outcome.stderrTruncated ? ' stderr_truncated=true' : '';
|
|
1649
|
+
return `exit=${exitCode} signal=${signal}${stdout}${stderr}${stdoutTruncated}${stderrTruncated}`;
|
|
1650
|
+
}
|
|
1651
|
+
|
|
1652
|
+
private isWindowsTaskkillTerminalRace(task: BgTask): boolean {
|
|
1653
|
+
return task.status !== 'running' || task.finalized === true;
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
private clearKillEscalationTimer(task: BgTask): void {
|
|
1657
|
+
if (task.killEscalationTimer !== undefined) {
|
|
1658
|
+
clearTimeout(task.killEscalationTimer);
|
|
1659
|
+
task.killEscalationTimer = undefined;
|
|
1660
|
+
}
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
private recordWindowsTaskkillNotice(task: BgTask, message: string): void {
|
|
1664
|
+
this.writeNotice(task, `\n[background task Windows termination: ${message}]\n`);
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
private recordWindowsSoftFailure(task: BgTask, pid: number, detail: string): void {
|
|
1668
|
+
const message =
|
|
1669
|
+
`Windows taskkill /T logical termination request failed for task ${task.id} pid ${String(pid)}: ` +
|
|
1670
|
+
`${detail}; force escalation remains scheduled`;
|
|
1671
|
+
task.error = BackgroundTaskRegistry.appendTaskError(task.error, message);
|
|
1672
|
+
this.recordWindowsTaskkillNotice(task, message);
|
|
1673
|
+
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1674
|
+
this.logger.error(
|
|
1675
|
+
`[background-tasks] failed to write Windows taskkill soft-failure metadata for ${task.id}:`,
|
|
1676
|
+
metadataError,
|
|
1677
|
+
);
|
|
1678
|
+
});
|
|
1679
|
+
}
|
|
1680
|
+
|
|
1681
|
+
private makeWindowsForceFailure(task: BgTask, pid: number, detail: string): Error {
|
|
1682
|
+
return new Error(
|
|
1683
|
+
`Windows taskkill /T /F force termination failed for task ${task.id} pid ${String(pid)}: ${detail}. Descendant processes may have leaked.`,
|
|
1684
|
+
);
|
|
1685
|
+
}
|
|
1686
|
+
|
|
1687
|
+
private recordWindowsForceFailure(task: BgTask, error: Error): void {
|
|
1688
|
+
const state = this.getWindowsKillState(task);
|
|
1689
|
+
state.forceFailure = error;
|
|
1690
|
+
task.error = BackgroundTaskRegistry.appendTaskError(task.error, error.message);
|
|
1691
|
+
this.recordWindowsTaskkillNotice(task, error.message);
|
|
1692
|
+
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1693
|
+
this.logger.error(
|
|
1694
|
+
`[background-tasks] failed to write Windows taskkill force-failure metadata for ${task.id}:`,
|
|
1695
|
+
metadataError,
|
|
1696
|
+
);
|
|
1697
|
+
});
|
|
1698
|
+
const listeners = state.forceFailureListeners;
|
|
1699
|
+
if (listeners !== undefined) {
|
|
1700
|
+
delete state.forceFailureListeners;
|
|
1701
|
+
for (const listener of listeners) listener(error);
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
private evaluateWindowsTaskkillOutcome(
|
|
1706
|
+
task: BgTask,
|
|
1707
|
+
pid: number,
|
|
1708
|
+
phase: WindowsKillPhase,
|
|
1709
|
+
outcome: TaskkillOutcome,
|
|
1710
|
+
): Error | undefined {
|
|
1711
|
+
if (outcome.exitCode === 0) return undefined;
|
|
1712
|
+
const detail = BackgroundTaskRegistry.describeTaskkillOutcome(outcome);
|
|
1713
|
+
if (outcome.exitCode === 128) {
|
|
1714
|
+
this.recordWindowsTaskkillNotice(
|
|
1715
|
+
task,
|
|
1716
|
+
`taskkill ${phase} reported process not found for pid ${String(pid)} (${detail}); treating as an already-exited race`,
|
|
1717
|
+
);
|
|
1718
|
+
return undefined;
|
|
1719
|
+
}
|
|
1720
|
+
if (this.isWindowsTaskkillTerminalRace(task)) {
|
|
1721
|
+
this.recordWindowsTaskkillNotice(
|
|
1722
|
+
task,
|
|
1723
|
+
`taskkill ${phase} finished after the task became terminal for pid ${String(pid)} (${detail}); treating as a terminal race`,
|
|
1724
|
+
);
|
|
1725
|
+
return undefined;
|
|
1726
|
+
}
|
|
1727
|
+
if (phase === 'terminate') {
|
|
1728
|
+
this.recordWindowsSoftFailure(task, pid, detail);
|
|
1729
|
+
return undefined;
|
|
1730
|
+
}
|
|
1731
|
+
return this.makeWindowsForceFailure(task, pid, detail);
|
|
1732
|
+
}
|
|
1733
|
+
|
|
1734
|
+
private handleWindowsSoftException(
|
|
1735
|
+
task: BgTask,
|
|
1736
|
+
pid: number,
|
|
1737
|
+
error: unknown,
|
|
1738
|
+
state: WindowsKillState,
|
|
1739
|
+
): void {
|
|
1740
|
+
const message = BackgroundTaskRegistry.errorMessage(error);
|
|
1741
|
+
if (state.forcePromise !== undefined || this.isWindowsTaskkillTerminalRace(task)) return;
|
|
1742
|
+
this.recordWindowsSoftFailure(task, pid, message);
|
|
1743
|
+
}
|
|
1744
|
+
|
|
1745
|
+
private startWindowsSoftKill(task: BgTask, pid: number): Promise<void> {
|
|
1746
|
+
const state = this.getWindowsKillState(task);
|
|
1747
|
+
if (state.softPromise !== undefined) return state.softPromise;
|
|
1748
|
+
const controller = new AbortController();
|
|
1749
|
+
state.softController = controller;
|
|
1750
|
+
|
|
1751
|
+
let launched: Promise<TaskkillOutcome>;
|
|
1752
|
+
try {
|
|
1753
|
+
launched = this.killTree(pid, 'terminate', controller.signal);
|
|
1754
|
+
} catch (error) {
|
|
1755
|
+
delete state.softController;
|
|
1756
|
+
throw new Error(
|
|
1757
|
+
`Could not kill task ${task.id}: Windows taskkill /T failed to start: ${BackgroundTaskRegistry.errorMessage(error)}`,
|
|
1758
|
+
);
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
const promise = launched
|
|
1762
|
+
.then((outcome) => {
|
|
1763
|
+
if (state.forcePromise !== undefined || this.isWindowsTaskkillTerminalRace(task)) return;
|
|
1764
|
+
const failure = this.evaluateWindowsTaskkillOutcome(task, pid, 'terminate', outcome);
|
|
1765
|
+
if (failure !== undefined) throw failure;
|
|
1766
|
+
})
|
|
1767
|
+
.catch((error: unknown) => {
|
|
1768
|
+
this.handleWindowsSoftException(task, pid, error, state);
|
|
1769
|
+
})
|
|
1770
|
+
.finally(() => {
|
|
1771
|
+
if (state.softController === controller) delete state.softController;
|
|
1772
|
+
});
|
|
1773
|
+
state.softPromise = promise;
|
|
1774
|
+
return promise;
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
private startWindowsForceKill(task: BgTask, pid: number): Promise<void> {
|
|
1778
|
+
const state = this.getWindowsKillState(task);
|
|
1779
|
+
if (state.forcePromise !== undefined) return state.forcePromise;
|
|
1780
|
+
|
|
1781
|
+
let resolveForce: (() => void) | undefined;
|
|
1782
|
+
let rejectForce: ((error: unknown) => void) | undefined;
|
|
1783
|
+
const forcePromise = new Promise<void>((resolve, reject) => {
|
|
1784
|
+
resolveForce = resolve;
|
|
1785
|
+
rejectForce = reject;
|
|
1786
|
+
});
|
|
1787
|
+
if (resolveForce === undefined || rejectForce === undefined) {
|
|
1788
|
+
throw new Error('Windows force termination promise could not be initialized');
|
|
1789
|
+
}
|
|
1790
|
+
const resolveForceReady = resolveForce;
|
|
1791
|
+
const rejectForceReady = rejectForce;
|
|
1792
|
+
state.forcePromise = forcePromise;
|
|
1793
|
+
void forcePromise.catch((error: unknown) => {
|
|
1794
|
+
this.logger.error(
|
|
1795
|
+
`[background-tasks] Windows force tree termination failed for ${task.id}:`,
|
|
1796
|
+
error,
|
|
1797
|
+
);
|
|
1798
|
+
});
|
|
1799
|
+
|
|
1800
|
+
this.clearKillEscalationTimer(task);
|
|
1801
|
+
if (state.softController !== undefined && !state.softController.signal.aborted) {
|
|
1802
|
+
state.softController.abort();
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
let launched: Promise<TaskkillOutcome>;
|
|
1806
|
+
try {
|
|
1807
|
+
launched = this.killTree(pid, 'force');
|
|
1808
|
+
} catch (error) {
|
|
1809
|
+
const failure = this.makeWindowsForceFailure(
|
|
1810
|
+
task,
|
|
1811
|
+
pid,
|
|
1812
|
+
`helper failed to start: ${BackgroundTaskRegistry.errorMessage(error)}`,
|
|
1813
|
+
);
|
|
1814
|
+
delete state.forcePromise;
|
|
1815
|
+
this.recordWindowsForceFailure(task, failure);
|
|
1816
|
+
rejectForceReady(failure);
|
|
1817
|
+
throw failure;
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
launched.then(
|
|
1821
|
+
(outcome) => {
|
|
1822
|
+
const failure = this.evaluateWindowsTaskkillOutcome(task, pid, 'force', outcome);
|
|
1823
|
+
if (failure !== undefined) {
|
|
1824
|
+
this.recordWindowsForceFailure(task, failure);
|
|
1825
|
+
rejectForceReady(failure);
|
|
1826
|
+
return;
|
|
1827
|
+
}
|
|
1828
|
+
resolveForceReady();
|
|
1829
|
+
},
|
|
1830
|
+
(error: unknown) => {
|
|
1831
|
+
if (this.isWindowsTaskkillTerminalRace(task)) {
|
|
1832
|
+
this.recordWindowsTaskkillNotice(
|
|
1833
|
+
task,
|
|
1834
|
+
`taskkill force rejected after the task became terminal for pid ${String(pid)} (${BackgroundTaskRegistry.errorMessage(error)}); treating as a terminal race`,
|
|
1835
|
+
);
|
|
1836
|
+
resolveForceReady();
|
|
1837
|
+
return;
|
|
1838
|
+
}
|
|
1839
|
+
const failure = this.makeWindowsForceFailure(
|
|
1840
|
+
task,
|
|
1841
|
+
pid,
|
|
1842
|
+
BackgroundTaskRegistry.errorMessage(error),
|
|
1843
|
+
);
|
|
1844
|
+
this.recordWindowsForceFailure(task, failure);
|
|
1845
|
+
rejectForceReady(failure);
|
|
1846
|
+
},
|
|
1847
|
+
);
|
|
1848
|
+
|
|
1849
|
+
return forcePromise;
|
|
1850
|
+
}
|
|
1851
|
+
|
|
1852
|
+
private requestWindowsKill(task: BgTask, pid: number, signal: NodeJS.Signals): void {
|
|
1853
|
+
if (signal === 'SIGKILL') {
|
|
1854
|
+
this.startWindowsForceKill(task, pid);
|
|
1855
|
+
task.killSignalSent = true;
|
|
1856
|
+
return;
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
this.startWindowsSoftKill(task, pid);
|
|
1860
|
+
task.killSignalSent = true;
|
|
1861
|
+
if (task.killEscalationTimer !== undefined) return;
|
|
1862
|
+
task.killEscalationTimer = setTimeout(() => {
|
|
1863
|
+
task.killEscalationTimer = undefined;
|
|
1864
|
+
if (task.status !== 'running') return;
|
|
1865
|
+
try {
|
|
1866
|
+
this.requestKill(task, 'SIGKILL');
|
|
1867
|
+
} catch (error) {
|
|
1868
|
+
task.error = BackgroundTaskRegistry.appendTaskError(
|
|
1869
|
+
task.error,
|
|
1870
|
+
`SIGKILL failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1871
|
+
);
|
|
1872
|
+
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1873
|
+
this.logger.error(
|
|
1874
|
+
`[background-tasks] failed to write metadata for ${task.id}:`,
|
|
1875
|
+
metadataError,
|
|
1876
|
+
);
|
|
1877
|
+
});
|
|
1878
|
+
}
|
|
1879
|
+
}, this.killGraceMs).unref();
|
|
1880
|
+
}
|
|
1881
|
+
|
|
1882
|
+
private requestKill(task: BgTask, signal: NodeJS.Signals = 'SIGTERM'): void {
|
|
1883
|
+
if (task.status !== 'running') {
|
|
1884
|
+
throw new Error(`Task ${task.id} is ${task.status}, not running`);
|
|
1885
|
+
}
|
|
1886
|
+
if (!task.child) {
|
|
1887
|
+
throw new Error(`Task ${task.id} has no child process handle`);
|
|
1888
|
+
}
|
|
1889
|
+
if (task.killSignalSent && signal === 'SIGTERM') return;
|
|
1890
|
+
|
|
1891
|
+
if (this.platform === 'win32') {
|
|
1892
|
+
if (!task.pid) throw new Error(`Task ${task.id} has no process id`);
|
|
1893
|
+
this.requestWindowsKill(task, task.pid, signal);
|
|
1894
|
+
return;
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1897
|
+
this.requestPosixKill(task, signal);
|
|
1898
|
+
}
|
|
1899
|
+
|
|
1900
|
+
private waitForEnd(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
1901
|
+
if (task.status !== 'running') return Promise.resolve(true);
|
|
1902
|
+
return new Promise((resolve) => {
|
|
1903
|
+
const timeout = setTimeout(() => {
|
|
1904
|
+
const idx = task.waiters.indexOf(done);
|
|
1905
|
+
if (idx >= 0) task.waiters.splice(idx, 1);
|
|
1906
|
+
resolve(false);
|
|
1907
|
+
}, timeoutMs);
|
|
1908
|
+
const done = () => {
|
|
1909
|
+
clearTimeout(timeout);
|
|
1910
|
+
resolve(true);
|
|
1911
|
+
};
|
|
1912
|
+
task.waiters.push(done);
|
|
1913
|
+
});
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
private waitForEndOrWindowsForceFailure(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
1917
|
+
const state = this.getWindowsKillState(task);
|
|
1918
|
+
if (state.forceFailure !== undefined) return Promise.reject(state.forceFailure);
|
|
1919
|
+
if (task.status !== 'running') return Promise.resolve(true);
|
|
1920
|
+
return new Promise((resolve, reject) => {
|
|
1921
|
+
const cleanup = () => {
|
|
1922
|
+
clearTimeout(timeout);
|
|
1923
|
+
const waiterIndex = task.waiters.indexOf(done);
|
|
1924
|
+
if (waiterIndex >= 0) task.waiters.splice(waiterIndex, 1);
|
|
1925
|
+
const listeners = state.forceFailureListeners;
|
|
1926
|
+
if (listeners !== undefined) {
|
|
1927
|
+
const listenerIndex = listeners.indexOf(failed);
|
|
1928
|
+
if (listenerIndex >= 0) listeners.splice(listenerIndex, 1);
|
|
1929
|
+
if (listeners.length === 0) delete state.forceFailureListeners;
|
|
1930
|
+
}
|
|
1931
|
+
};
|
|
1932
|
+
const timeout = setTimeout(() => {
|
|
1933
|
+
cleanup();
|
|
1934
|
+
resolve(false);
|
|
1935
|
+
}, timeoutMs);
|
|
1936
|
+
const done = () => {
|
|
1937
|
+
cleanup();
|
|
1938
|
+
resolve(true);
|
|
1939
|
+
};
|
|
1940
|
+
const failed = (error: Error) => {
|
|
1941
|
+
cleanup();
|
|
1942
|
+
reject(error);
|
|
1943
|
+
};
|
|
1944
|
+
task.waiters.push(done);
|
|
1945
|
+
if (state.forceFailureListeners === undefined) state.forceFailureListeners = [];
|
|
1946
|
+
state.forceFailureListeners.push(failed);
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
1949
|
+
|
|
1950
|
+
private async awaitWindowsForceBeforeTerminal(task: BgTask): Promise<Error | undefined> {
|
|
1951
|
+
const state = this.windowsKillStates.get(task);
|
|
1952
|
+
if (state === undefined) return undefined;
|
|
1953
|
+
const forcePromise = state.forcePromise;
|
|
1954
|
+
if (forcePromise === undefined) return state.forceFailure;
|
|
1955
|
+
try {
|
|
1956
|
+
await forcePromise;
|
|
1957
|
+
} catch (error) {
|
|
1958
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1959
|
+
}
|
|
1960
|
+
return state.forceFailure;
|
|
1961
|
+
}
|
|
1962
|
+
|
|
1963
|
+
private async deliverReloadTerminal(
|
|
1964
|
+
execution: ReloadableShellExecutionV1,
|
|
1965
|
+
lease: ReloadShellActivationLeaseV1,
|
|
1966
|
+
): Promise<void> {
|
|
1967
|
+
const task = execution.task;
|
|
1968
|
+
if (task.reloadHostDeliveryInFlight || task.reloadHostDeliverySettled) {
|
|
1969
|
+
this.maybeReleaseReloadExecution(task);
|
|
1970
|
+
return;
|
|
1971
|
+
}
|
|
1972
|
+
task.reloadHostDeliveryInFlight = true;
|
|
1973
|
+
try {
|
|
1974
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
1975
|
+
this.publishTerminal(task);
|
|
1976
|
+
const deliveryGate = await this.waitForTerminalPublicationGate(task);
|
|
1977
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
1978
|
+
if (deliveryGate.kind === 'rejected') {
|
|
1979
|
+
this.logger.error(
|
|
1980
|
+
`[background-tasks] completion delivery gate failed for ${task.id}: ${this.terminalPublicationError(deliveryGate.error)}`,
|
|
1981
|
+
);
|
|
1982
|
+
} else if (
|
|
1983
|
+
task.notifyOnCompletion &&
|
|
1984
|
+
!task.notified &&
|
|
1985
|
+
!this.shuttingDown &&
|
|
1986
|
+
execution.notificationState === 'pending'
|
|
1987
|
+
) {
|
|
1988
|
+
const token = execution.beginNotification(lease);
|
|
1989
|
+
if (token !== undefined) {
|
|
1990
|
+
try {
|
|
1991
|
+
this.notifyCompletion(task);
|
|
1992
|
+
execution.finishNotification(token, task.notified);
|
|
1993
|
+
} catch (error) {
|
|
1994
|
+
execution.finishNotification(token, false);
|
|
1995
|
+
this.logger.error(`[background-tasks] notification failed for ${task.id}:`, error);
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
}
|
|
1999
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
2000
|
+
task.reloadHostNotificationSettled = true;
|
|
2001
|
+
task.reloadHostDeliverySettled = true;
|
|
2002
|
+
try {
|
|
2003
|
+
await this.writeMetadata(task);
|
|
2004
|
+
} catch (error) {
|
|
2005
|
+
this.logger.error(
|
|
2006
|
+
`[background-tasks] failed to update survivor notification metadata for ${task.id}:`,
|
|
2007
|
+
error,
|
|
2008
|
+
);
|
|
2009
|
+
}
|
|
2010
|
+
} finally {
|
|
2011
|
+
task.reloadHostDeliveryInFlight = false;
|
|
2012
|
+
this.maybeReleaseReloadExecution(task);
|
|
2013
|
+
}
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
private maybeReleaseReloadExecution(task: BgTask): void {
|
|
2017
|
+
const execution = task.reloadExecution;
|
|
2018
|
+
const lease = this.reloadShellLease;
|
|
2019
|
+
if (
|
|
2020
|
+
execution === undefined ||
|
|
2021
|
+
execution.phase !== 'terminal' ||
|
|
2022
|
+
task.reloadHostNotificationSettled !== true ||
|
|
2023
|
+
task.terminalPublicationState === 'pending' ||
|
|
2024
|
+
!this.ownsReloadExecution(execution, lease) ||
|
|
2025
|
+
this.reloadShellOwner === undefined ||
|
|
2026
|
+
lease === undefined
|
|
2027
|
+
) {
|
|
2028
|
+
return;
|
|
2029
|
+
}
|
|
2030
|
+
try {
|
|
2031
|
+
this.reloadShellOwner.releaseExecution(lease, execution);
|
|
2032
|
+
} catch (error) {
|
|
2033
|
+
if (
|
|
2034
|
+
typeof error !== 'object' ||
|
|
2035
|
+
error === null ||
|
|
2036
|
+
Reflect.get(error, 'code') !== 'pi_bg_reload_owner_stale_claim'
|
|
2037
|
+
) {
|
|
2038
|
+
this.logger.error(
|
|
2039
|
+
`[background-tasks] failed to release reload shell execution ${task.id}:`,
|
|
2040
|
+
error,
|
|
2041
|
+
);
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
private terminalPublicationAbandonSignal(task: BgTask): TerminalPublicationAbandonSignal {
|
|
2047
|
+
const existing = this.terminalPublicationAbandonSignals.get(task);
|
|
2048
|
+
if (existing !== undefined) return existing;
|
|
2049
|
+
let resolveSignal: (reason: TerminalPublicationAbandonReason) => void = () => {};
|
|
2050
|
+
const promise = new Promise<TerminalPublicationAbandonReason>((resolve) => {
|
|
2051
|
+
resolveSignal = resolve;
|
|
2052
|
+
});
|
|
2053
|
+
const signal = { promise, resolve: resolveSignal };
|
|
2054
|
+
this.terminalPublicationAbandonSignals.set(task, signal);
|
|
2055
|
+
return signal;
|
|
2056
|
+
}
|
|
2057
|
+
|
|
2058
|
+
private publishTerminal(task: BgTask): void {
|
|
2059
|
+
if (task.reloadExecution !== undefined && this.tasks.get(task.id) !== task) return;
|
|
2060
|
+
if (task.terminalPublicationState !== 'pending' || task.terminalPublishInFlight) return;
|
|
2061
|
+
if (this.terminalPublicationClosed) {
|
|
2062
|
+
this.abandonTerminalPublication(
|
|
2063
|
+
task,
|
|
2064
|
+
this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
2065
|
+
);
|
|
2066
|
+
return;
|
|
2067
|
+
}
|
|
2068
|
+
task.terminalPublishInFlight = true;
|
|
2069
|
+
if (task.terminalPublicationGate === undefined) {
|
|
2070
|
+
this.tryPublishTerminalNow(task);
|
|
2071
|
+
return;
|
|
2072
|
+
}
|
|
2073
|
+
void this.publishTerminalWhenReady(task);
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
private async publishTerminalWhenReady(task: BgTask): Promise<void> {
|
|
2077
|
+
const outcome = await this.waitForTerminalPublicationGate(task);
|
|
2078
|
+
if (task.reloadExecution !== undefined && this.tasks.get(task.id) !== task) {
|
|
2079
|
+
task.terminalPublishInFlight = false;
|
|
2080
|
+
return;
|
|
2081
|
+
}
|
|
2082
|
+
if (outcome.kind === 'closed') {
|
|
2083
|
+
this.abandonTerminalPublication(task, outcome.reason);
|
|
2084
|
+
return;
|
|
2085
|
+
}
|
|
2086
|
+
if (outcome.kind === 'rejected') {
|
|
2087
|
+
this.abandonTerminalPublication(task, 'gate_rejected', outcome.error);
|
|
2088
|
+
return;
|
|
2089
|
+
}
|
|
2090
|
+
// The gate and registry closure can settle in the same microtask turn.
|
|
2091
|
+
// Re-check after the await so a late gate cannot publish into a disposed
|
|
2092
|
+
// activation or revive a task already abandoned by shutdown.
|
|
2093
|
+
if (this.terminalPublicationClosed || task.terminalPublicationState !== 'pending') {
|
|
2094
|
+
this.abandonTerminalPublication(
|
|
2095
|
+
task,
|
|
2096
|
+
this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
2097
|
+
);
|
|
2098
|
+
return;
|
|
2099
|
+
}
|
|
2100
|
+
this.tryPublishTerminalNow(task);
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
private async waitForTerminalPublicationGate(
|
|
2104
|
+
task: BgTask,
|
|
2105
|
+
): Promise<TerminalPublicationGateOutcome> {
|
|
2106
|
+
if (this.terminalPublicationClosed) {
|
|
2107
|
+
return {
|
|
2108
|
+
kind: 'closed',
|
|
2109
|
+
reason: this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
2110
|
+
};
|
|
2111
|
+
}
|
|
2112
|
+
if (task.terminalPublicationState === 'abandoned') {
|
|
2113
|
+
if (task.terminalPublicationAbandonReason === 'gate_rejected') {
|
|
2114
|
+
return { kind: 'rejected', error: new Error('terminal publication gate rejected') };
|
|
2115
|
+
}
|
|
2116
|
+
return {
|
|
2117
|
+
kind: 'closed',
|
|
2118
|
+
reason: task.terminalPublicationAbandonReason ?? 'registry_shutdown',
|
|
2119
|
+
};
|
|
2120
|
+
}
|
|
2121
|
+
const gate = task.terminalPublicationGate;
|
|
2122
|
+
if (gate === undefined) return { kind: 'released' };
|
|
2123
|
+
|
|
2124
|
+
const gateOutcome: Promise<TerminalPublicationGateOutcome> = gate.then(
|
|
2125
|
+
() => ({ kind: 'released' }),
|
|
2126
|
+
(error: unknown) => ({ kind: 'rejected', error }),
|
|
2127
|
+
);
|
|
2128
|
+
const closureOutcome: Promise<TerminalPublicationGateOutcome> =
|
|
2129
|
+
this.terminalPublicationClosedSignal.then((reason) => ({ kind: 'closed', reason }));
|
|
2130
|
+
const abandonmentOutcome: Promise<TerminalPublicationGateOutcome> =
|
|
2131
|
+
this.terminalPublicationAbandonSignal(task).promise.then((reason) => ({
|
|
2132
|
+
kind: 'closed',
|
|
2133
|
+
reason,
|
|
2134
|
+
}));
|
|
2135
|
+
const outcome = await Promise.race([gateOutcome, closureOutcome, abandonmentOutcome]);
|
|
2136
|
+
|
|
2137
|
+
// Closure wins if it happened before this continuation resumed, regardless
|
|
2138
|
+
// of which promise queued its reaction first.
|
|
2139
|
+
if (this.terminalPublicationClosed) {
|
|
2140
|
+
return {
|
|
2141
|
+
kind: 'closed',
|
|
2142
|
+
reason: this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
2143
|
+
};
|
|
2144
|
+
}
|
|
2145
|
+
return outcome;
|
|
2146
|
+
}
|
|
2147
|
+
|
|
2148
|
+
private tryPublishTerminalNow(task: BgTask): void {
|
|
2149
|
+
if (task.reloadExecution !== undefined && this.tasks.get(task.id) !== task) {
|
|
2150
|
+
task.terminalPublishInFlight = false;
|
|
2151
|
+
return;
|
|
2152
|
+
}
|
|
2153
|
+
if (task.terminalPublicationState !== 'pending') {
|
|
2154
|
+
task.terminalPublishInFlight = false;
|
|
2155
|
+
return;
|
|
2156
|
+
}
|
|
2157
|
+
if (this.terminalPublicationClosed) {
|
|
2158
|
+
task.terminalPublishInFlight = false;
|
|
2159
|
+
this.abandonTerminalPublication(
|
|
2160
|
+
task,
|
|
2161
|
+
this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
2162
|
+
);
|
|
2163
|
+
this.pruneOldTasks();
|
|
2164
|
+
return;
|
|
2165
|
+
}
|
|
2166
|
+
|
|
2167
|
+
task.terminalPublishAttempts += 1;
|
|
2168
|
+
task.terminalEmitInFlight = true;
|
|
2169
|
+
let emitFailed = false;
|
|
2170
|
+
let emitError: unknown;
|
|
2171
|
+
try {
|
|
2172
|
+
this.publishTerminalSnapshot(snapshot(task));
|
|
2173
|
+
} catch (error) {
|
|
2174
|
+
emitFailed = true;
|
|
2175
|
+
emitError = error;
|
|
2176
|
+
}
|
|
2177
|
+
task.terminalEmitInFlight = false;
|
|
2178
|
+
task.terminalPublishInFlight = false;
|
|
2179
|
+
|
|
2180
|
+
if (emitFailed) {
|
|
2181
|
+
const execution = task.reloadExecution;
|
|
2182
|
+
if (execution !== undefined && !this.ownsReloadExecution(execution, this.reloadShellLease)) {
|
|
2183
|
+
// The emitter synchronously detached this task into a reload handoff
|
|
2184
|
+
// before throwing. Attempt 1 is consumed, but only the fresh owner may
|
|
2185
|
+
// retry or decide abandonment on the shared publication ledger.
|
|
2186
|
+
return;
|
|
2187
|
+
}
|
|
2188
|
+
this.handleTerminalPublishFailure(task, emitError);
|
|
2189
|
+
} else {
|
|
2190
|
+
this.markTerminalPublicationDelivered(task);
|
|
2191
|
+
}
|
|
2192
|
+
this.pruneOldTasks();
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
private markTerminalPublicationDelivered(task: BgTask): void {
|
|
2196
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
2197
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
2198
|
+
task.terminalPublishRetryHandle = undefined;
|
|
2199
|
+
}
|
|
2200
|
+
task.terminalPublicationGate = undefined;
|
|
2201
|
+
task.terminalEmitInFlight = false;
|
|
2202
|
+
task.terminalPublishInFlight = false;
|
|
2203
|
+
task.terminalPublicationState = 'delivered';
|
|
2204
|
+
delete task.terminalPublicationAbandonReason;
|
|
2205
|
+
task.terminalPublished = true;
|
|
2206
|
+
this.maybeReleaseReloadExecution(task);
|
|
2207
|
+
}
|
|
2208
|
+
|
|
2209
|
+
private abandonTerminalPublication(
|
|
2210
|
+
task: BgTask,
|
|
2211
|
+
reason: TerminalPublicationAbandonReason,
|
|
2212
|
+
error?: unknown,
|
|
2213
|
+
log = true,
|
|
2214
|
+
): void {
|
|
2215
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
2216
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
2217
|
+
task.terminalPublishRetryHandle = undefined;
|
|
2218
|
+
}
|
|
2219
|
+
task.terminalPublicationGate = undefined;
|
|
2220
|
+
if (task.terminalEmitInFlight !== true) task.terminalPublishInFlight = false;
|
|
2221
|
+
if (task.terminalPublicationState === 'delivered') return;
|
|
2222
|
+
if (task.terminalPublicationState === 'abandoned') return;
|
|
2223
|
+
|
|
2224
|
+
task.terminalPublicationState = 'abandoned';
|
|
2225
|
+
task.terminalPublicationAbandonReason = reason;
|
|
2226
|
+
task.terminalPublished = false;
|
|
2227
|
+
this.terminalPublicationAbandonSignal(task).resolve(reason);
|
|
2228
|
+
if (log) {
|
|
2229
|
+
const detail = error === undefined ? '' : `: ${this.terminalPublicationError(error)}`;
|
|
2230
|
+
this.logger.error(
|
|
2231
|
+
`[background-tasks] terminal publication abandoned for ${task.id} (${reason}) after ${String(task.terminalPublishAttempts)}/${String(TERMINAL_PUBLICATION_MAX_ATTEMPTS)} emit attempts${detail}`,
|
|
2232
|
+
);
|
|
2233
|
+
}
|
|
2234
|
+
this.maybeReleaseReloadExecution(task);
|
|
2235
|
+
}
|
|
2236
|
+
|
|
2237
|
+
private terminalPublicationError(error: unknown): string {
|
|
2238
|
+
const compact = BackgroundTaskRegistry.errorMessage(error).replace(/\s+/gu, ' ').trim();
|
|
2239
|
+
if (compact.length <= TERMINAL_PUBLICATION_DIAGNOSTIC_CHARS) return compact;
|
|
2240
|
+
return `${compact.slice(0, TERMINAL_PUBLICATION_DIAGNOSTIC_CHARS - 1)}…`;
|
|
2241
|
+
}
|
|
2242
|
+
|
|
2243
|
+
private handleTerminalPublishFailure(task: BgTask, error: unknown): void {
|
|
2244
|
+
task.terminalPublishInFlight = false;
|
|
2245
|
+
if (task.terminalPublicationState !== 'pending') return;
|
|
2246
|
+
if (error instanceof BackgroundTaskExtensionServiceClosedError) {
|
|
2247
|
+
this.closeTerminalPublication('publisher_closed');
|
|
2248
|
+
return;
|
|
2249
|
+
}
|
|
2250
|
+
if (this.terminalPublicationClosed) {
|
|
2251
|
+
this.abandonTerminalPublication(
|
|
2252
|
+
task,
|
|
2253
|
+
this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
2254
|
+
error,
|
|
2255
|
+
);
|
|
2256
|
+
this.pruneOldTasks();
|
|
2257
|
+
return;
|
|
2258
|
+
}
|
|
2259
|
+
if (task.terminalPublishAttempts >= TERMINAL_PUBLICATION_MAX_ATTEMPTS) {
|
|
2260
|
+
this.abandonTerminalPublication(task, 'retry_exhausted', error);
|
|
2261
|
+
this.pruneOldTasks();
|
|
2262
|
+
return;
|
|
2263
|
+
}
|
|
2264
|
+
|
|
2265
|
+
this.logger.error(
|
|
2266
|
+
`[background-tasks] terminal publication failed for ${task.id} (attempt ${String(task.terminalPublishAttempts)}/${String(TERMINAL_PUBLICATION_MAX_ATTEMPTS)}; retrying): ${this.terminalPublicationError(error)}`,
|
|
2267
|
+
);
|
|
2268
|
+
if (task.terminalPublishRetryHandle !== undefined) return;
|
|
2269
|
+
task.terminalPublishRetryHandle = setTimeout(() => {
|
|
2270
|
+
task.terminalPublishRetryHandle = undefined;
|
|
2271
|
+
if (
|
|
2272
|
+
this.terminalPublicationClosed ||
|
|
2273
|
+
task.terminalPublicationState !== 'pending' ||
|
|
2274
|
+
(task.reloadExecution !== undefined && this.tasks.get(task.id) !== task)
|
|
2275
|
+
)
|
|
2276
|
+
return;
|
|
2277
|
+
this.publishTerminal(task);
|
|
2278
|
+
}, TERMINAL_PUBLICATION_RETRY_MS);
|
|
2279
|
+
task.terminalPublishRetryHandle.unref();
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
private notifyCompletion(task: BgTask): void {
|
|
2283
|
+
if (!task.notifyOnCompletion || task.notified || this.shuttingDown) return;
|
|
2284
|
+
task.notified = true;
|
|
2285
|
+
const exit =
|
|
2286
|
+
task.exitCode === undefined ? '' : `\n <exit-code>${String(task.exitCode)}</exit-code>`;
|
|
2287
|
+
const error = task.error ? `\n <error>${escapeXml(task.error)}</error>` : '';
|
|
2288
|
+
const taskName = taskDisplayName(task);
|
|
2289
|
+
const guidance =
|
|
2290
|
+
'Terminal state and output metadata are durable. Do not call bg_status to reconfirm; use bg_logs only if output is needed.';
|
|
2291
|
+
const content = [
|
|
2292
|
+
'<background-task-notification>',
|
|
2293
|
+
` <task-id>${task.id}</task-id>`,
|
|
2294
|
+
` <task-name>${escapeXml(taskName)}</task-name>`,
|
|
2295
|
+
` <status>${task.status}</status>`,
|
|
2296
|
+
exit,
|
|
2297
|
+
error,
|
|
2298
|
+
` <output-file>${escapeXml(task.outputPath)}</output-file>`,
|
|
2299
|
+
` <summary>${escapeXml(`Background task ${JSON.stringify(taskName)} ${task.status}`)}</summary>`,
|
|
2300
|
+
` <guidance>${escapeXml(guidance)}</guidance>`,
|
|
2301
|
+
'</background-task-notification>',
|
|
2302
|
+
]
|
|
2303
|
+
.filter(Boolean)
|
|
2304
|
+
.join('\n');
|
|
2305
|
+
|
|
2306
|
+
try {
|
|
2307
|
+
this.sendCompletionNotification(
|
|
2308
|
+
{
|
|
2309
|
+
customType: 'background-task-notification',
|
|
2310
|
+
content,
|
|
2311
|
+
display: true,
|
|
2312
|
+
details: snapshot(task),
|
|
2313
|
+
},
|
|
2314
|
+
{ deliverAs: 'followUp', triggerTurn: task.triggerOnCompletion },
|
|
2315
|
+
);
|
|
2316
|
+
} catch (error) {
|
|
2317
|
+
task.notified = false;
|
|
2318
|
+
throw new Error(
|
|
2319
|
+
`Failed to send background task notification for ${task.id}: ${error instanceof Error ? error.message : String(error)}`,
|
|
2320
|
+
);
|
|
2321
|
+
}
|
|
2322
|
+
}
|
|
2323
|
+
|
|
2324
|
+
private async finalizeTask(
|
|
2325
|
+
task: BgTask,
|
|
2326
|
+
status: TaskStatus,
|
|
2327
|
+
exitCode: number | null,
|
|
2328
|
+
signal?: string | null,
|
|
2329
|
+
error?: string,
|
|
2330
|
+
): Promise<void> {
|
|
2331
|
+
if (task.finalized) return;
|
|
2332
|
+
task.finalized = true;
|
|
2333
|
+
if (task.timeoutHandle) clearTimeout(task.timeoutHandle);
|
|
2334
|
+
if (this.platform === 'win32') this.clearKillEscalationTimer(task);
|
|
2335
|
+
let finalStatus = status;
|
|
2336
|
+
let finalError = error;
|
|
2337
|
+
const posixForceFailure = await this.awaitPosixProcessGroupBeforeTerminal(task);
|
|
2338
|
+
if (posixForceFailure !== undefined) {
|
|
2339
|
+
finalStatus = 'failed';
|
|
2340
|
+
finalError = BackgroundTaskRegistry.appendTaskError(finalError, posixForceFailure.message);
|
|
2341
|
+
}
|
|
2342
|
+
const windowsForceFailure = await this.awaitWindowsForceBeforeTerminal(task);
|
|
2343
|
+
if (windowsForceFailure !== undefined) {
|
|
2344
|
+
finalStatus = 'failed';
|
|
2345
|
+
finalError = BackgroundTaskRegistry.appendTaskError(finalError, windowsForceFailure.message);
|
|
2346
|
+
}
|
|
2347
|
+
task.exitCode = exitCode;
|
|
2348
|
+
task.signal = signal ?? null;
|
|
2349
|
+
|
|
2350
|
+
try {
|
|
2351
|
+
if (task.stream && !task.stream.destroyed) await closeAndFsyncOutputStream(task.stream);
|
|
2352
|
+
} catch (finalizeError) {
|
|
2353
|
+
finalStatus = 'failed';
|
|
2354
|
+
const message =
|
|
2355
|
+
finalizeError instanceof Error ? finalizeError.message : String(finalizeError);
|
|
2356
|
+
finalError = finalError
|
|
2357
|
+
? `${finalError}; final output durability failed: ${message}`
|
|
2358
|
+
: `Final output durability failed: ${message}`;
|
|
2359
|
+
}
|
|
2360
|
+
|
|
2361
|
+
task.endTime = this.now();
|
|
2362
|
+
if (finalError) task.error = finalError;
|
|
2363
|
+
try {
|
|
2364
|
+
await this.writeMetadataSnapshot(task, { ...snapshot(task), status: finalStatus });
|
|
2365
|
+
task.status = finalStatus;
|
|
2366
|
+
} catch (metadataError) {
|
|
2367
|
+
finalStatus = 'failed';
|
|
2368
|
+
task.status = 'failed';
|
|
2369
|
+
task.error = `Terminal metadata write failed: ${metadataError instanceof Error ? metadataError.message : String(metadataError)}`;
|
|
2370
|
+
this.logger.error(
|
|
2371
|
+
`[background-tasks] failed to write metadata for ${task.id}:`,
|
|
2372
|
+
metadataError,
|
|
2373
|
+
);
|
|
2374
|
+
await this.writeMetadata(task).catch((retryError: unknown) => {
|
|
2375
|
+
this.logger.error(
|
|
2376
|
+
`[background-tasks] failed to write failed terminal metadata for ${task.id}:`,
|
|
2377
|
+
retryError,
|
|
2378
|
+
);
|
|
2379
|
+
});
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
for (const waiter of task.waiters.splice(0)) waiter();
|
|
2383
|
+
this.publishTerminal(task);
|
|
2384
|
+
const deliveryGate = await this.waitForTerminalPublicationGate(task);
|
|
2385
|
+
if (deliveryGate.kind === 'rejected') {
|
|
2386
|
+
this.logger.error(
|
|
2387
|
+
`[background-tasks] completion delivery gate failed for ${task.id}: ${this.terminalPublicationError(deliveryGate.error)}`,
|
|
2388
|
+
);
|
|
2389
|
+
} else {
|
|
2390
|
+
// EventBus disposal abandons only EventBus publication. Notification truth
|
|
2391
|
+
// remains independent; notifyCompletion itself suppresses session shutdown.
|
|
2392
|
+
try {
|
|
2393
|
+
this.notifyCompletion(task);
|
|
2394
|
+
} catch (notificationError) {
|
|
2395
|
+
this.logger.error(
|
|
2396
|
+
`[background-tasks] notification failed for ${task.id}:`,
|
|
2397
|
+
notificationError,
|
|
2398
|
+
);
|
|
2399
|
+
}
|
|
2400
|
+
}
|
|
2401
|
+
try {
|
|
2402
|
+
await this.writeMetadata(task);
|
|
2403
|
+
} catch (metadataError) {
|
|
2404
|
+
this.logger.error(
|
|
2405
|
+
`[background-tasks] failed to update notification metadata for ${task.id}:`,
|
|
2406
|
+
metadataError,
|
|
2407
|
+
);
|
|
2408
|
+
}
|
|
2409
|
+
this.pruneOldTasks();
|
|
2410
|
+
}
|
|
2411
|
+
|
|
2412
|
+
private pruneOldTasks(): void {
|
|
2413
|
+
if (this.tasks.size <= this.maxRecentTasks) return;
|
|
2414
|
+
const removable = [...this.tasks.values()]
|
|
2415
|
+
.filter((task) => task.status !== 'running' && task.terminalEmitInFlight !== true)
|
|
2416
|
+
.sort((a, b) => (a.endTime ?? a.startTime) - (b.endTime ?? b.startTime));
|
|
2417
|
+
while (this.tasks.size > this.maxRecentTasks && removable.length > 0) {
|
|
2418
|
+
const task = removable.shift();
|
|
2419
|
+
if (task === undefined) continue;
|
|
2420
|
+
if (task.terminalPublicationState === 'pending') {
|
|
2421
|
+
this.abandonTerminalPublication(task, 'retention_limit');
|
|
2422
|
+
}
|
|
2423
|
+
this.tasks.delete(task.id);
|
|
2424
|
+
}
|
|
2425
|
+
}
|
|
2426
|
+
}
|