pi-simple-background-tasks 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/FORK-CHANGES.md +10 -3
- package/README.md +19 -9
- package/package.json +3 -2
- package/src/core/common.ts +26 -207
- package/src/core/detached-spawn.ts +317 -0
- package/src/core/registry.ts +586 -1203
- package/src/core/task-cleanup.ts +6 -0
- package/src/core/windows-taskkill.ts +4 -11
- package/src/core/windows-wmi-spawn.ts +192 -0
- package/src/extension.ts +34 -180
- package/src/core/reload-shell-owner.ts +0 -1512
package/src/core/registry.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
import { spawn as nodeSpawn, type SpawnOptions } from 'node:child_process';
|
|
2
1
|
import { randomBytes } from 'node:crypto';
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { existsSync } from 'node:fs';
|
|
3
|
+
import { appendFile, mkdir, readdir, readFile, rm, stat } from 'node:fs/promises';
|
|
6
4
|
import { join } from 'node:path';
|
|
7
5
|
import { formatSize } from '@earendil-works/pi-coding-agent';
|
|
8
6
|
import {
|
|
@@ -15,18 +13,11 @@ import {
|
|
|
15
13
|
formatDuration,
|
|
16
14
|
type KillKind,
|
|
17
15
|
normalizeTaskName,
|
|
18
|
-
type ReloadableShellExecutionV1,
|
|
19
|
-
type ReloadShellActivationClaimV1,
|
|
20
|
-
type ReloadShellActivationLeaseV1,
|
|
21
|
-
type ReloadShellHostAdapterV1,
|
|
22
|
-
type ReloadShellIdentityV1,
|
|
23
|
-
type ReloadShellOwnerHubV1,
|
|
24
|
-
ReloadSurvivalError,
|
|
25
16
|
type ResolvedShellPolicy,
|
|
26
17
|
resolveShellPolicy,
|
|
18
|
+
type ShellPolicySnapshot,
|
|
27
19
|
type StartTaskOptions,
|
|
28
20
|
sanitizePathSegment,
|
|
29
|
-
shellInvocationForPolicy,
|
|
30
21
|
shellPolicySnapshot,
|
|
31
22
|
snapshot,
|
|
32
23
|
type TaskStatus,
|
|
@@ -34,21 +25,18 @@ import {
|
|
|
34
25
|
taskDisplayName,
|
|
35
26
|
} from './common.js';
|
|
36
27
|
import { defaultTasksDir } from './config.js';
|
|
28
|
+
import type { DetachedSpawnFn } from './detached-spawn.js';
|
|
29
|
+
import { exitMarkerPath, runnerScriptPath, spawnDetachedTask } from './detached-spawn.js';
|
|
37
30
|
import { BackgroundTaskExtensionServiceClosedError } from './extension-api.js';
|
|
38
|
-
import {
|
|
39
|
-
createReloadableShellExecutionV1,
|
|
40
|
-
RELOAD_SHELL_OWNER_PROTOCOL,
|
|
41
|
-
} from './reload-shell-owner.js';
|
|
42
31
|
import {
|
|
43
32
|
type CleanupScope,
|
|
44
33
|
type CleanupSummary,
|
|
45
34
|
cleanupFinishedTasks as cleanupTasksOnDisk,
|
|
46
35
|
} from './task-cleanup.js';
|
|
47
|
-
import {
|
|
36
|
+
import { writeJsonAtomic } from './task-durable.js';
|
|
48
37
|
import {
|
|
49
38
|
runWindowsTaskkill,
|
|
50
39
|
type TaskkillOutcome,
|
|
51
|
-
type WindowsKillPhase,
|
|
52
40
|
type WindowsTaskkillOptions,
|
|
53
41
|
} from './windows-taskkill.js';
|
|
54
42
|
|
|
@@ -58,7 +46,8 @@ export const STOP_WAIT_MS = KILL_GRACE_MS + 1500;
|
|
|
58
46
|
export const MAX_RECENT_TASKS = 100;
|
|
59
47
|
export const TERMINAL_PUBLICATION_MAX_ATTEMPTS = 3;
|
|
60
48
|
export const TERMINAL_PUBLICATION_RETRY_MS = 100;
|
|
61
|
-
|
|
49
|
+
/** One poller for all tasks: markers, byte cap and timeouts are checked here. */
|
|
50
|
+
export const TASK_POLL_INTERVAL_MS = 1000;
|
|
62
51
|
const TERMINAL_PUBLICATION_DIAGNOSTIC_CHARS = 500;
|
|
63
52
|
|
|
64
53
|
export type TerminalPublicationClosureReason = Extract<
|
|
@@ -80,107 +69,36 @@ export class BackgroundTaskAdmissionClosedError extends Error {
|
|
|
80
69
|
readonly code = 'pi_background_tasks_admission_closed';
|
|
81
70
|
|
|
82
71
|
constructor(kind: string) {
|
|
83
|
-
super(`Cannot start ${kind} after background
|
|
72
|
+
super(`Cannot start ${kind} after background tasks are shutting down`);
|
|
84
73
|
this.name = 'BackgroundTaskAdmissionClosedError';
|
|
85
74
|
}
|
|
86
75
|
}
|
|
87
76
|
|
|
88
|
-
export class BackgroundTaskAdmissionTimeoutError extends Error {
|
|
89
|
-
readonly code = 'pi_background_tasks_admission_timeout';
|
|
90
|
-
|
|
91
|
-
constructor(kind: string, timeoutMs: number) {
|
|
92
|
-
super(`Timed out while preparing ${kind} after ${String(timeoutMs)}ms`);
|
|
93
|
-
this.name = 'BackgroundTaskAdmissionTimeoutError';
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
interface TaskAdmission {
|
|
98
|
-
readonly kind: string;
|
|
99
|
-
readonly controller: AbortController;
|
|
100
|
-
readonly deadlineAt: number;
|
|
101
|
-
readonly timeoutMs: number;
|
|
102
|
-
timeoutHandle: NodeJS.Timeout | undefined;
|
|
103
|
-
released: boolean;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
export interface BackgroundTaskContext {
|
|
107
|
-
cwd: string;
|
|
108
|
-
sessionId?: string;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
interface OutputEventSource {
|
|
112
|
-
on(event: 'data', listener: (data: Buffer | string) => void): unknown;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
interface ChildStdin {
|
|
116
|
-
write(data: Buffer, callback: (error?: Error | null) => void): boolean;
|
|
117
|
-
end(callback?: () => void): unknown;
|
|
118
|
-
once(event: 'error', listener: (error: Error) => void): unknown;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
export interface BackgroundTaskChildProcess {
|
|
122
|
-
pid?: number | undefined;
|
|
123
|
-
stdin?: ChildStdin | null | undefined;
|
|
124
|
-
stdout?: OutputEventSource | null | undefined;
|
|
125
|
-
stderr?: OutputEventSource | null | undefined;
|
|
126
|
-
kill(signal?: NodeJS.Signals): boolean;
|
|
127
|
-
on(event: 'error', listener: (error: Error) => void): unknown;
|
|
128
|
-
on(
|
|
129
|
-
event: 'close',
|
|
130
|
-
listener: (code: number | null, signal: NodeJS.Signals | null) => void,
|
|
131
|
-
): unknown;
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
export type BackgroundTaskSpawn = (
|
|
135
|
-
command: string,
|
|
136
|
-
args: string[],
|
|
137
|
-
options: SpawnOptions,
|
|
138
|
-
) => BackgroundTaskChildProcess;
|
|
139
|
-
|
|
140
|
-
type KillProcessFn = (pid: number, signal?: NodeJS.Signals | number) => boolean;
|
|
141
|
-
type KillTreeFn = (
|
|
142
|
-
pid: number,
|
|
143
|
-
phase: WindowsKillPhase,
|
|
144
|
-
signal?: AbortSignal,
|
|
145
|
-
) => Promise<TaskkillOutcome>;
|
|
146
|
-
|
|
147
77
|
interface PosixProcessGroupKillState {
|
|
148
78
|
readonly groupId: number;
|
|
149
79
|
readonly completion: Promise<void>;
|
|
150
80
|
readonly resolveCompletion: () => void;
|
|
151
81
|
readonly deadlineAt: number;
|
|
82
|
+
verificationTimer?: NodeJS.Timeout | undefined;
|
|
152
83
|
forceAttempted: boolean;
|
|
153
84
|
settled: boolean;
|
|
154
85
|
failure?: Error | undefined;
|
|
155
86
|
lastProbeError?: Error | undefined;
|
|
156
|
-
verificationTimer?: NodeJS.Timeout | undefined;
|
|
157
87
|
failureListeners?: Array<(error: Error) => void> | undefined;
|
|
158
88
|
}
|
|
159
89
|
|
|
160
90
|
interface WindowsKillState {
|
|
161
|
-
softController?: AbortController | undefined;
|
|
162
|
-
softPromise?: Promise<void> | undefined;
|
|
163
91
|
forcePromise?: Promise<void> | undefined;
|
|
164
92
|
forceFailure?: Error | undefined;
|
|
165
93
|
forceFailureListeners?: Array<(error: Error) => void> | undefined;
|
|
166
94
|
}
|
|
167
95
|
|
|
168
|
-
export interface
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
display: true;
|
|
172
|
-
details: BgTaskSnapshot;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
export interface CompletionNotificationOptions {
|
|
176
|
-
deliverAs: 'followUp';
|
|
177
|
-
triggerTurn: boolean;
|
|
96
|
+
export interface BackgroundTaskContext {
|
|
97
|
+
cwd: string;
|
|
98
|
+
sessionId?: string;
|
|
178
99
|
}
|
|
179
100
|
|
|
180
|
-
export type
|
|
181
|
-
message: CompletionNotificationMessage,
|
|
182
|
-
options: CompletionNotificationOptions,
|
|
183
|
-
) => void;
|
|
101
|
+
export type BackgroundTaskSpawn = DetachedSpawnFn;
|
|
184
102
|
|
|
185
103
|
export interface BackgroundTaskRegistryOptions {
|
|
186
104
|
sendCompletionNotification: CompletionNotificationSender;
|
|
@@ -198,9 +116,28 @@ export interface BackgroundTaskRegistryOptions {
|
|
|
198
116
|
maxRecentTasks?: number;
|
|
199
117
|
killGraceMs?: number;
|
|
200
118
|
stopWaitMs?: number;
|
|
201
|
-
|
|
119
|
+
pollIntervalMs?: number;
|
|
202
120
|
logger?: Pick<Console, 'error'>;
|
|
203
|
-
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export type KillProcessFn = (pid: number, signal?: NodeJS.Signals | number) => boolean;
|
|
124
|
+
export type KillTreeFn = (pid: number, signal?: AbortSignal) => Promise<TaskkillOutcome>;
|
|
125
|
+
|
|
126
|
+
export type CompletionNotificationSender = (
|
|
127
|
+
message: CompletionNotificationMessage,
|
|
128
|
+
options: CompletionNotificationOptions,
|
|
129
|
+
) => void;
|
|
130
|
+
|
|
131
|
+
interface CompletionNotificationMessage {
|
|
132
|
+
readonly customType: 'background-task-notification';
|
|
133
|
+
readonly content: string;
|
|
134
|
+
readonly display: boolean;
|
|
135
|
+
readonly details: BgTaskSnapshot;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
interface CompletionNotificationOptions {
|
|
139
|
+
readonly deliverAs: 'followUp';
|
|
140
|
+
readonly triggerTurn: boolean;
|
|
204
141
|
}
|
|
205
142
|
|
|
206
143
|
interface RuntimeDir {
|
|
@@ -214,21 +151,38 @@ function defaultTaskId(): string {
|
|
|
214
151
|
|
|
215
152
|
function noopOnChange(): void {}
|
|
216
153
|
|
|
154
|
+
/** Strictly serializable fields adopted from a previous run's metadata file. */
|
|
155
|
+
type PersistedTaskRecord = {
|
|
156
|
+
status?: unknown;
|
|
157
|
+
readonly command: string;
|
|
158
|
+
readonly cwd: string;
|
|
159
|
+
readonly startTime: number;
|
|
160
|
+
readonly endTime?: number | undefined;
|
|
161
|
+
readonly exitCode?: number | null | undefined;
|
|
162
|
+
readonly signal?: string | null | undefined;
|
|
163
|
+
readonly pid?: number | undefined;
|
|
164
|
+
readonly posixProcessGroupId?: number | undefined;
|
|
165
|
+
readonly bytesWritten: number;
|
|
166
|
+
readonly error?: string | undefined;
|
|
167
|
+
readonly notified: boolean;
|
|
168
|
+
readonly notifyOnCompletion: boolean;
|
|
169
|
+
readonly triggerOnCompletion: boolean;
|
|
170
|
+
readonly timeoutSeconds?: number | undefined;
|
|
171
|
+
readonly shellPolicy?: ShellPolicySnapshot | undefined;
|
|
172
|
+
};
|
|
173
|
+
|
|
217
174
|
export class BackgroundTaskRegistry {
|
|
218
175
|
private readonly tasks = new Map<string, BgTask>();
|
|
219
176
|
private runtimeDir: RuntimeDir | undefined;
|
|
220
177
|
private readonly tasksDir: string;
|
|
221
178
|
private shuttingDown = false;
|
|
222
|
-
private taskAdmissionsClosed = false;
|
|
223
|
-
private readonly activeTaskAdmissions = new Set<TaskAdmission>();
|
|
224
|
-
private readonly taskAdmissionDrainWaiters = new Set<() => void>();
|
|
225
179
|
private terminalPublicationClosed = false;
|
|
226
180
|
private terminalPublicationCloseReason: TerminalPublicationClosureReason | undefined;
|
|
227
181
|
private readonly terminalPublicationClosedSignal: Promise<TerminalPublicationClosureReason>;
|
|
228
182
|
private resolveTerminalPublicationClosedSignal: (
|
|
229
183
|
reason: TerminalPublicationClosureReason,
|
|
230
184
|
) => void = () => {};
|
|
231
|
-
private readonly spawn: BackgroundTaskSpawn;
|
|
185
|
+
private readonly spawn: BackgroundTaskSpawn | undefined;
|
|
232
186
|
private readonly killProcess: KillProcessFn;
|
|
233
187
|
private readonly killTree: KillTreeFn;
|
|
234
188
|
private readonly platform: NodeJS.Platform;
|
|
@@ -241,7 +195,7 @@ export class BackgroundTaskRegistry {
|
|
|
241
195
|
private readonly maxRecentTasks: number;
|
|
242
196
|
private readonly killGraceMs: number;
|
|
243
197
|
private readonly stopWaitMs: number;
|
|
244
|
-
private readonly
|
|
198
|
+
private readonly pollIntervalMs: number;
|
|
245
199
|
private readonly logger: Pick<Console, 'error'>;
|
|
246
200
|
private readonly sendCompletionNotification: CompletionNotificationSender;
|
|
247
201
|
private readonly publishTerminalSnapshot: (task: BgTaskSnapshot) => void;
|
|
@@ -251,16 +205,13 @@ export class BackgroundTaskRegistry {
|
|
|
251
205
|
BgTask,
|
|
252
206
|
TerminalPublicationAbandonSignal
|
|
253
207
|
>();
|
|
254
|
-
private
|
|
255
|
-
private reloadShellLease: ReloadShellActivationLeaseV1 | undefined;
|
|
256
|
-
private reloadShellIdentity: ReloadShellIdentityV1 | undefined;
|
|
208
|
+
private pollTimer: NodeJS.Timeout | undefined;
|
|
257
209
|
|
|
258
210
|
constructor(options: BackgroundTaskRegistryOptions) {
|
|
259
211
|
this.terminalPublicationClosedSignal = new Promise((resolve) => {
|
|
260
212
|
this.resolveTerminalPublicationClosedSignal = resolve;
|
|
261
213
|
});
|
|
262
|
-
this.spawn =
|
|
263
|
-
options.spawn ?? ((command, args, spawnOptions) => nodeSpawn(command, args, spawnOptions));
|
|
214
|
+
this.spawn = options.spawn;
|
|
264
215
|
this.killProcess = options.killProcess ?? process.kill.bind(process);
|
|
265
216
|
this.platform = options.platform ?? process.platform;
|
|
266
217
|
this.env = options.env ?? process.env;
|
|
@@ -270,10 +221,10 @@ export class BackgroundTaskRegistry {
|
|
|
270
221
|
const taskkillEnv = this.env;
|
|
271
222
|
this.killTree =
|
|
272
223
|
options.killTree ??
|
|
273
|
-
((pid,
|
|
224
|
+
((pid, signal) => {
|
|
274
225
|
const taskkillOptions: WindowsTaskkillOptions =
|
|
275
226
|
signal === undefined ? { env: taskkillEnv } : { env: taskkillEnv, signal };
|
|
276
|
-
return runWindowsTaskkill(pid,
|
|
227
|
+
return runWindowsTaskkill(pid, taskkillOptions);
|
|
277
228
|
});
|
|
278
229
|
this.makeTaskIdFn = options.makeTaskId ?? defaultTaskId;
|
|
279
230
|
this.now = options.now ?? Date.now;
|
|
@@ -281,15 +232,10 @@ export class BackgroundTaskRegistry {
|
|
|
281
232
|
this.maxRecentTasks = options.maxRecentTasks ?? MAX_RECENT_TASKS;
|
|
282
233
|
this.killGraceMs = options.killGraceMs ?? KILL_GRACE_MS;
|
|
283
234
|
this.stopWaitMs = options.stopWaitMs ?? STOP_WAIT_MS;
|
|
284
|
-
this.
|
|
285
|
-
options.taskAdmissionTimeoutMs,
|
|
286
|
-
TASK_ADMISSION_TIMEOUT_MS,
|
|
287
|
-
'taskAdmissionTimeoutMs',
|
|
288
|
-
);
|
|
235
|
+
this.pollIntervalMs = options.pollIntervalMs ?? TASK_POLL_INTERVAL_MS;
|
|
289
236
|
this.logger = options.logger ?? console;
|
|
290
237
|
this.sendCompletionNotification = options.sendCompletionNotification;
|
|
291
238
|
this.publishTerminalSnapshot = options.publishTerminal ?? noopOnChange;
|
|
292
|
-
this.reloadShellOwner = options.reloadShellOwner;
|
|
293
239
|
}
|
|
294
240
|
|
|
295
241
|
isShuttingDown(): boolean {
|
|
@@ -308,137 +254,17 @@ export class BackgroundTaskRegistry {
|
|
|
308
254
|
return resolved;
|
|
309
255
|
}
|
|
310
256
|
|
|
311
|
-
private static positiveTimeout(
|
|
312
|
-
value: number | undefined,
|
|
313
|
-
fallback: number,
|
|
314
|
-
label: string,
|
|
315
|
-
): number {
|
|
316
|
-
const candidate = value ?? fallback;
|
|
317
|
-
if (!Number.isFinite(candidate) || candidate <= 0) {
|
|
318
|
-
throw new Error(`${label} must be a positive finite number`);
|
|
319
|
-
}
|
|
320
|
-
return Math.max(1, Math.floor(candidate));
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
private beginTaskAdmission(kind: string): TaskAdmission {
|
|
324
|
-
this.assertTaskAdmissionOpen(kind);
|
|
325
|
-
const controller = new AbortController();
|
|
326
|
-
const admission: TaskAdmission = {
|
|
327
|
-
kind,
|
|
328
|
-
controller,
|
|
329
|
-
deadlineAt: Date.now() + this.taskAdmissionTimeoutMs,
|
|
330
|
-
timeoutMs: this.taskAdmissionTimeoutMs,
|
|
331
|
-
timeoutHandle: undefined,
|
|
332
|
-
released: false,
|
|
333
|
-
};
|
|
334
|
-
admission.timeoutHandle = setTimeout(() => {
|
|
335
|
-
if (admission.released || admission.controller.signal.aborted) return;
|
|
336
|
-
admission.controller.abort(
|
|
337
|
-
new BackgroundTaskAdmissionTimeoutError(admission.kind, admission.timeoutMs),
|
|
338
|
-
);
|
|
339
|
-
}, admission.timeoutMs);
|
|
340
|
-
this.activeTaskAdmissions.add(admission);
|
|
341
|
-
return admission;
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
private releaseTaskAdmission(admission: TaskAdmission): void {
|
|
345
|
-
if (admission.released) return;
|
|
346
|
-
admission.released = true;
|
|
347
|
-
if (admission.timeoutHandle !== undefined) {
|
|
348
|
-
clearTimeout(admission.timeoutHandle);
|
|
349
|
-
admission.timeoutHandle = undefined;
|
|
350
|
-
}
|
|
351
|
-
this.activeTaskAdmissions.delete(admission);
|
|
352
|
-
if (this.activeTaskAdmissions.size !== 0) return;
|
|
353
|
-
for (const resolve of this.taskAdmissionDrainWaiters) resolve();
|
|
354
|
-
this.taskAdmissionDrainWaiters.clear();
|
|
355
|
-
}
|
|
356
|
-
|
|
357
|
-
private taskAdmissionError(admission: TaskAdmission): Error {
|
|
358
|
-
const reason = admission.controller.signal.reason;
|
|
359
|
-
if (reason instanceof Error) return reason;
|
|
360
|
-
if (this.shuttingDown || this.taskAdmissionsClosed) {
|
|
361
|
-
return new BackgroundTaskAdmissionClosedError(admission.kind);
|
|
362
|
-
}
|
|
363
|
-
return new BackgroundTaskAdmissionTimeoutError(admission.kind, admission.timeoutMs);
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
private surfacedTaskAdmissionError(admission: TaskAdmission, ...details: unknown[]): Error {
|
|
367
|
-
const primary = this.taskAdmissionError(admission);
|
|
368
|
-
const meaningful = details.filter((detail) => {
|
|
369
|
-
if (detail === undefined || detail === primary) return false;
|
|
370
|
-
if (typeof detail !== 'object' || detail === null) return true;
|
|
371
|
-
return (
|
|
372
|
-
Reflect.get(detail, 'name') !== 'AbortError' &&
|
|
373
|
-
Reflect.get(detail, 'code') !== Reflect.get(primary, 'code')
|
|
374
|
-
);
|
|
375
|
-
});
|
|
376
|
-
if (meaningful.length === 0) return primary;
|
|
377
|
-
return new AggregateError(
|
|
378
|
-
[primary, ...meaningful],
|
|
379
|
-
`${primary.message}; admission cancellation or cleanup reported additional failures: ${meaningful.map(BackgroundTaskRegistry.errorMessage).join('; ')}`,
|
|
380
|
-
);
|
|
381
|
-
}
|
|
382
|
-
|
|
383
|
-
private assertTaskAdmissionOpen(kind: string, admission?: TaskAdmission): void {
|
|
384
|
-
if (admission?.controller.signal.aborted === true) throw this.taskAdmissionError(admission);
|
|
385
|
-
if (this.shuttingDown || this.taskAdmissionsClosed) {
|
|
386
|
-
throw new BackgroundTaskAdmissionClosedError(kind);
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
|
|
390
|
-
private async awaitTaskAdmissionBoundary<T>(
|
|
391
|
-
promise: Promise<T>,
|
|
392
|
-
admission: TaskAdmission,
|
|
393
|
-
): Promise<T> {
|
|
394
|
-
try {
|
|
395
|
-
const value = await promise;
|
|
396
|
-
this.assertTaskAdmissionOpen(admission.kind, admission);
|
|
397
|
-
return value;
|
|
398
|
-
} catch (error) {
|
|
399
|
-
if (admission.controller.signal.aborted && typeof error === 'object' && error !== null) {
|
|
400
|
-
const isPlainAbort = Reflect.get(error, 'name') === 'AbortError';
|
|
401
|
-
const isCleanDurableCancellation =
|
|
402
|
-
Reflect.get(error, 'code') === 'durable_file_cancelled' &&
|
|
403
|
-
Reflect.get(error, 'renameCompleted') !== true &&
|
|
404
|
-
Array.isArray(Reflect.get(error, 'cleanupFailures')) &&
|
|
405
|
-
(Reflect.get(error, 'cleanupFailures') as unknown[]).length === 0;
|
|
406
|
-
if (isPlainAbort || isCleanDurableCancellation) {
|
|
407
|
-
throw this.taskAdmissionError(admission);
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
throw error;
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
closeTaskAdmissions(): void {
|
|
415
|
-
if (this.taskAdmissionsClosed) return;
|
|
416
|
-
this.taskAdmissionsClosed = true;
|
|
417
|
-
for (const admission of this.activeTaskAdmissions) {
|
|
418
|
-
if (!admission.controller.signal.aborted) {
|
|
419
|
-
admission.controller.abort(new BackgroundTaskAdmissionClosedError(admission.kind));
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
|
|
424
|
-
waitForTaskAdmissions(): Promise<void> {
|
|
425
|
-
if (this.activeTaskAdmissions.size === 0) return Promise.resolve();
|
|
426
|
-
return new Promise((resolve) => {
|
|
427
|
-
this.taskAdmissionDrainWaiters.add(resolve);
|
|
428
|
-
});
|
|
429
|
-
}
|
|
430
|
-
|
|
431
257
|
setShuttingDown(value: boolean): void {
|
|
432
258
|
if (value) {
|
|
433
259
|
this.shuttingDown = true;
|
|
434
|
-
this.
|
|
260
|
+
this.stopPoller();
|
|
435
261
|
this.closeTerminalPublication('registry_shutdown');
|
|
436
262
|
return;
|
|
437
263
|
}
|
|
438
|
-
// Publication
|
|
439
|
-
//
|
|
440
|
-
//
|
|
441
|
-
if (!this.terminalPublicationClosed
|
|
264
|
+
// Publication closure belongs to one extension activation and is one-way.
|
|
265
|
+
// A reloaded Pi creates a fresh registry; an old registry must not be
|
|
266
|
+
// reopened by a late lifecycle continuation.
|
|
267
|
+
if (!this.terminalPublicationClosed) this.shuttingDown = false;
|
|
442
268
|
}
|
|
443
269
|
|
|
444
270
|
closeTerminalPublication(reason: TerminalPublicationClosureReason): void {
|
|
@@ -495,185 +321,6 @@ export class BackgroundTaskRegistry {
|
|
|
495
321
|
return summary;
|
|
496
322
|
}
|
|
497
323
|
|
|
498
|
-
hasCurrentReloadLease(): boolean {
|
|
499
|
-
const lease = this.reloadShellLease;
|
|
500
|
-
return lease !== undefined && this.reloadShellOwner?.isCurrentLease(lease) === true;
|
|
501
|
-
}
|
|
502
|
-
|
|
503
|
-
async stageReloadActivation(
|
|
504
|
-
claim: ReloadShellActivationClaimV1,
|
|
505
|
-
): Promise<ReloadShellHostAdapterV1> {
|
|
506
|
-
if (claim.protocol !== RELOAD_SHELL_OWNER_PROTOCOL) {
|
|
507
|
-
throw new ReloadSurvivalError(
|
|
508
|
-
'pi_bg_reload_owner_protocol_incompatible',
|
|
509
|
-
'activation claim does not use the supported reload shell owner protocol',
|
|
510
|
-
);
|
|
511
|
-
}
|
|
512
|
-
const staged: ReloadableShellExecutionV1[] = [];
|
|
513
|
-
const ids = new Set<string>();
|
|
514
|
-
for (const execution of claim.executions) {
|
|
515
|
-
if (
|
|
516
|
-
execution.protocol !== RELOAD_SHELL_OWNER_PROTOCOL ||
|
|
517
|
-
execution.task.reloadExecution !== execution ||
|
|
518
|
-
execution.task.surviveReload !== true
|
|
519
|
-
) {
|
|
520
|
-
throw new ReloadSurvivalError(
|
|
521
|
-
'pi_bg_reload_owner_protocol_incompatible',
|
|
522
|
-
'activation claim contains an incompatible reload shell execution',
|
|
523
|
-
);
|
|
524
|
-
}
|
|
525
|
-
if (ids.has(execution.task.id) || this.tasks.has(execution.task.id)) {
|
|
526
|
-
throw new ReloadSurvivalError(
|
|
527
|
-
'pi_bg_reload_owner_activation_conflict',
|
|
528
|
-
`claimed task id ${execution.task.id} conflicts with the fresh registry`,
|
|
529
|
-
);
|
|
530
|
-
}
|
|
531
|
-
ids.add(execution.task.id);
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
try {
|
|
535
|
-
for (const execution of claim.executions) {
|
|
536
|
-
const task = execution.task;
|
|
537
|
-
task.reloadHostDeliveryInFlight = false;
|
|
538
|
-
task.reloadHostDeliverySettled = false;
|
|
539
|
-
task.reloadHostNotificationSettled = false;
|
|
540
|
-
execution.updateLeaseAudit(claim.generation, (task.reloadSurvival?.handoffCount ?? 0) + 1);
|
|
541
|
-
this.tasks.set(task.id, task);
|
|
542
|
-
staged.push(execution);
|
|
543
|
-
}
|
|
544
|
-
await Promise.all(staged.map(async (execution) => this.writeMetadata(execution.task)));
|
|
545
|
-
} catch (error) {
|
|
546
|
-
for (const execution of staged) this.tasks.delete(execution.task.id);
|
|
547
|
-
throw error;
|
|
548
|
-
}
|
|
549
|
-
|
|
550
|
-
let boundLease: ReloadShellActivationLeaseV1 | undefined;
|
|
551
|
-
return {
|
|
552
|
-
activationNonce: claim.activationNonce,
|
|
553
|
-
onBound: (lease) => {
|
|
554
|
-
if (
|
|
555
|
-
lease.activationNonce !== claim.activationNonce ||
|
|
556
|
-
lease.generation !== claim.generation ||
|
|
557
|
-
lease.identityKey !== claim.identityKey
|
|
558
|
-
) {
|
|
559
|
-
throw new ReloadSurvivalError(
|
|
560
|
-
'pi_bg_reload_owner_stale_claim',
|
|
561
|
-
'committed lease does not match its staged activation claim',
|
|
562
|
-
);
|
|
563
|
-
}
|
|
564
|
-
boundLease = lease;
|
|
565
|
-
this.reloadShellLease = lease;
|
|
566
|
-
this.reloadShellIdentity = claim.identity;
|
|
567
|
-
},
|
|
568
|
-
onChanged: (execution) => {
|
|
569
|
-
const lease = boundLease;
|
|
570
|
-
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
571
|
-
},
|
|
572
|
-
onTerminal: (execution) => {
|
|
573
|
-
const lease = boundLease;
|
|
574
|
-
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
575
|
-
void this.deliverReloadTerminal(execution, lease);
|
|
576
|
-
},
|
|
577
|
-
};
|
|
578
|
-
}
|
|
579
|
-
|
|
580
|
-
abortReloadActivation(claim: ReloadShellActivationClaimV1): void {
|
|
581
|
-
for (const execution of claim.executions) {
|
|
582
|
-
const task = execution.task;
|
|
583
|
-
if (this.tasks.get(task.id) !== task) continue;
|
|
584
|
-
if (task.terminalPublishRetryHandle !== undefined) {
|
|
585
|
-
clearTimeout(task.terminalPublishRetryHandle);
|
|
586
|
-
task.terminalPublishRetryHandle = undefined;
|
|
587
|
-
}
|
|
588
|
-
task.terminalPublicationGate = undefined;
|
|
589
|
-
task.terminalPublishInFlight = false;
|
|
590
|
-
this.tasks.delete(task.id);
|
|
591
|
-
}
|
|
592
|
-
if (this.reloadShellLease?.activationNonce === claim.activationNonce) {
|
|
593
|
-
this.reloadShellLease = undefined;
|
|
594
|
-
this.reloadShellIdentity = undefined;
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
prepareReloadHandoff(lease: ReloadShellActivationLeaseV1): readonly BgTask[] {
|
|
599
|
-
if (this.reloadShellOwner === undefined || this.reloadShellLease !== lease) {
|
|
600
|
-
throw new ReloadSurvivalError(
|
|
601
|
-
'pi_bg_reload_owner_stale_claim',
|
|
602
|
-
'registry does not own the requested reload activation lease',
|
|
603
|
-
);
|
|
604
|
-
}
|
|
605
|
-
const executions = this.reloadShellOwner.beginReloadHandoff(lease);
|
|
606
|
-
const tasks: BgTask[] = [];
|
|
607
|
-
for (const execution of executions) {
|
|
608
|
-
const task = execution.task;
|
|
609
|
-
if (this.tasks.get(task.id) !== task) {
|
|
610
|
-
throw new ReloadSurvivalError(
|
|
611
|
-
'pi_bg_reload_owner_stale_claim',
|
|
612
|
-
`registry no longer owns survivor ${task.id}`,
|
|
613
|
-
);
|
|
614
|
-
}
|
|
615
|
-
if (task.terminalPublishRetryHandle !== undefined) {
|
|
616
|
-
clearTimeout(task.terminalPublishRetryHandle);
|
|
617
|
-
task.terminalPublishRetryHandle = undefined;
|
|
618
|
-
}
|
|
619
|
-
task.terminalPublicationGate = undefined;
|
|
620
|
-
task.terminalPublishInFlight = false;
|
|
621
|
-
task.reloadHostDeliveryInFlight = false;
|
|
622
|
-
task.reloadHostDeliverySettled = false;
|
|
623
|
-
task.reloadHostNotificationSettled = false;
|
|
624
|
-
this.tasks.delete(task.id);
|
|
625
|
-
tasks.push(task);
|
|
626
|
-
}
|
|
627
|
-
this.reloadShellLease = undefined;
|
|
628
|
-
this.reloadShellIdentity = undefined;
|
|
629
|
-
return Object.freeze(tasks);
|
|
630
|
-
}
|
|
631
|
-
|
|
632
|
-
async waitForReloadHostSettlement(timeoutMs = this.stopWaitMs): Promise<void> {
|
|
633
|
-
const deadline = Date.now() + timeoutMs;
|
|
634
|
-
while (true) {
|
|
635
|
-
const unsettled = [...this.tasks.values()].filter(
|
|
636
|
-
(task) =>
|
|
637
|
-
task.reloadExecution?.phase === 'terminal' &&
|
|
638
|
-
(task.reloadHostNotificationSettled !== true ||
|
|
639
|
-
task.terminalPublicationState === 'pending'),
|
|
640
|
-
);
|
|
641
|
-
if (unsettled.length === 0) return;
|
|
642
|
-
const remaining = deadline - Date.now();
|
|
643
|
-
if (remaining <= 0) {
|
|
644
|
-
throw new Error(
|
|
645
|
-
`Timed out waiting for reload shell host settlement: ${unsettled.map((task) => task.id).join(', ')}`,
|
|
646
|
-
);
|
|
647
|
-
}
|
|
648
|
-
await new Promise<void>((resolve) => setTimeout(resolve, Math.min(10, remaining)));
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
|
|
652
|
-
releaseReloadActivation(lease: ReloadShellActivationLeaseV1): void {
|
|
653
|
-
if (this.reloadShellOwner === undefined) return;
|
|
654
|
-
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) return;
|
|
655
|
-
this.reloadShellOwner.releaseActivation(lease);
|
|
656
|
-
this.reloadShellLease = undefined;
|
|
657
|
-
this.reloadShellIdentity = undefined;
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
currentReloadLease(): ReloadShellActivationLeaseV1 | undefined {
|
|
661
|
-
return this.hasCurrentReloadLease() ? this.reloadShellLease : undefined;
|
|
662
|
-
}
|
|
663
|
-
|
|
664
|
-
private ownsReloadExecution(
|
|
665
|
-
execution: ReloadableShellExecutionV1,
|
|
666
|
-
lease: ReloadShellActivationLeaseV1 | undefined,
|
|
667
|
-
): lease is ReloadShellActivationLeaseV1 {
|
|
668
|
-
return (
|
|
669
|
-
lease !== undefined &&
|
|
670
|
-
this.reloadShellLease === lease &&
|
|
671
|
-
this.reloadShellOwner?.isCurrentLease(lease) === true &&
|
|
672
|
-
this.tasks.get(execution.task.id) === execution.task &&
|
|
673
|
-
execution.task.reloadExecution === execution
|
|
674
|
-
);
|
|
675
|
-
}
|
|
676
|
-
|
|
677
324
|
async ensureRuntimeDir(ctx: BackgroundTaskContext): Promise<RuntimeDir> {
|
|
678
325
|
if (this.runtimeDir) return this.runtimeDir;
|
|
679
326
|
const sessionId = sanitizePathSegment(ctx.sessionId ?? `session-${String(process.pid)}`);
|
|
@@ -684,164 +331,27 @@ export class BackgroundTaskRegistry {
|
|
|
684
331
|
return this.runtimeDir;
|
|
685
332
|
}
|
|
686
333
|
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
};
|
|
695
|
-
stream.once('close', closed);
|
|
696
|
-
if (!stream.destroyed) stream.destroy();
|
|
697
|
-
if (stream.closed) closed();
|
|
698
|
-
});
|
|
699
|
-
}
|
|
700
|
-
|
|
701
|
-
private async discardUnspawnedTask(task: BgTask, paths: readonly string[]): Promise<void> {
|
|
702
|
-
this.tasks.delete(task.id);
|
|
703
|
-
task.finalized = true;
|
|
704
|
-
task.status = 'failed';
|
|
705
|
-
if (task.timeoutHandle !== undefined) clearTimeout(task.timeoutHandle);
|
|
706
|
-
if (task.killEscalationTimer !== undefined) clearTimeout(task.killEscalationTimer);
|
|
707
|
-
await this.destroyTaskStream(task);
|
|
708
|
-
const removals = await Promise.allSettled(paths.map((path) => rm(path, { force: true })));
|
|
709
|
-
const failures: Error[] = [];
|
|
710
|
-
for (let index = 0; index < removals.length; index++) {
|
|
711
|
-
const result = removals[index];
|
|
712
|
-
if (result?.status !== 'rejected') continue;
|
|
713
|
-
const path = paths[index] ?? '<unknown admission artifact>';
|
|
714
|
-
failures.push(
|
|
715
|
-
new Error(
|
|
716
|
-
`Failed to remove interrupted admission artifact ${path}: ${BackgroundTaskRegistry.errorMessage(result.reason)}`,
|
|
717
|
-
),
|
|
718
|
-
);
|
|
719
|
-
}
|
|
720
|
-
if (failures.length > 0) {
|
|
721
|
-
throw new AggregateError(failures, 'Interrupted task admission artifact cleanup failed');
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
private captureSpawnedChild(task: BgTask, child: BackgroundTaskChildProcess): void {
|
|
726
|
-
task.child = child;
|
|
727
|
-
task.pid = child.pid;
|
|
728
|
-
if (
|
|
729
|
-
this.platform !== 'win32' &&
|
|
730
|
-
child.pid !== undefined &&
|
|
731
|
-
Number.isSafeInteger(child.pid) &&
|
|
732
|
-
child.pid > 0
|
|
733
|
-
) {
|
|
734
|
-
// Capture detached-group ownership once, directly from this spawn. Never
|
|
735
|
-
// reconstruct signal authority from mutable task metadata or a later PID.
|
|
736
|
-
task.ownedPosixProcessGroupId = child.pid;
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
|
|
740
|
-
private bindOwnedTaskToAdmission(task: BgTask, admission: TaskAdmission): () => void {
|
|
741
|
-
const cancelOwnedTask = (): void => {
|
|
742
|
-
this.stopOwnedTaskAfterAdmissionCancellation(task, this.taskAdmissionError(admission));
|
|
743
|
-
};
|
|
744
|
-
admission.controller.signal.addEventListener('abort', cancelOwnedTask, { once: true });
|
|
745
|
-
if (admission.controller.signal.aborted) cancelOwnedTask();
|
|
746
|
-
return () => {
|
|
747
|
-
admission.controller.signal.removeEventListener('abort', cancelOwnedTask);
|
|
748
|
-
};
|
|
749
|
-
}
|
|
750
|
-
|
|
751
|
-
private stopOwnedTaskAfterAdmissionCancellation(task: BgTask, error: Error): void {
|
|
752
|
-
if (task.status !== 'running') return;
|
|
753
|
-
task.killKind = error instanceof BackgroundTaskAdmissionTimeoutError ? 'timeout' : 'shutdown';
|
|
754
|
-
task.error = error.message;
|
|
755
|
-
try {
|
|
756
|
-
this.requestKill(task, 'SIGTERM');
|
|
757
|
-
} catch (killError) {
|
|
758
|
-
this.logger.error(
|
|
759
|
-
`[background-tasks] failed to stop ${task.id} after admission cancellation:`,
|
|
760
|
-
killError,
|
|
761
|
-
);
|
|
762
|
-
}
|
|
763
|
-
}
|
|
764
|
-
|
|
334
|
+
/**
|
|
335
|
+
* The single launch path. The child is spawned detached from the very first
|
|
336
|
+
* step and the extension never owns it at runtime: ownership facts (pid, pgid,
|
|
337
|
+
* marker path) are recorded on disk, so a same-process /reload or a full Pi
|
|
338
|
+
* restart cannot lose a running task. Node exit is impossible for detached
|
|
339
|
+
* children to report back, so the wrapper's exit marker is the terminal truth.
|
|
340
|
+
*/
|
|
765
341
|
async startTask(
|
|
766
342
|
ctx: BackgroundTaskContext,
|
|
767
343
|
command: string,
|
|
768
344
|
options: StartTaskOptions = {},
|
|
769
345
|
): Promise<BgTask> {
|
|
770
|
-
|
|
771
|
-
if (hasSurvival && typeof options.surviveReload !== 'boolean') {
|
|
772
|
-
throw new ReloadSurvivalError(
|
|
773
|
-
'pi_bg_survive_reload_invalid',
|
|
774
|
-
'surviveReload must be a boolean when present',
|
|
775
|
-
);
|
|
776
|
-
}
|
|
777
|
-
const surviveReload = options.surviveReload === true;
|
|
778
|
-
const reloadLease = surviveReload ? this.currentReloadLease() : undefined;
|
|
779
|
-
if (surviveReload && reloadLease === undefined) {
|
|
780
|
-
throw new ReloadSurvivalError(
|
|
781
|
-
'pi_bg_reload_owner_unavailable',
|
|
782
|
-
'no successfully bound same-process reload owner activation is available',
|
|
783
|
-
);
|
|
784
|
-
}
|
|
785
|
-
|
|
786
|
-
const admission = this.beginTaskAdmission('a background task');
|
|
787
|
-
try {
|
|
788
|
-
if (surviveReload && reloadLease !== undefined) {
|
|
789
|
-
return await this.startReloadableTaskAdmitted(
|
|
790
|
-
ctx,
|
|
791
|
-
command,
|
|
792
|
-
options,
|
|
793
|
-
admission,
|
|
794
|
-
reloadLease,
|
|
795
|
-
);
|
|
796
|
-
}
|
|
797
|
-
return await this.startTaskAdmitted(ctx, command, options, admission);
|
|
798
|
-
} finally {
|
|
799
|
-
this.releaseTaskAdmission(admission);
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
|
|
803
|
-
private async startReloadableTaskAdmitted(
|
|
804
|
-
ctx: BackgroundTaskContext,
|
|
805
|
-
command: string,
|
|
806
|
-
options: StartTaskOptions,
|
|
807
|
-
admission: TaskAdmission,
|
|
808
|
-
lease: ReloadShellActivationLeaseV1,
|
|
809
|
-
): Promise<BgTask> {
|
|
346
|
+
if (this.shuttingDown) throw new BackgroundTaskAdmissionClosedError('a background task');
|
|
810
347
|
const normalizedCommand = command.trim();
|
|
811
348
|
if (!normalizedCommand) throw new Error('Background command is empty');
|
|
812
|
-
if (
|
|
813
|
-
this.reloadShellOwner === undefined ||
|
|
814
|
-
this.reloadShellIdentity === undefined ||
|
|
815
|
-
this.reloadShellLease !== lease ||
|
|
816
|
-
!this.reloadShellOwner.isCurrentLease(lease)
|
|
817
|
-
) {
|
|
818
|
-
throw new ReloadSurvivalError(
|
|
819
|
-
'pi_bg_reload_owner_unavailable',
|
|
820
|
-
'reload owner activation became unavailable before launch',
|
|
821
|
-
);
|
|
822
|
-
}
|
|
823
|
-
if (ctx.sessionId !== this.reloadShellIdentity.sessionId) {
|
|
824
|
-
throw new ReloadSurvivalError(
|
|
825
|
-
'pi_bg_reload_owner_stale_claim',
|
|
826
|
-
'launch context session id does not match the bound reload owner identity',
|
|
827
|
-
);
|
|
828
|
-
}
|
|
829
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
830
|
-
|
|
831
|
-
const shellPolicy = this.resolvedShellPolicy();
|
|
832
|
-
const invocation = shellInvocationForPolicy(normalizedCommand, shellPolicy);
|
|
833
|
-
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
834
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
835
|
-
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) {
|
|
836
|
-
throw new ReloadSurvivalError(
|
|
837
|
-
'pi_bg_reload_owner_stale_claim',
|
|
838
|
-
'reload owner activation changed during task preflight',
|
|
839
|
-
);
|
|
840
|
-
}
|
|
841
349
|
|
|
350
|
+
const dir = await this.ensureRuntimeDir(ctx);
|
|
842
351
|
const id = this.makeTaskIdFn();
|
|
843
352
|
const outputAbsPath = join(dir.abs, `${id}.output`);
|
|
844
353
|
const metadataAbsPath = join(dir.abs, `${id}.json`);
|
|
354
|
+
const shellPolicy = this.resolvedShellPolicy();
|
|
845
355
|
const outputPath = join(dir.display, `${id}.output`);
|
|
846
356
|
const timeoutSeconds =
|
|
847
357
|
typeof options.timeoutSeconds === 'number' &&
|
|
@@ -856,6 +366,7 @@ export class BackgroundTaskRegistry {
|
|
|
856
366
|
const trimmedDescription = options.description?.trim();
|
|
857
367
|
const description =
|
|
858
368
|
trimmedDescription && trimmedDescription.length > 0 ? trimmedDescription : undefined;
|
|
369
|
+
|
|
859
370
|
const task: BgTask = {
|
|
860
371
|
id,
|
|
861
372
|
name: taskName,
|
|
@@ -869,8 +380,8 @@ export class BackgroundTaskRegistry {
|
|
|
869
380
|
startTime: this.now(),
|
|
870
381
|
exitCode: undefined,
|
|
871
382
|
pid: undefined,
|
|
383
|
+
posixProcessGroupId: undefined,
|
|
872
384
|
bytesWritten: 0,
|
|
873
|
-
surviveReload: true,
|
|
874
385
|
notified: false,
|
|
875
386
|
notifyOnCompletion: options.notifyOnCompletion ?? true,
|
|
876
387
|
triggerOnCompletion: options.triggerOnCompletion ?? false,
|
|
@@ -880,296 +391,57 @@ export class BackgroundTaskRegistry {
|
|
|
880
391
|
terminalPublishAttempts: 0,
|
|
881
392
|
terminalPublicationGate: options.terminalPublicationGate,
|
|
882
393
|
shellPolicy: shellPolicySnapshot(shellPolicy),
|
|
394
|
+
exitMarkerAbsPath: exitMarkerPath(dir.abs, id),
|
|
395
|
+
launched: false,
|
|
883
396
|
waiters: [],
|
|
884
397
|
};
|
|
885
|
-
const launchNonce = randomBytes(16).toString('hex');
|
|
886
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
887
398
|
this.tasks.set(id, task);
|
|
399
|
+
// Persist before spawning so a crash mid-launch leaves an adoptable record.
|
|
400
|
+
await this.writeMetadata(task);
|
|
888
401
|
|
|
889
|
-
let execution: ReloadableShellExecutionV1 | undefined;
|
|
890
|
-
let registered = false;
|
|
891
|
-
let committed = false;
|
|
892
|
-
let abortListener: (() => void) | undefined;
|
|
893
402
|
try {
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
spawn: this.spawn,
|
|
901
|
-
killProcess: this.killProcess,
|
|
902
|
-
killTree: this.killTree,
|
|
903
|
-
platform: this.platform,
|
|
403
|
+
const result = await spawnDetachedTask({
|
|
404
|
+
id,
|
|
405
|
+
taskDirAbs: dir.abs,
|
|
406
|
+
command: normalizedCommand,
|
|
407
|
+
policy: shellPolicy,
|
|
408
|
+
cwd: ctx.cwd,
|
|
904
409
|
env: this.env,
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
stopWaitMs: this.stopWaitMs,
|
|
908
|
-
now: this.now,
|
|
909
|
-
logger: this.logger,
|
|
410
|
+
...(this.spawn === undefined ? {} : { spawn: this.spawn }),
|
|
411
|
+
platform: this.platform,
|
|
910
412
|
});
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
const error = this.taskAdmissionError(admission);
|
|
916
|
-
execution.failAdmission(error);
|
|
917
|
-
const kind: KillKind =
|
|
918
|
-
error instanceof BackgroundTaskAdmissionTimeoutError ? 'timeout' : 'shutdown';
|
|
919
|
-
void execution.requestStop(kind, error.message).catch((stopError: unknown) => {
|
|
920
|
-
this.logger.error(
|
|
921
|
-
`[background-tasks] failed to stop reloadable task ${task.id} after admission cancellation:`,
|
|
922
|
-
stopError,
|
|
923
|
-
);
|
|
924
|
-
});
|
|
925
|
-
};
|
|
926
|
-
admission.controller.signal.addEventListener('abort', abortListener, { once: true });
|
|
927
|
-
if (admission.controller.signal.aborted) abortListener();
|
|
928
|
-
|
|
929
|
-
await this.awaitTaskAdmissionBoundary(
|
|
930
|
-
execution.commitInitialMetadata(admission.controller.signal),
|
|
931
|
-
admission,
|
|
932
|
-
);
|
|
933
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
934
|
-
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) {
|
|
935
|
-
throw new ReloadSurvivalError(
|
|
936
|
-
'pi_bg_reload_owner_stale_claim',
|
|
937
|
-
'reload owner activation changed before admission commit',
|
|
938
|
-
);
|
|
413
|
+
task.pid = result.pid;
|
|
414
|
+
task.posixProcessGroupId = result.posixProcessGroupId;
|
|
415
|
+
if (result.posixProcessGroupId !== undefined) {
|
|
416
|
+
task.ownedPosixProcessGroupId = result.posixProcessGroupId;
|
|
939
417
|
}
|
|
940
|
-
|
|
941
|
-
|
|
418
|
+
task.exitMarkerAbsPath = result.exitMarkerFile;
|
|
419
|
+
task.exitMarkerTrusted = result.exitMarkerTrusted;
|
|
420
|
+
task.runnerScriptFile = result.runnerScriptFile;
|
|
421
|
+
// The launch is settled: the poller may reap this task from here on.
|
|
422
|
+
task.launched = true;
|
|
423
|
+
await this.writeMetadata(task);
|
|
424
|
+
this.ensurePoller();
|
|
942
425
|
return task;
|
|
943
|
-
} catch (
|
|
944
|
-
const
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
if (task.status === 'running') {
|
|
954
|
-
await execution.requestStop(
|
|
955
|
-
admission.controller.signal.aborted &&
|
|
956
|
-
primary instanceof BackgroundTaskAdmissionTimeoutError
|
|
957
|
-
? 'timeout'
|
|
958
|
-
: 'shutdown',
|
|
959
|
-
primary.message,
|
|
960
|
-
);
|
|
961
|
-
}
|
|
962
|
-
} catch (stopError) {
|
|
963
|
-
cleanupError = stopError;
|
|
964
|
-
}
|
|
965
|
-
}
|
|
966
|
-
if (registered && !committed && execution !== undefined) {
|
|
967
|
-
if (this.reloadShellOwner.isCurrentLease(lease)) {
|
|
968
|
-
this.reloadShellOwner.releaseExecution(lease, execution);
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
this.tasks.delete(task.id);
|
|
972
|
-
if (execution === undefined) {
|
|
973
|
-
const removals = await Promise.allSettled([
|
|
426
|
+
} catch (launchError) {
|
|
427
|
+
const message = launchError instanceof Error ? launchError.message : String(launchError);
|
|
428
|
+
if (task.pid === undefined) {
|
|
429
|
+
// Nothing was launched; sweep every artifact so no half-task remains.
|
|
430
|
+
task.finalized = true;
|
|
431
|
+
task.status = 'failed';
|
|
432
|
+
task.error = message;
|
|
433
|
+
this.tasks.delete(id);
|
|
434
|
+
const runner = join(dir.abs, `${id}.runner.cmd`);
|
|
435
|
+
await Promise.allSettled([
|
|
974
436
|
rm(outputAbsPath, { force: true }),
|
|
975
437
|
rm(metadataAbsPath, { force: true }),
|
|
438
|
+
rm(runner, { force: true }),
|
|
976
439
|
]);
|
|
977
|
-
|
|
978
|
-
if (removalFailure?.status === 'rejected') cleanupError = removalFailure.reason;
|
|
979
|
-
}
|
|
980
|
-
if (admission.controller.signal.aborted) {
|
|
981
|
-
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
982
|
-
}
|
|
983
|
-
if (cleanupError !== undefined) {
|
|
984
|
-
throw new AggregateError(
|
|
985
|
-
[primary, cleanupError],
|
|
986
|
-
`Failed to start reloadable background task and cleanup also failed: ${BackgroundTaskRegistry.errorMessage(cleanupError)}`,
|
|
987
|
-
);
|
|
988
|
-
}
|
|
989
|
-
throw new Error(`Failed to start background task: ${primary.message}`);
|
|
990
|
-
} finally {
|
|
991
|
-
if (abortListener !== undefined) {
|
|
992
|
-
admission.controller.signal.removeEventListener('abort', abortListener);
|
|
993
|
-
}
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
|
|
997
|
-
private async startTaskAdmitted(
|
|
998
|
-
ctx: BackgroundTaskContext,
|
|
999
|
-
command: string,
|
|
1000
|
-
options: StartTaskOptions,
|
|
1001
|
-
admission: TaskAdmission,
|
|
1002
|
-
): Promise<BgTask> {
|
|
1003
|
-
const normalizedCommand = command.trim();
|
|
1004
|
-
if (!normalizedCommand) throw new Error('Background command is empty');
|
|
1005
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1006
|
-
|
|
1007
|
-
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
1008
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1009
|
-
const id = this.makeTaskIdFn();
|
|
1010
|
-
const outputAbsPath = join(dir.abs, `${id}.output`);
|
|
1011
|
-
const metadataAbsPath = join(dir.abs, `${id}.json`);
|
|
1012
|
-
const outputPath = join(dir.display, `${id}.output`);
|
|
1013
|
-
const shellPolicy = this.resolvedShellPolicy();
|
|
1014
|
-
const invocation = shellInvocationForPolicy(normalizedCommand, shellPolicy);
|
|
1015
|
-
const timeoutSeconds =
|
|
1016
|
-
typeof options.timeoutSeconds === 'number' &&
|
|
1017
|
-
Number.isFinite(options.timeoutSeconds) &&
|
|
1018
|
-
options.timeoutSeconds > 0
|
|
1019
|
-
? Math.floor(options.timeoutSeconds)
|
|
1020
|
-
: undefined;
|
|
1021
|
-
const taskName =
|
|
1022
|
-
normalizeTaskName(options.name) ??
|
|
1023
|
-
normalizeTaskName(options.description) ??
|
|
1024
|
-
deriveTaskNameFromCommand(normalizedCommand);
|
|
1025
|
-
const trimmedDescription = options.description?.trim();
|
|
1026
|
-
const description =
|
|
1027
|
-
trimmedDescription && trimmedDescription.length > 0 ? trimmedDescription : undefined;
|
|
1028
|
-
|
|
1029
|
-
const task: BgTask = {
|
|
1030
|
-
id,
|
|
1031
|
-
name: taskName,
|
|
1032
|
-
command: normalizedCommand,
|
|
1033
|
-
description,
|
|
1034
|
-
status: 'running',
|
|
1035
|
-
outputPath,
|
|
1036
|
-
outputAbsPath,
|
|
1037
|
-
metadataAbsPath,
|
|
1038
|
-
cwd: ctx.cwd,
|
|
1039
|
-
startTime: this.now(),
|
|
1040
|
-
exitCode: undefined,
|
|
1041
|
-
pid: undefined,
|
|
1042
|
-
bytesWritten: 0,
|
|
1043
|
-
surviveReload: false,
|
|
1044
|
-
notified: false,
|
|
1045
|
-
notifyOnCompletion: options.notifyOnCompletion ?? true,
|
|
1046
|
-
triggerOnCompletion: options.triggerOnCompletion ?? false,
|
|
1047
|
-
timeoutSeconds,
|
|
1048
|
-
terminalPublished: false,
|
|
1049
|
-
terminalPublicationState: 'pending',
|
|
1050
|
-
terminalPublishAttempts: 0,
|
|
1051
|
-
terminalPublicationGate: options.terminalPublicationGate,
|
|
1052
|
-
shellPolicy: shellPolicySnapshot(shellPolicy),
|
|
1053
|
-
waiters: [],
|
|
1054
|
-
};
|
|
1055
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1056
|
-
this.tasks.set(id, task);
|
|
1057
|
-
|
|
1058
|
-
const stream = createWriteStream(outputAbsPath, { flags: 'a', encoding: 'utf8' });
|
|
1059
|
-
task.stream = stream;
|
|
1060
|
-
stream.on('error', (error) => {
|
|
1061
|
-
task.error = `Output file write failed: ${error.message}`;
|
|
1062
|
-
if (task.status === 'running') {
|
|
1063
|
-
task.killKind = 'output_cap';
|
|
1064
|
-
try {
|
|
1065
|
-
this.requestKill(task, 'SIGTERM');
|
|
1066
|
-
} catch (killError) {
|
|
1067
|
-
void this.finalizeTask(
|
|
1068
|
-
task,
|
|
1069
|
-
'failed',
|
|
1070
|
-
null,
|
|
1071
|
-
undefined,
|
|
1072
|
-
`${task.error}; kill failed: ${killError instanceof Error ? killError.message : String(killError)}`,
|
|
1073
|
-
);
|
|
1074
|
-
}
|
|
1075
|
-
}
|
|
1076
|
-
});
|
|
1077
|
-
|
|
1078
|
-
let unbindAdmissionCancellation = (): void => undefined;
|
|
1079
|
-
try {
|
|
1080
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1081
|
-
const child = this.spawn(invocation.shell, invocation.args, {
|
|
1082
|
-
cwd: ctx.cwd,
|
|
1083
|
-
detached: this.platform !== 'win32',
|
|
1084
|
-
stdio: ['ignore', 'pipe', 'pipe'],
|
|
1085
|
-
env: this.env,
|
|
1086
|
-
windowsHide: true,
|
|
1087
|
-
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
|
|
1088
|
-
});
|
|
1089
|
-
|
|
1090
|
-
this.captureSpawnedChild(task, child);
|
|
1091
|
-
|
|
1092
|
-
child.stdout?.on('data', (data) => {
|
|
1093
|
-
this.appendChildOutput(task, data);
|
|
1094
|
-
});
|
|
1095
|
-
child.stderr?.on('data', (data) => {
|
|
1096
|
-
this.appendChildOutput(task, data);
|
|
1097
|
-
});
|
|
1098
|
-
|
|
1099
|
-
child.on('error', (error) => {
|
|
1100
|
-
this.writeNotice(task, `\n[background task spawn error: ${error.message}]\n`);
|
|
1101
|
-
void this.finalizeTask(task, 'failed', null, undefined, error.message);
|
|
1102
|
-
});
|
|
1103
|
-
|
|
1104
|
-
child.on('close', (code, signalName) => {
|
|
1105
|
-
let status: TaskStatus;
|
|
1106
|
-
let error: string | undefined;
|
|
1107
|
-
if (task.killKind === 'user' || task.killKind === 'shutdown') {
|
|
1108
|
-
status = 'killed';
|
|
1109
|
-
} else if (task.killKind === 'timeout') {
|
|
1110
|
-
status = 'failed';
|
|
1111
|
-
error = task.error ?? `Timed out after ${String(timeoutSeconds)}s`;
|
|
1112
|
-
} else if (task.killKind === 'output_cap') {
|
|
1113
|
-
status = 'failed';
|
|
1114
|
-
error = task.error ?? `Output exceeded cap of ${formatSize(this.maxOutputBytes)}`;
|
|
1115
|
-
} else if ((code ?? 0) === 0) {
|
|
1116
|
-
status = 'completed';
|
|
1117
|
-
} else {
|
|
1118
|
-
status = 'failed';
|
|
1119
|
-
const exitCode = code === null ? 'null' : String(code);
|
|
1120
|
-
error = `Exited with code ${exitCode}${signalName ? ` (${signalName})` : ''}`;
|
|
1121
|
-
}
|
|
1122
|
-
void this.finalizeTask(task, status, code, signalName, error);
|
|
1123
|
-
});
|
|
1124
|
-
|
|
1125
|
-
if (timeoutSeconds !== undefined) {
|
|
1126
|
-
task.timeoutHandle = setTimeout(() => {
|
|
1127
|
-
if (task.status !== 'running') return;
|
|
1128
|
-
task.killKind = 'timeout';
|
|
1129
|
-
task.error = `Timed out after ${String(timeoutSeconds)}s`;
|
|
1130
|
-
this.writeNotice(task, `\n[background task timeout: ${task.error}]\n`);
|
|
1131
|
-
try {
|
|
1132
|
-
this.requestKill(task, 'SIGTERM');
|
|
1133
|
-
} catch (error) {
|
|
1134
|
-
void this.finalizeTask(
|
|
1135
|
-
task,
|
|
1136
|
-
'failed',
|
|
1137
|
-
null,
|
|
1138
|
-
undefined,
|
|
1139
|
-
`${task.error}; kill failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1140
|
-
);
|
|
1141
|
-
}
|
|
1142
|
-
}, timeoutSeconds * 1000);
|
|
1143
|
-
}
|
|
1144
|
-
|
|
1145
|
-
unbindAdmissionCancellation = this.bindOwnedTaskToAdmission(task, admission);
|
|
1146
|
-
await this.awaitTaskAdmissionBoundary(
|
|
1147
|
-
this.writeMetadata(task, admission.controller.signal),
|
|
1148
|
-
admission,
|
|
1149
|
-
);
|
|
1150
|
-
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1151
|
-
return task;
|
|
1152
|
-
} catch (error) {
|
|
1153
|
-
if (admission.controller.signal.aborted) {
|
|
1154
|
-
const admissionError = this.taskAdmissionError(admission);
|
|
1155
|
-
let cleanupError: unknown;
|
|
1156
|
-
if (task.child === undefined) {
|
|
1157
|
-
try {
|
|
1158
|
-
await this.discardUnspawnedTask(task, [outputAbsPath, metadataAbsPath]);
|
|
1159
|
-
} catch (cleanupFailure) {
|
|
1160
|
-
cleanupError = cleanupFailure;
|
|
1161
|
-
}
|
|
1162
|
-
} else {
|
|
1163
|
-
this.stopOwnedTaskAfterAdmissionCancellation(task, admissionError);
|
|
1164
|
-
}
|
|
1165
|
-
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
440
|
+
throw launchError;
|
|
1166
441
|
}
|
|
1167
|
-
|
|
1168
|
-
this.writeNotice(task, `\n[background task spawn exception: ${message}]\n`);
|
|
442
|
+
await this.appendNotice(task, `\n[background task launch error: ${message}]\n`);
|
|
1169
443
|
await this.finalizeTask(task, 'failed', null, undefined, message);
|
|
1170
444
|
throw new Error(`Failed to start background task: ${message}`);
|
|
1171
|
-
} finally {
|
|
1172
|
-
unbindAdmissionCancellation();
|
|
1173
445
|
}
|
|
1174
446
|
}
|
|
1175
447
|
|
|
@@ -1192,9 +464,6 @@ export class BackgroundTaskRegistry {
|
|
|
1192
464
|
if (task.status !== 'running') {
|
|
1193
465
|
throw new Error(`Task ${task.id} is ${task.status}, not running`);
|
|
1194
466
|
}
|
|
1195
|
-
if (task.reloadExecution !== undefined) {
|
|
1196
|
-
return task.reloadExecution.requestStop(kind, reason);
|
|
1197
|
-
}
|
|
1198
467
|
const stopWaitMs = this.stopWaitMs;
|
|
1199
468
|
if (
|
|
1200
469
|
this.platform !== 'win32' &&
|
|
@@ -1280,70 +549,365 @@ export class BackgroundTaskRegistry {
|
|
|
1280
549
|
};
|
|
1281
550
|
}
|
|
1282
551
|
|
|
1283
|
-
|
|
1284
|
-
|
|
552
|
+
// ---------- adoption: running tasks recorded on disk by an earlier activation ----------
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Scan the tasks folder for persisted records still marked `running` and
|
|
556
|
+
* either adopt them (process group alive), finalize them from their exit
|
|
557
|
+
* marker, or mark them failed (process gone without a marker). Runs on every
|
|
558
|
+
* activation, which is exactly what makes /reload and Pi restarts safe.
|
|
559
|
+
*/
|
|
560
|
+
async reconcileTasksFromDisk(): Promise<number> {
|
|
561
|
+
const runDirs = await this.listRunDirs();
|
|
562
|
+
let adopted = 0;
|
|
563
|
+
for (const runDir of runDirs) {
|
|
564
|
+
const ids = await this.listTaskIds(runDir);
|
|
565
|
+
for (const id of ids) {
|
|
566
|
+
if (this.tasks.has(id)) continue;
|
|
567
|
+
const metadataPath = join(runDir, `${id}.json`);
|
|
568
|
+
let record: PersistedTaskRecord | undefined;
|
|
569
|
+
try {
|
|
570
|
+
record = JSON.parse(await readFile(metadataPath, 'utf8')) as PersistedTaskRecord;
|
|
571
|
+
} catch {
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
const task = this.taskFromMetadata(id, runDir, metadataPath, record);
|
|
575
|
+
if (task === undefined) continue;
|
|
576
|
+
// Only records whose durable status is `running` are adopted. A task
|
|
577
|
+
// that already reached terminal state while another activation owned
|
|
578
|
+
// it stays on disk untouched: re-importing it would replicate its
|
|
579
|
+
// notification instead of merely observing it.
|
|
580
|
+
if (record.status !== 'running') continue;
|
|
581
|
+
this.tasks.set(task.id, task);
|
|
582
|
+
adopted++;
|
|
583
|
+
|
|
584
|
+
const marker =
|
|
585
|
+
task.exitMarkerTrusted === true ? await this.readExitMarker(task) : undefined;
|
|
586
|
+
if (marker !== undefined) {
|
|
587
|
+
void this.finalizeTask(task, ...this.terminalFromMarker(task, marker));
|
|
588
|
+
continue;
|
|
589
|
+
}
|
|
590
|
+
if (this.probeProcessAlive(task)) {
|
|
591
|
+
this.ensurePoller();
|
|
592
|
+
continue;
|
|
593
|
+
}
|
|
594
|
+
void this.finalizeTask(
|
|
595
|
+
task,
|
|
596
|
+
'failed',
|
|
597
|
+
null,
|
|
598
|
+
undefined,
|
|
599
|
+
'Task process is gone and no completion marker was written; the task ran under an earlier Pi activation',
|
|
600
|
+
);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
return adopted;
|
|
1285
604
|
}
|
|
1286
605
|
|
|
1287
|
-
private async
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
606
|
+
private async listRunDirs(): Promise<string[]> {
|
|
607
|
+
try {
|
|
608
|
+
const entries = await readdir(this.tasksDir, { withFileTypes: true });
|
|
609
|
+
return entries
|
|
610
|
+
.filter((entry) => entry.isDirectory())
|
|
611
|
+
.map((entry) => join(this.tasksDir, entry.name));
|
|
612
|
+
} catch {
|
|
613
|
+
return [];
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
private async listTaskIds(runDir: string): Promise<string[]> {
|
|
618
|
+
try {
|
|
619
|
+
const entries = await readdir(runDir);
|
|
620
|
+
return entries
|
|
621
|
+
.filter((name) => name.endsWith('.json'))
|
|
622
|
+
.map((name) => name.slice(0, name.length - '.json'.length));
|
|
623
|
+
} catch {
|
|
624
|
+
return [];
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
private taskFromMetadata(
|
|
629
|
+
id: string,
|
|
630
|
+
runDir: string,
|
|
631
|
+
metadataAbsPath: string,
|
|
632
|
+
record: PersistedTaskRecord,
|
|
633
|
+
): BgTask | undefined {
|
|
634
|
+
const command = typeof record.command === 'string' ? record.command : undefined;
|
|
635
|
+
const cwd = typeof record.cwd === 'string' ? record.cwd : undefined;
|
|
636
|
+
const startTime = typeof record.startTime === 'number' ? record.startTime : undefined;
|
|
637
|
+
if (command === undefined || cwd === undefined || startTime === undefined) return undefined;
|
|
638
|
+
const dialect = record.shellPolicy?.dialect;
|
|
639
|
+
// An unclassified user shell ran unwrapped and never writes a marker.
|
|
640
|
+
const exitMarkerTrusted = dialect !== 'user-non-posix';
|
|
641
|
+
return {
|
|
642
|
+
id,
|
|
643
|
+
name: deriveTaskNameFromCommand(command),
|
|
644
|
+
command,
|
|
645
|
+
description: undefined,
|
|
646
|
+
status: 'running',
|
|
647
|
+
outputPath: join(runDir, `${id}.output`),
|
|
648
|
+
outputAbsPath: join(runDir, `${id}.output`),
|
|
649
|
+
metadataAbsPath,
|
|
650
|
+
cwd,
|
|
651
|
+
startTime,
|
|
652
|
+
exitCode: undefined,
|
|
653
|
+
pid: typeof record.pid === 'number' ? record.pid : undefined,
|
|
654
|
+
posixProcessGroupId:
|
|
655
|
+
typeof record.posixProcessGroupId === 'number' ? record.posixProcessGroupId : undefined,
|
|
656
|
+
ownedPosixProcessGroupId:
|
|
657
|
+
this.platform !== 'win32' && typeof record.posixProcessGroupId === 'number'
|
|
658
|
+
? record.posixProcessGroupId
|
|
659
|
+
: undefined,
|
|
660
|
+
bytesWritten: typeof record.bytesWritten === 'number' ? record.bytesWritten : 0,
|
|
661
|
+
notified: record.notified === true,
|
|
662
|
+
notifyOnCompletion: record.notifyOnCompletion !== false,
|
|
663
|
+
triggerOnCompletion: record.triggerOnCompletion === true,
|
|
664
|
+
timeoutSeconds: typeof record.timeoutSeconds === 'number' ? record.timeoutSeconds : undefined,
|
|
665
|
+
terminalPublished: false,
|
|
666
|
+
terminalPublicationState: 'pending',
|
|
667
|
+
terminalPublishAttempts: 0,
|
|
668
|
+
shellPolicy: record.shellPolicy,
|
|
669
|
+
exitMarkerAbsPath: exitMarkerPath(runDir, id),
|
|
670
|
+
exitMarkerTrusted,
|
|
671
|
+
runnerScriptFile:
|
|
672
|
+
dialect === 'cmd' && record.status === 'running' ? runnerScriptPath(runDir, id) : undefined,
|
|
673
|
+
// Adopted records fight no in-flight spawn: never in the launching window.
|
|
674
|
+
launched: true,
|
|
675
|
+
waiters: [],
|
|
1294
676
|
};
|
|
1295
|
-
const previous = task.metadataWriteChain ?? Promise.resolve();
|
|
1296
|
-
const next = previous.then(write, write);
|
|
1297
|
-
task.metadataWriteChain = next.catch(() => undefined);
|
|
1298
|
-
await next;
|
|
1299
677
|
}
|
|
1300
678
|
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
679
|
+
private async readExitMarker(task: BgTask): Promise<string | undefined> {
|
|
680
|
+
if (task.exitMarkerAbsPath === undefined) return undefined;
|
|
681
|
+
try {
|
|
682
|
+
return (await readFile(task.exitMarkerAbsPath, 'utf8')).trim();
|
|
683
|
+
} catch {
|
|
684
|
+
return undefined;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
/** Terminal status derived from a wrapper exit marker. */
|
|
689
|
+
private terminalFromMarker(
|
|
690
|
+
task: BgTask,
|
|
691
|
+
markerValue: string,
|
|
692
|
+
): [
|
|
693
|
+
status: TaskStatus,
|
|
694
|
+
exitCode: number | null,
|
|
695
|
+
signal: string | null,
|
|
696
|
+
error: string | undefined,
|
|
697
|
+
] {
|
|
698
|
+
if (!/^\d+$/u.test(markerValue)) {
|
|
699
|
+
return ['failed', null, null, 'Completion marker did not contain a valid exit code'];
|
|
700
|
+
}
|
|
701
|
+
const exitCode = Number(markerValue);
|
|
702
|
+
if (task.capExceeded === true) {
|
|
703
|
+
return ['failed', exitCode, null, task.error ?? 'Output cap exceeded'];
|
|
704
|
+
}
|
|
705
|
+
if (task.killKind === 'user' || task.killKind === 'shutdown') {
|
|
706
|
+
return ['killed', exitCode, null, task.error];
|
|
707
|
+
}
|
|
708
|
+
if (task.killKind === 'timeout') {
|
|
709
|
+
return ['failed', exitCode, null, task.error ?? 'Timed out'];
|
|
710
|
+
}
|
|
711
|
+
if (exitCode === 0) return ['completed', exitCode, null, undefined];
|
|
712
|
+
return ['failed', exitCode, null, `Exited with code ${String(exitCode)}`];
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
/** Group liveness on POSIX; pid existence on Windows. Never sends a signal. */
|
|
716
|
+
private probeProcessAlive(task: BgTask): boolean {
|
|
717
|
+
if (this.platform !== 'win32') {
|
|
718
|
+
const group = task.ownedPosixProcessGroupId;
|
|
719
|
+
if (group === undefined) return false;
|
|
720
|
+
try {
|
|
721
|
+
return this.killProcess(-group, 0) !== false;
|
|
722
|
+
} catch (error) {
|
|
723
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
724
|
+
return false;
|
|
725
|
+
}
|
|
726
|
+
// Probe errors other than ESRCH are inconclusive: stay a poller tick.
|
|
727
|
+
return true;
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
if (task.pid === undefined) return false;
|
|
731
|
+
try {
|
|
732
|
+
return this.killProcess(task.pid, 0) !== false;
|
|
733
|
+
} catch (error) {
|
|
734
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
735
|
+
return false;
|
|
736
|
+
}
|
|
737
|
+
return true;
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
// ---------- one poller: completion markers, byte cap, timeouts, exits ----------
|
|
742
|
+
|
|
743
|
+
private ensurePoller(): void {
|
|
744
|
+
if (this.shuttingDown || this.pollTimer !== undefined) return;
|
|
745
|
+
this.pollTimer = setInterval(() => this.tick(), this.pollIntervalMs);
|
|
746
|
+
this.pollTimer.unref();
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
private stopPoller(): void {
|
|
750
|
+
if (this.pollTimer === undefined) return;
|
|
751
|
+
clearInterval(this.pollTimer);
|
|
752
|
+
this.pollTimer = undefined;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
private tick(): void {
|
|
756
|
+
if (this.shuttingDown) {
|
|
757
|
+
this.stopPoller();
|
|
758
|
+
return;
|
|
759
|
+
}
|
|
760
|
+
let hasRunning = false;
|
|
761
|
+
for (const task of [...this.tasks.values()]) {
|
|
762
|
+
if (task.status !== 'running' || task.finalized === true) continue;
|
|
763
|
+
hasRunning = true;
|
|
764
|
+
if (task.pollInFlight === true) continue;
|
|
765
|
+
task.pollInFlight = true;
|
|
766
|
+
void this.pollTask(task)
|
|
767
|
+
.catch((error: unknown) => {
|
|
768
|
+
this.logger.error(
|
|
769
|
+
`[background-tasks] poll failed for ${task.id}:`,
|
|
770
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
771
|
+
);
|
|
772
|
+
})
|
|
773
|
+
.finally(() => {
|
|
774
|
+
task.pollInFlight = false;
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
if (!hasRunning) this.stopPoller();
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
private async pollTask(task: BgTask): Promise<void> {
|
|
781
|
+
if (task.status !== 'running' || task.finalized === true) return;
|
|
782
|
+
|
|
783
|
+
// 1. output size bookkeeping: kept current for snapshots on every tick.
|
|
784
|
+
let outputSize: number | undefined;
|
|
785
|
+
try {
|
|
786
|
+
outputSize = (await stat(task.outputAbsPath)).size;
|
|
787
|
+
task.bytesWritten = outputSize;
|
|
788
|
+
} catch {
|
|
789
|
+
// stat failures are transient (or the file is gone); the next tick retries.
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
// 2. completion marker: terminal truth.
|
|
793
|
+
const marker = task.exitMarkerTrusted === true ? await this.readExitMarker(task) : undefined;
|
|
794
|
+
if (marker !== undefined) {
|
|
795
|
+
const [status, exitCode, signal, error] = this.terminalFromMarker(task, marker);
|
|
796
|
+
await this.finalizeTask(task, status, exitCode, signal, error);
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
// 3. cap enforcement on a still-running task
|
|
801
|
+
if (task.capExceeded !== true && outputSize !== undefined && outputSize > this.maxOutputBytes) {
|
|
802
|
+
task.capExceeded = true;
|
|
803
|
+
task.killKind = 'output_cap';
|
|
804
|
+
task.error = `Output exceeded cap of ${formatSize(this.maxOutputBytes)}; terminating task`;
|
|
805
|
+
const notice = `\n\n[background task error: ${task.error}]\n`;
|
|
806
|
+
if (this.platform === 'win32') {
|
|
807
|
+
// The child's own `1>` redirection keeps writing past the append: the
|
|
808
|
+
// tree must be gone before the notice, or the flood overwrites it.
|
|
809
|
+
this.startTerminate(task);
|
|
810
|
+
await this.waitForEnd(task, this.stopWaitMs);
|
|
811
|
+
await this.appendNotice(task, notice);
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
await this.appendNotice(task, notice);
|
|
815
|
+
this.startTerminate(task);
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
// 4. timeout enforcement
|
|
820
|
+
if (
|
|
821
|
+
task.killSignalSent !== true &&
|
|
822
|
+
task.timeoutSeconds !== undefined &&
|
|
823
|
+
this.now() - task.startTime >= task.timeoutSeconds * 1000
|
|
824
|
+
) {
|
|
825
|
+
task.killKind = 'timeout';
|
|
826
|
+
task.error = `Timed out after ${String(task.timeoutSeconds)}s`;
|
|
827
|
+
const notice = `\n[background task timeout: ${task.error}]\n`;
|
|
828
|
+
if (this.platform === 'win32') {
|
|
829
|
+
// Same ordering caveat as the output cap: flood bytes must not run
|
|
830
|
+
// past the append, so the notice goes after the tree stop.
|
|
831
|
+
this.startTerminate(task);
|
|
832
|
+
await this.waitForEnd(task, this.stopWaitMs);
|
|
833
|
+
await this.appendNotice(task, notice);
|
|
834
|
+
return;
|
|
835
|
+
}
|
|
836
|
+
await this.appendNotice(task, notice);
|
|
837
|
+
this.startTerminate(task);
|
|
838
|
+
return;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
// Launched tasks only: while the detached spawn is still in flight (WMI
|
|
842
|
+
// create can take seconds on Windows) the record has no pid and no marker,
|
|
843
|
+
// which is indistinguishable at this step from a reaped orphan. The spawn
|
|
844
|
+
// itself settles the record if it fails, so polling it is pure harm.
|
|
845
|
+
if (task.launched === false) return;
|
|
1305
846
|
|
|
1306
|
-
|
|
1307
|
-
if (
|
|
1308
|
-
|
|
1309
|
-
task
|
|
1310
|
-
return;
|
|
847
|
+
// 5. exited without a marker (wrapper killed externally, or no wrapper)
|
|
848
|
+
if (!this.probeProcessAlive(task)) {
|
|
849
|
+
const [status, error] = this.terminalFromKindWithoutMarker(task);
|
|
850
|
+
await this.finalizeTask(task, status, null, undefined, error);
|
|
1311
851
|
}
|
|
852
|
+
}
|
|
1312
853
|
|
|
1313
|
-
|
|
1314
|
-
if (
|
|
1315
|
-
task.
|
|
1316
|
-
|
|
854
|
+
private terminalFromKindWithoutMarker(task: BgTask): [TaskStatus, string | undefined] {
|
|
855
|
+
if (task.capExceeded === true || task.killKind === 'output_cap') {
|
|
856
|
+
return ['failed', task.error ?? 'Output cap exceeded'];
|
|
857
|
+
}
|
|
858
|
+
if (task.killKind === 'timeout') {
|
|
859
|
+
return ['failed', task.error ?? 'Timed out'];
|
|
1317
860
|
}
|
|
861
|
+
if (task.killKind === 'user' || task.killKind === 'shutdown') {
|
|
862
|
+
return ['killed', task.error];
|
|
863
|
+
}
|
|
864
|
+
return [
|
|
865
|
+
'failed',
|
|
866
|
+
'Process exited without writing a completion marker; its exit code is unknown',
|
|
867
|
+
];
|
|
868
|
+
}
|
|
1318
869
|
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
task.
|
|
1325
|
-
task
|
|
1326
|
-
try {
|
|
1327
|
-
this.requestKill(task, 'SIGTERM');
|
|
1328
|
-
} catch (error) {
|
|
1329
|
-
task.error = `${task.error}; kill failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
1330
|
-
void this.finalizeTask(task, 'failed', null, undefined, task.error);
|
|
1331
|
-
}
|
|
870
|
+
/** Request the one TREE stop; errors surface on the poller or stopTask path. */
|
|
871
|
+
private startTerminate(task: BgTask): void {
|
|
872
|
+
try {
|
|
873
|
+
this.requestKill(task, 'SIGTERM');
|
|
874
|
+
} catch (error) {
|
|
875
|
+
task.error = `${task.error}; kill failed: ${BackgroundTaskRegistry.errorMessage(error)}`;
|
|
876
|
+
void this.finalizeTask(task, 'failed', null, undefined, task.error);
|
|
1332
877
|
}
|
|
1333
878
|
}
|
|
1334
879
|
|
|
1335
|
-
/** Persist an internally generated notice (spawn/timeout/cap
|
|
1336
|
-
private
|
|
880
|
+
/** Persist an internally generated notice (spawn/timeout/cap) verbatim. */
|
|
881
|
+
private async appendNotice(task: BgTask, text: string): Promise<void> {
|
|
1337
882
|
if (!text) return;
|
|
1338
|
-
|
|
883
|
+
// On Windows the child's own `1>` redirection holds the output file with a
|
|
884
|
+
// lock that rejects a concurrent append until the tree is gone; a bounded
|
|
885
|
+
// retry lets the notice land after the termination release.
|
|
886
|
+
const attempts = this.platform === 'win32' ? 6 : 1;
|
|
887
|
+
for (let attempt = 1; attempt <= attempts; attempt++) {
|
|
888
|
+
try {
|
|
889
|
+
await appendFile(task.outputAbsPath, text, 'utf8');
|
|
890
|
+
task.bytesWritten += Buffer.byteLength(text, 'utf8');
|
|
891
|
+
return;
|
|
892
|
+
} catch (error) {
|
|
893
|
+
const busy =
|
|
894
|
+
this.platform === 'win32' &&
|
|
895
|
+
error instanceof Error &&
|
|
896
|
+
'code' in error &&
|
|
897
|
+
(error as NodeJS.ErrnoException).code === 'EBUSY';
|
|
898
|
+
if (!busy || attempt === attempts) {
|
|
899
|
+
this.logger.error(
|
|
900
|
+
`[background-tasks] failed to append notice for ${task.id}:`,
|
|
901
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
902
|
+
);
|
|
903
|
+
return;
|
|
904
|
+
}
|
|
905
|
+
await new Promise((resolve) => setTimeout(resolve, 400).unref?.());
|
|
906
|
+
}
|
|
907
|
+
}
|
|
1339
908
|
}
|
|
1340
909
|
|
|
1341
|
-
|
|
1342
|
-
if (!task.stream || task.stream.destroyed) return;
|
|
1343
|
-
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf8');
|
|
1344
|
-
if (buffer.length === 0) return;
|
|
1345
|
-
this.writeToStream(task, buffer);
|
|
1346
|
-
}
|
|
910
|
+
// ---------- POSIX process-group kill machinery ----------
|
|
1347
911
|
|
|
1348
912
|
private beginPosixProcessGroupKill(task: BgTask, armGrace: boolean): PosixProcessGroupKillState {
|
|
1349
913
|
const existing = this.posixProcessGroupKillStates.get(task);
|
|
@@ -1351,18 +915,15 @@ export class BackgroundTaskRegistry {
|
|
|
1351
915
|
if (task.posixProcessGroupSignalAuthorityReleased === true) {
|
|
1352
916
|
throw new Error(`Task ${task.id} has released its POSIX process-group signal authority`);
|
|
1353
917
|
}
|
|
1354
|
-
|
|
1355
|
-
if (groupId === undefined) {
|
|
918
|
+
if (task.ownedPosixProcessGroupId === undefined) {
|
|
1356
919
|
throw new Error(`Task ${task.id} has no owned POSIX process group`);
|
|
1357
920
|
}
|
|
921
|
+
const groupId = task.ownedPosixProcessGroupId;
|
|
1358
922
|
|
|
1359
923
|
let resolveCompletion = (): void => undefined;
|
|
1360
924
|
const completion = new Promise<void>((resolve) => {
|
|
1361
925
|
resolveCompletion = resolve;
|
|
1362
926
|
});
|
|
1363
|
-
// Finish ownership slightly before stopTask's waiter so a force/proof
|
|
1364
|
-
// failure is observed as that specific loud error instead of a generic
|
|
1365
|
-
// cancellation timeout.
|
|
1366
927
|
const reserveMs = Math.min(25, Math.max(1, Math.floor(this.stopWaitMs / 4)));
|
|
1367
928
|
const ownershipMs = Math.max(1, this.stopWaitMs - reserveMs);
|
|
1368
929
|
const state: PosixProcessGroupKillState = {
|
|
@@ -1376,8 +937,6 @@ export class BackgroundTaskRegistry {
|
|
|
1376
937
|
this.posixProcessGroupKillStates.set(task, state);
|
|
1377
938
|
|
|
1378
939
|
if (armGrace) {
|
|
1379
|
-
// Publish the sole force owner before TERM. An injected signal can emit
|
|
1380
|
-
// close reentrantly; that close must see and await this exact state.
|
|
1381
940
|
task.killEscalationTimer = setTimeout(
|
|
1382
941
|
() => {
|
|
1383
942
|
task.killEscalationTimer = undefined;
|
|
@@ -1385,8 +944,8 @@ export class BackgroundTaskRegistry {
|
|
|
1385
944
|
},
|
|
1386
945
|
Math.min(this.killGraceMs, ownershipMs),
|
|
1387
946
|
);
|
|
1388
|
-
//
|
|
1389
|
-
//
|
|
947
|
+
// Stays referenced on purpose: a departed leader must not let the host
|
|
948
|
+
// exit before its owned group is verified gone.
|
|
1390
949
|
}
|
|
1391
950
|
return state;
|
|
1392
951
|
}
|
|
@@ -1449,7 +1008,7 @@ export class BackgroundTaskRegistry {
|
|
|
1449
1008
|
state.failure = error;
|
|
1450
1009
|
this.finishPosixProcessGroupKill(task, state, false);
|
|
1451
1010
|
task.error = BackgroundTaskRegistry.appendTaskError(task.error, error.message);
|
|
1452
|
-
this.
|
|
1011
|
+
void this.appendNotice(task, `\n[background task POSIX termination: ${error.message}]\n`);
|
|
1453
1012
|
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1454
1013
|
this.logger.error(
|
|
1455
1014
|
`[background-tasks] failed to write POSIX process-group failure metadata for ${task.id}:`,
|
|
@@ -1490,8 +1049,6 @@ export class BackgroundTaskRegistry {
|
|
|
1490
1049
|
|
|
1491
1050
|
private forceOwnedPosixProcessGroup(task: BgTask, state: PosixProcessGroupKillState): void {
|
|
1492
1051
|
if (state.settled || state.forceAttempted) return;
|
|
1493
|
-
// Latch before either probe or signal: both are injected boundaries that can
|
|
1494
|
-
// reentrantly emit root close, and no continuation may launch a second KILL.
|
|
1495
1052
|
state.forceAttempted = true;
|
|
1496
1053
|
this.clearKillEscalationTimer(task);
|
|
1497
1054
|
if (this.observeOwnedPosixProcessGroupGone(task, state)) return;
|
|
@@ -1535,8 +1092,6 @@ export class BackgroundTaskRegistry {
|
|
|
1535
1092
|
return;
|
|
1536
1093
|
}
|
|
1537
1094
|
if (task.killSignalSent) return;
|
|
1538
|
-
// Publish de-duplication before the signal boundary for the same reason the
|
|
1539
|
-
// state/timer is published above: close may be emitted synchronously.
|
|
1540
1095
|
task.killSignalSent = true;
|
|
1541
1096
|
|
|
1542
1097
|
const errors: string[] = [];
|
|
@@ -1553,17 +1108,8 @@ export class BackgroundTaskRegistry {
|
|
|
1553
1108
|
}
|
|
1554
1109
|
|
|
1555
1110
|
if (!killed) {
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
if (!killed) errors.push(`child ${signal} returned false`);
|
|
1559
|
-
} catch (error) {
|
|
1560
|
-
errors.push(`child kill failed: ${BackgroundTaskRegistry.errorMessage(error)}`);
|
|
1561
|
-
}
|
|
1562
|
-
}
|
|
1563
|
-
|
|
1564
|
-
if (!killed) {
|
|
1565
|
-
// If TERM reached neither target, waiting out grace has no benefit. Keep
|
|
1566
|
-
// the same owner but attempt its one force phase immediately.
|
|
1111
|
+
// If TERM reached no target, waiting out grace has no benefit. Keep the
|
|
1112
|
+
// same owner but attempt its one force phase immediately.
|
|
1567
1113
|
if (!state.settled) this.forceOwnedPosixProcessGroup(task, state);
|
|
1568
1114
|
throw new Error(`Could not kill task ${task.id}: ${errors.join('; ')}`);
|
|
1569
1115
|
}
|
|
@@ -1573,9 +1119,9 @@ export class BackgroundTaskRegistry {
|
|
|
1573
1119
|
if (this.platform === 'win32') return undefined;
|
|
1574
1120
|
const state = this.posixProcessGroupKillStates.get(task);
|
|
1575
1121
|
if (state === undefined) {
|
|
1576
|
-
// No tree stop won the race before
|
|
1577
|
-
//
|
|
1578
|
-
//
|
|
1122
|
+
// No tree stop won the race before terminalization. Release signal
|
|
1123
|
+
// authority synchronously so a concurrent late stop waits for terminal
|
|
1124
|
+
// settlement instead of targeting a potentially reused group id.
|
|
1579
1125
|
task.posixProcessGroupSignalAuthorityReleased = true;
|
|
1580
1126
|
delete task.ownedPosixProcessGroupId;
|
|
1581
1127
|
return undefined;
|
|
@@ -1620,6 +1166,71 @@ export class BackgroundTaskRegistry {
|
|
|
1620
1166
|
});
|
|
1621
1167
|
}
|
|
1622
1168
|
|
|
1169
|
+
// ---------- Windows taskkill tree machinery ----------
|
|
1170
|
+
|
|
1171
|
+
private waitForEnd(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
1172
|
+
if (task.status !== 'running') return Promise.resolve(true);
|
|
1173
|
+
return new Promise((resolve) => {
|
|
1174
|
+
const timeout = setTimeout(() => {
|
|
1175
|
+
const idx = task.waiters.indexOf(done);
|
|
1176
|
+
if (idx >= 0) task.waiters.splice(idx, 1);
|
|
1177
|
+
resolve(false);
|
|
1178
|
+
}, timeoutMs);
|
|
1179
|
+
const done = () => {
|
|
1180
|
+
clearTimeout(timeout);
|
|
1181
|
+
resolve(true);
|
|
1182
|
+
};
|
|
1183
|
+
task.waiters.push(done);
|
|
1184
|
+
});
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
private waitForEndOrWindowsForceFailure(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
1188
|
+
const state = this.getWindowsKillState(task);
|
|
1189
|
+
if (state.forceFailure !== undefined) return Promise.reject(state.forceFailure);
|
|
1190
|
+
if (task.status !== 'running') return Promise.resolve(true);
|
|
1191
|
+
return new Promise((resolve, reject) => {
|
|
1192
|
+
const cleanup = () => {
|
|
1193
|
+
clearTimeout(timeout);
|
|
1194
|
+
const waiterIndex = task.waiters.indexOf(done);
|
|
1195
|
+
if (waiterIndex >= 0) task.waiters.splice(waiterIndex, 1);
|
|
1196
|
+
const listeners = state.forceFailureListeners;
|
|
1197
|
+
if (listeners !== undefined) {
|
|
1198
|
+
const listenerIndex = listeners.indexOf(failed);
|
|
1199
|
+
if (listenerIndex >= 0) listeners.splice(listenerIndex, 1);
|
|
1200
|
+
if (listeners.length === 0) delete state.forceFailureListeners;
|
|
1201
|
+
}
|
|
1202
|
+
};
|
|
1203
|
+
const timeout = setTimeout(() => {
|
|
1204
|
+
cleanup();
|
|
1205
|
+
resolve(false);
|
|
1206
|
+
}, timeoutMs);
|
|
1207
|
+
const done = () => {
|
|
1208
|
+
cleanup();
|
|
1209
|
+
resolve(true);
|
|
1210
|
+
};
|
|
1211
|
+
const failed = (error: Error) => {
|
|
1212
|
+
cleanup();
|
|
1213
|
+
reject(error);
|
|
1214
|
+
};
|
|
1215
|
+
task.waiters.push(done);
|
|
1216
|
+
if (state.forceFailureListeners === undefined) state.forceFailureListeners = [];
|
|
1217
|
+
state.forceFailureListeners.push(failed);
|
|
1218
|
+
});
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
private async awaitWindowsForceBeforeTerminal(task: BgTask): Promise<Error | undefined> {
|
|
1222
|
+
const state = this.windowsKillStates.get(task);
|
|
1223
|
+
if (state === undefined) return undefined;
|
|
1224
|
+
const forcePromise = state.forcePromise;
|
|
1225
|
+
if (forcePromise === undefined) return state.forceFailure;
|
|
1226
|
+
try {
|
|
1227
|
+
await forcePromise;
|
|
1228
|
+
} catch (error) {
|
|
1229
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1230
|
+
}
|
|
1231
|
+
return state.forceFailure;
|
|
1232
|
+
}
|
|
1233
|
+
|
|
1623
1234
|
private getWindowsKillState(task: BgTask): WindowsKillState {
|
|
1624
1235
|
let state = this.windowsKillStates.get(task);
|
|
1625
1236
|
if (state === undefined) {
|
|
@@ -1661,26 +1272,14 @@ export class BackgroundTaskRegistry {
|
|
|
1661
1272
|
}
|
|
1662
1273
|
|
|
1663
1274
|
private recordWindowsTaskkillNotice(task: BgTask, message: string): void {
|
|
1664
|
-
this.
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
private recordWindowsSoftFailure(task: BgTask, pid: number, detail: string): void {
|
|
1668
|
-
const message =
|
|
1669
|
-
`Windows taskkill /T logical termination request failed for task ${task.id} pid ${String(pid)}: ` +
|
|
1670
|
-
`${detail}; force escalation remains scheduled`;
|
|
1671
|
-
task.error = BackgroundTaskRegistry.appendTaskError(task.error, message);
|
|
1672
|
-
this.recordWindowsTaskkillNotice(task, message);
|
|
1673
|
-
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1674
|
-
this.logger.error(
|
|
1675
|
-
`[background-tasks] failed to write Windows taskkill soft-failure metadata for ${task.id}:`,
|
|
1676
|
-
metadataError,
|
|
1677
|
-
);
|
|
1678
|
-
});
|
|
1275
|
+
void this.appendNotice(task, `\n[background task Windows termination: ${message}]\n`).catch(
|
|
1276
|
+
() => undefined,
|
|
1277
|
+
);
|
|
1679
1278
|
}
|
|
1680
1279
|
|
|
1681
1280
|
private makeWindowsForceFailure(task: BgTask, pid: number, detail: string): Error {
|
|
1682
1281
|
return new Error(
|
|
1683
|
-
`Windows taskkill /T /F
|
|
1282
|
+
`Windows taskkill /T /F tree termination failed for task ${task.id} pid ${String(pid)}: ${detail}. Descendant processes may have leaked.`,
|
|
1684
1283
|
);
|
|
1685
1284
|
}
|
|
1686
1285
|
|
|
@@ -1705,76 +1304,30 @@ export class BackgroundTaskRegistry {
|
|
|
1705
1304
|
private evaluateWindowsTaskkillOutcome(
|
|
1706
1305
|
task: BgTask,
|
|
1707
1306
|
pid: number,
|
|
1708
|
-
phase: WindowsKillPhase,
|
|
1709
1307
|
outcome: TaskkillOutcome,
|
|
1710
1308
|
): Error | undefined {
|
|
1711
1309
|
if (outcome.exitCode === 0) return undefined;
|
|
1712
1310
|
const detail = BackgroundTaskRegistry.describeTaskkillOutcome(outcome);
|
|
1713
|
-
if (
|
|
1311
|
+
if (this.isWindowsTaskkillTerminalRace(task)) {
|
|
1714
1312
|
this.recordWindowsTaskkillNotice(
|
|
1715
1313
|
task,
|
|
1716
|
-
`taskkill
|
|
1314
|
+
`taskkill finished after the task became terminal for pid ${String(pid)} (${detail}); treating as a terminal race`,
|
|
1717
1315
|
);
|
|
1718
1316
|
return undefined;
|
|
1719
1317
|
}
|
|
1720
|
-
if (
|
|
1318
|
+
if (outcome.exitCode === 128) {
|
|
1319
|
+
// With a single force kill, 128 means taskkill could not find the tree:
|
|
1320
|
+
// an already-exited race.
|
|
1721
1321
|
this.recordWindowsTaskkillNotice(
|
|
1722
1322
|
task,
|
|
1723
|
-
`taskkill
|
|
1323
|
+
`taskkill reported process not found for pid ${String(pid)} (${detail}); treating as an already-exited race`,
|
|
1724
1324
|
);
|
|
1725
1325
|
return undefined;
|
|
1726
1326
|
}
|
|
1727
|
-
if (phase === 'terminate') {
|
|
1728
|
-
this.recordWindowsSoftFailure(task, pid, detail);
|
|
1729
|
-
return undefined;
|
|
1730
|
-
}
|
|
1731
1327
|
return this.makeWindowsForceFailure(task, pid, detail);
|
|
1732
1328
|
}
|
|
1733
1329
|
|
|
1734
|
-
private
|
|
1735
|
-
task: BgTask,
|
|
1736
|
-
pid: number,
|
|
1737
|
-
error: unknown,
|
|
1738
|
-
state: WindowsKillState,
|
|
1739
|
-
): void {
|
|
1740
|
-
const message = BackgroundTaskRegistry.errorMessage(error);
|
|
1741
|
-
if (state.forcePromise !== undefined || this.isWindowsTaskkillTerminalRace(task)) return;
|
|
1742
|
-
this.recordWindowsSoftFailure(task, pid, message);
|
|
1743
|
-
}
|
|
1744
|
-
|
|
1745
|
-
private startWindowsSoftKill(task: BgTask, pid: number): Promise<void> {
|
|
1746
|
-
const state = this.getWindowsKillState(task);
|
|
1747
|
-
if (state.softPromise !== undefined) return state.softPromise;
|
|
1748
|
-
const controller = new AbortController();
|
|
1749
|
-
state.softController = controller;
|
|
1750
|
-
|
|
1751
|
-
let launched: Promise<TaskkillOutcome>;
|
|
1752
|
-
try {
|
|
1753
|
-
launched = this.killTree(pid, 'terminate', controller.signal);
|
|
1754
|
-
} catch (error) {
|
|
1755
|
-
delete state.softController;
|
|
1756
|
-
throw new Error(
|
|
1757
|
-
`Could not kill task ${task.id}: Windows taskkill /T failed to start: ${BackgroundTaskRegistry.errorMessage(error)}`,
|
|
1758
|
-
);
|
|
1759
|
-
}
|
|
1760
|
-
|
|
1761
|
-
const promise = launched
|
|
1762
|
-
.then((outcome) => {
|
|
1763
|
-
if (state.forcePromise !== undefined || this.isWindowsTaskkillTerminalRace(task)) return;
|
|
1764
|
-
const failure = this.evaluateWindowsTaskkillOutcome(task, pid, 'terminate', outcome);
|
|
1765
|
-
if (failure !== undefined) throw failure;
|
|
1766
|
-
})
|
|
1767
|
-
.catch((error: unknown) => {
|
|
1768
|
-
this.handleWindowsSoftException(task, pid, error, state);
|
|
1769
|
-
})
|
|
1770
|
-
.finally(() => {
|
|
1771
|
-
if (state.softController === controller) delete state.softController;
|
|
1772
|
-
});
|
|
1773
|
-
state.softPromise = promise;
|
|
1774
|
-
return promise;
|
|
1775
|
-
}
|
|
1776
|
-
|
|
1777
|
-
private startWindowsForceKill(task: BgTask, pid: number): Promise<void> {
|
|
1330
|
+
private startWindowsTaskkill(task: BgTask, pid: number): Promise<void> {
|
|
1778
1331
|
const state = this.getWindowsKillState(task);
|
|
1779
1332
|
if (state.forcePromise !== undefined) return state.forcePromise;
|
|
1780
1333
|
|
|
@@ -1792,19 +1345,23 @@ export class BackgroundTaskRegistry {
|
|
|
1792
1345
|
state.forcePromise = forcePromise;
|
|
1793
1346
|
void forcePromise.catch((error: unknown) => {
|
|
1794
1347
|
this.logger.error(
|
|
1795
|
-
`[background-tasks] Windows
|
|
1348
|
+
`[background-tasks] Windows tree termination failed for ${task.id}:`,
|
|
1796
1349
|
error,
|
|
1797
1350
|
);
|
|
1798
1351
|
});
|
|
1799
1352
|
|
|
1800
1353
|
this.clearKillEscalationTimer(task);
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1354
|
+
|
|
1355
|
+
const afterTreeStop = (): void => {
|
|
1356
|
+
// Terminalize without waiting for the next poll tick.
|
|
1357
|
+
if (task.status === 'running' && task.finalized !== true) {
|
|
1358
|
+
void this.pollTask(task);
|
|
1359
|
+
}
|
|
1360
|
+
};
|
|
1804
1361
|
|
|
1805
1362
|
let launched: Promise<TaskkillOutcome>;
|
|
1806
1363
|
try {
|
|
1807
|
-
launched = this.killTree(pid
|
|
1364
|
+
launched = this.killTree(pid);
|
|
1808
1365
|
} catch (error) {
|
|
1809
1366
|
const failure = this.makeWindowsForceFailure(
|
|
1810
1367
|
task,
|
|
@@ -1819,19 +1376,20 @@ export class BackgroundTaskRegistry {
|
|
|
1819
1376
|
|
|
1820
1377
|
launched.then(
|
|
1821
1378
|
(outcome) => {
|
|
1822
|
-
const failure = this.evaluateWindowsTaskkillOutcome(task, pid,
|
|
1379
|
+
const failure = this.evaluateWindowsTaskkillOutcome(task, pid, outcome);
|
|
1823
1380
|
if (failure !== undefined) {
|
|
1824
1381
|
this.recordWindowsForceFailure(task, failure);
|
|
1825
1382
|
rejectForceReady(failure);
|
|
1826
1383
|
return;
|
|
1827
1384
|
}
|
|
1828
1385
|
resolveForceReady();
|
|
1386
|
+
afterTreeStop();
|
|
1829
1387
|
},
|
|
1830
1388
|
(error: unknown) => {
|
|
1831
1389
|
if (this.isWindowsTaskkillTerminalRace(task)) {
|
|
1832
1390
|
this.recordWindowsTaskkillNotice(
|
|
1833
1391
|
task,
|
|
1834
|
-
`taskkill
|
|
1392
|
+
`taskkill rejected after the task became terminal for pid ${String(pid)} (${BackgroundTaskRegistry.errorMessage(error)}); treating as a terminal race`,
|
|
1835
1393
|
);
|
|
1836
1394
|
resolveForceReady();
|
|
1837
1395
|
return;
|
|
@@ -1849,199 +1407,42 @@ export class BackgroundTaskRegistry {
|
|
|
1849
1407
|
return forcePromise;
|
|
1850
1408
|
}
|
|
1851
1409
|
|
|
1852
|
-
private requestWindowsKill(task: BgTask, pid: number, signal: NodeJS.Signals): void {
|
|
1853
|
-
if (signal === 'SIGKILL') {
|
|
1854
|
-
this.startWindowsForceKill(task, pid);
|
|
1855
|
-
task.killSignalSent = true;
|
|
1856
|
-
return;
|
|
1857
|
-
}
|
|
1858
|
-
|
|
1859
|
-
this.startWindowsSoftKill(task, pid);
|
|
1860
|
-
task.killSignalSent = true;
|
|
1861
|
-
if (task.killEscalationTimer !== undefined) return;
|
|
1862
|
-
task.killEscalationTimer = setTimeout(() => {
|
|
1863
|
-
task.killEscalationTimer = undefined;
|
|
1864
|
-
if (task.status !== 'running') return;
|
|
1865
|
-
try {
|
|
1866
|
-
this.requestKill(task, 'SIGKILL');
|
|
1867
|
-
} catch (error) {
|
|
1868
|
-
task.error = BackgroundTaskRegistry.appendTaskError(
|
|
1869
|
-
task.error,
|
|
1870
|
-
`SIGKILL failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1871
|
-
);
|
|
1872
|
-
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1873
|
-
this.logger.error(
|
|
1874
|
-
`[background-tasks] failed to write metadata for ${task.id}:`,
|
|
1875
|
-
metadataError,
|
|
1876
|
-
);
|
|
1877
|
-
});
|
|
1878
|
-
}
|
|
1879
|
-
}, this.killGraceMs).unref();
|
|
1880
|
-
}
|
|
1881
|
-
|
|
1882
1410
|
private requestKill(task: BgTask, signal: NodeJS.Signals = 'SIGTERM'): void {
|
|
1883
1411
|
if (task.status !== 'running') {
|
|
1884
1412
|
throw new Error(`Task ${task.id} is ${task.status}, not running`);
|
|
1885
1413
|
}
|
|
1886
|
-
if (
|
|
1887
|
-
throw new Error(`Task ${task.id} has no child process handle`);
|
|
1888
|
-
}
|
|
1889
|
-
if (task.killSignalSent && signal === 'SIGTERM') return;
|
|
1414
|
+
if (task.killSignalSent) return;
|
|
1890
1415
|
|
|
1891
1416
|
if (this.platform === 'win32') {
|
|
1892
1417
|
if (!task.pid) throw new Error(`Task ${task.id} has no process id`);
|
|
1893
|
-
|
|
1418
|
+
// One Windows kill: `taskkill /T /F`. Console applications do not honor
|
|
1419
|
+
// window-close termination, so there is nothing for a graceful phase to
|
|
1420
|
+
// contain and no escalation ladder to arm.
|
|
1421
|
+
task.killSignalSent = true;
|
|
1422
|
+
this.startWindowsTaskkill(task, task.pid);
|
|
1894
1423
|
return;
|
|
1895
1424
|
}
|
|
1896
1425
|
|
|
1897
1426
|
this.requestPosixKill(task, signal);
|
|
1898
1427
|
}
|
|
1899
1428
|
|
|
1900
|
-
|
|
1901
|
-
if (task.status !== 'running') return Promise.resolve(true);
|
|
1902
|
-
return new Promise((resolve) => {
|
|
1903
|
-
const timeout = setTimeout(() => {
|
|
1904
|
-
const idx = task.waiters.indexOf(done);
|
|
1905
|
-
if (idx >= 0) task.waiters.splice(idx, 1);
|
|
1906
|
-
resolve(false);
|
|
1907
|
-
}, timeoutMs);
|
|
1908
|
-
const done = () => {
|
|
1909
|
-
clearTimeout(timeout);
|
|
1910
|
-
resolve(true);
|
|
1911
|
-
};
|
|
1912
|
-
task.waiters.push(done);
|
|
1913
|
-
});
|
|
1914
|
-
}
|
|
1915
|
-
|
|
1916
|
-
private waitForEndOrWindowsForceFailure(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
1917
|
-
const state = this.getWindowsKillState(task);
|
|
1918
|
-
if (state.forceFailure !== undefined) return Promise.reject(state.forceFailure);
|
|
1919
|
-
if (task.status !== 'running') return Promise.resolve(true);
|
|
1920
|
-
return new Promise((resolve, reject) => {
|
|
1921
|
-
const cleanup = () => {
|
|
1922
|
-
clearTimeout(timeout);
|
|
1923
|
-
const waiterIndex = task.waiters.indexOf(done);
|
|
1924
|
-
if (waiterIndex >= 0) task.waiters.splice(waiterIndex, 1);
|
|
1925
|
-
const listeners = state.forceFailureListeners;
|
|
1926
|
-
if (listeners !== undefined) {
|
|
1927
|
-
const listenerIndex = listeners.indexOf(failed);
|
|
1928
|
-
if (listenerIndex >= 0) listeners.splice(listenerIndex, 1);
|
|
1929
|
-
if (listeners.length === 0) delete state.forceFailureListeners;
|
|
1930
|
-
}
|
|
1931
|
-
};
|
|
1932
|
-
const timeout = setTimeout(() => {
|
|
1933
|
-
cleanup();
|
|
1934
|
-
resolve(false);
|
|
1935
|
-
}, timeoutMs);
|
|
1936
|
-
const done = () => {
|
|
1937
|
-
cleanup();
|
|
1938
|
-
resolve(true);
|
|
1939
|
-
};
|
|
1940
|
-
const failed = (error: Error) => {
|
|
1941
|
-
cleanup();
|
|
1942
|
-
reject(error);
|
|
1943
|
-
};
|
|
1944
|
-
task.waiters.push(done);
|
|
1945
|
-
if (state.forceFailureListeners === undefined) state.forceFailureListeners = [];
|
|
1946
|
-
state.forceFailureListeners.push(failed);
|
|
1947
|
-
});
|
|
1948
|
-
}
|
|
1429
|
+
// ---------- metadata ----------
|
|
1949
1430
|
|
|
1950
|
-
private async
|
|
1951
|
-
|
|
1952
|
-
if (state === undefined) return undefined;
|
|
1953
|
-
const forcePromise = state.forcePromise;
|
|
1954
|
-
if (forcePromise === undefined) return state.forceFailure;
|
|
1955
|
-
try {
|
|
1956
|
-
await forcePromise;
|
|
1957
|
-
} catch (error) {
|
|
1958
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
1959
|
-
}
|
|
1960
|
-
return state.forceFailure;
|
|
1431
|
+
private async writeMetadata(task: BgTask): Promise<void> {
|
|
1432
|
+
await this.writeMetadataSnapshot(task, snapshot(task));
|
|
1961
1433
|
}
|
|
1962
1434
|
|
|
1963
|
-
private async
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
const
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
}
|
|
1972
|
-
task.reloadHostDeliveryInFlight = true;
|
|
1973
|
-
try {
|
|
1974
|
-
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
1975
|
-
this.publishTerminal(task);
|
|
1976
|
-
const deliveryGate = await this.waitForTerminalPublicationGate(task);
|
|
1977
|
-
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
1978
|
-
if (deliveryGate.kind === 'rejected') {
|
|
1979
|
-
this.logger.error(
|
|
1980
|
-
`[background-tasks] completion delivery gate failed for ${task.id}: ${this.terminalPublicationError(deliveryGate.error)}`,
|
|
1981
|
-
);
|
|
1982
|
-
} else if (
|
|
1983
|
-
task.notifyOnCompletion &&
|
|
1984
|
-
!task.notified &&
|
|
1985
|
-
!this.shuttingDown &&
|
|
1986
|
-
execution.notificationState === 'pending'
|
|
1987
|
-
) {
|
|
1988
|
-
const token = execution.beginNotification(lease);
|
|
1989
|
-
if (token !== undefined) {
|
|
1990
|
-
try {
|
|
1991
|
-
this.notifyCompletion(task);
|
|
1992
|
-
execution.finishNotification(token, task.notified);
|
|
1993
|
-
} catch (error) {
|
|
1994
|
-
execution.finishNotification(token, false);
|
|
1995
|
-
this.logger.error(`[background-tasks] notification failed for ${task.id}:`, error);
|
|
1996
|
-
}
|
|
1997
|
-
}
|
|
1998
|
-
}
|
|
1999
|
-
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
2000
|
-
task.reloadHostNotificationSettled = true;
|
|
2001
|
-
task.reloadHostDeliverySettled = true;
|
|
2002
|
-
try {
|
|
2003
|
-
await this.writeMetadata(task);
|
|
2004
|
-
} catch (error) {
|
|
2005
|
-
this.logger.error(
|
|
2006
|
-
`[background-tasks] failed to update survivor notification metadata for ${task.id}:`,
|
|
2007
|
-
error,
|
|
2008
|
-
);
|
|
2009
|
-
}
|
|
2010
|
-
} finally {
|
|
2011
|
-
task.reloadHostDeliveryInFlight = false;
|
|
2012
|
-
this.maybeReleaseReloadExecution(task);
|
|
2013
|
-
}
|
|
1435
|
+
private async writeMetadataSnapshot(task: BgTask, value: BgTaskSnapshot): Promise<void> {
|
|
1436
|
+
const write = async () => {
|
|
1437
|
+
await writeJsonAtomic(task.metadataAbsPath, value);
|
|
1438
|
+
};
|
|
1439
|
+
const previous = task.metadataWriteChain ?? Promise.resolve();
|
|
1440
|
+
const next = previous.then(write, write);
|
|
1441
|
+
task.metadataWriteChain = next.catch(() => undefined);
|
|
1442
|
+
await next;
|
|
2014
1443
|
}
|
|
2015
1444
|
|
|
2016
|
-
|
|
2017
|
-
const execution = task.reloadExecution;
|
|
2018
|
-
const lease = this.reloadShellLease;
|
|
2019
|
-
if (
|
|
2020
|
-
execution === undefined ||
|
|
2021
|
-
execution.phase !== 'terminal' ||
|
|
2022
|
-
task.reloadHostNotificationSettled !== true ||
|
|
2023
|
-
task.terminalPublicationState === 'pending' ||
|
|
2024
|
-
!this.ownsReloadExecution(execution, lease) ||
|
|
2025
|
-
this.reloadShellOwner === undefined ||
|
|
2026
|
-
lease === undefined
|
|
2027
|
-
) {
|
|
2028
|
-
return;
|
|
2029
|
-
}
|
|
2030
|
-
try {
|
|
2031
|
-
this.reloadShellOwner.releaseExecution(lease, execution);
|
|
2032
|
-
} catch (error) {
|
|
2033
|
-
if (
|
|
2034
|
-
typeof error !== 'object' ||
|
|
2035
|
-
error === null ||
|
|
2036
|
-
Reflect.get(error, 'code') !== 'pi_bg_reload_owner_stale_claim'
|
|
2037
|
-
) {
|
|
2038
|
-
this.logger.error(
|
|
2039
|
-
`[background-tasks] failed to release reload shell execution ${task.id}:`,
|
|
2040
|
-
error,
|
|
2041
|
-
);
|
|
2042
|
-
}
|
|
2043
|
-
}
|
|
2044
|
-
}
|
|
1445
|
+
// ---------- terminal publication and notification ----------
|
|
2045
1446
|
|
|
2046
1447
|
private terminalPublicationAbandonSignal(task: BgTask): TerminalPublicationAbandonSignal {
|
|
2047
1448
|
const existing = this.terminalPublicationAbandonSignals.get(task);
|
|
@@ -2056,7 +1457,7 @@ export class BackgroundTaskRegistry {
|
|
|
2056
1457
|
}
|
|
2057
1458
|
|
|
2058
1459
|
private publishTerminal(task: BgTask): void {
|
|
2059
|
-
if (
|
|
1460
|
+
if (this.tasks.get(task.id) !== task) return;
|
|
2060
1461
|
if (task.terminalPublicationState !== 'pending' || task.terminalPublishInFlight) return;
|
|
2061
1462
|
if (this.terminalPublicationClosed) {
|
|
2062
1463
|
this.abandonTerminalPublication(
|
|
@@ -2075,7 +1476,7 @@ export class BackgroundTaskRegistry {
|
|
|
2075
1476
|
|
|
2076
1477
|
private async publishTerminalWhenReady(task: BgTask): Promise<void> {
|
|
2077
1478
|
const outcome = await this.waitForTerminalPublicationGate(task);
|
|
2078
|
-
if (
|
|
1479
|
+
if (this.tasks.get(task.id) !== task) {
|
|
2079
1480
|
task.terminalPublishInFlight = false;
|
|
2080
1481
|
return;
|
|
2081
1482
|
}
|
|
@@ -2087,9 +1488,6 @@ export class BackgroundTaskRegistry {
|
|
|
2087
1488
|
this.abandonTerminalPublication(task, 'gate_rejected', outcome.error);
|
|
2088
1489
|
return;
|
|
2089
1490
|
}
|
|
2090
|
-
// The gate and registry closure can settle in the same microtask turn.
|
|
2091
|
-
// Re-check after the await so a late gate cannot publish into a disposed
|
|
2092
|
-
// activation or revive a task already abandoned by shutdown.
|
|
2093
1491
|
if (this.terminalPublicationClosed || task.terminalPublicationState !== 'pending') {
|
|
2094
1492
|
this.abandonTerminalPublication(
|
|
2095
1493
|
task,
|
|
@@ -2134,8 +1532,6 @@ export class BackgroundTaskRegistry {
|
|
|
2134
1532
|
}));
|
|
2135
1533
|
const outcome = await Promise.race([gateOutcome, closureOutcome, abandonmentOutcome]);
|
|
2136
1534
|
|
|
2137
|
-
// Closure wins if it happened before this continuation resumed, regardless
|
|
2138
|
-
// of which promise queued its reaction first.
|
|
2139
1535
|
if (this.terminalPublicationClosed) {
|
|
2140
1536
|
return {
|
|
2141
1537
|
kind: 'closed',
|
|
@@ -2146,7 +1542,7 @@ export class BackgroundTaskRegistry {
|
|
|
2146
1542
|
}
|
|
2147
1543
|
|
|
2148
1544
|
private tryPublishTerminalNow(task: BgTask): void {
|
|
2149
|
-
if (
|
|
1545
|
+
if (this.tasks.get(task.id) !== task) {
|
|
2150
1546
|
task.terminalPublishInFlight = false;
|
|
2151
1547
|
return;
|
|
2152
1548
|
}
|
|
@@ -2178,13 +1574,6 @@ export class BackgroundTaskRegistry {
|
|
|
2178
1574
|
task.terminalPublishInFlight = false;
|
|
2179
1575
|
|
|
2180
1576
|
if (emitFailed) {
|
|
2181
|
-
const execution = task.reloadExecution;
|
|
2182
|
-
if (execution !== undefined && !this.ownsReloadExecution(execution, this.reloadShellLease)) {
|
|
2183
|
-
// The emitter synchronously detached this task into a reload handoff
|
|
2184
|
-
// before throwing. Attempt 1 is consumed, but only the fresh owner may
|
|
2185
|
-
// retry or decide abandonment on the shared publication ledger.
|
|
2186
|
-
return;
|
|
2187
|
-
}
|
|
2188
1577
|
this.handleTerminalPublishFailure(task, emitError);
|
|
2189
1578
|
} else {
|
|
2190
1579
|
this.markTerminalPublicationDelivered(task);
|
|
@@ -2203,7 +1592,6 @@ export class BackgroundTaskRegistry {
|
|
|
2203
1592
|
task.terminalPublicationState = 'delivered';
|
|
2204
1593
|
delete task.terminalPublicationAbandonReason;
|
|
2205
1594
|
task.terminalPublished = true;
|
|
2206
|
-
this.maybeReleaseReloadExecution(task);
|
|
2207
1595
|
}
|
|
2208
1596
|
|
|
2209
1597
|
private abandonTerminalPublication(
|
|
@@ -2231,7 +1619,6 @@ export class BackgroundTaskRegistry {
|
|
|
2231
1619
|
`[background-tasks] terminal publication abandoned for ${task.id} (${reason}) after ${String(task.terminalPublishAttempts)}/${String(TERMINAL_PUBLICATION_MAX_ATTEMPTS)} emit attempts${detail}`,
|
|
2232
1620
|
);
|
|
2233
1621
|
}
|
|
2234
|
-
this.maybeReleaseReloadExecution(task);
|
|
2235
1622
|
}
|
|
2236
1623
|
|
|
2237
1624
|
private terminalPublicationError(error: unknown): string {
|
|
@@ -2271,9 +1658,10 @@ export class BackgroundTaskRegistry {
|
|
|
2271
1658
|
if (
|
|
2272
1659
|
this.terminalPublicationClosed ||
|
|
2273
1660
|
task.terminalPublicationState !== 'pending' ||
|
|
2274
|
-
|
|
2275
|
-
)
|
|
1661
|
+
this.tasks.get(task.id) !== task
|
|
1662
|
+
) {
|
|
2276
1663
|
return;
|
|
1664
|
+
}
|
|
2277
1665
|
this.publishTerminal(task);
|
|
2278
1666
|
}, TERMINAL_PUBLICATION_RETRY_MS);
|
|
2279
1667
|
task.terminalPublishRetryHandle.unref();
|
|
@@ -2330,8 +1718,7 @@ export class BackgroundTaskRegistry {
|
|
|
2330
1718
|
): Promise<void> {
|
|
2331
1719
|
if (task.finalized) return;
|
|
2332
1720
|
task.finalized = true;
|
|
2333
|
-
|
|
2334
|
-
if (this.platform === 'win32') this.clearKillEscalationTimer(task);
|
|
1721
|
+
this.clearKillEscalationTimer(task);
|
|
2335
1722
|
let finalStatus = status;
|
|
2336
1723
|
let finalError = error;
|
|
2337
1724
|
const posixForceFailure = await this.awaitPosixProcessGroupBeforeTerminal(task);
|
|
@@ -2347,15 +1734,11 @@ export class BackgroundTaskRegistry {
|
|
|
2347
1734
|
task.exitCode = exitCode;
|
|
2348
1735
|
task.signal = signal ?? null;
|
|
2349
1736
|
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
finalizeError instanceof Error ? finalizeError.message : String(finalizeError);
|
|
2356
|
-
finalError = finalError
|
|
2357
|
-
? `${finalError}; final output durability failed: ${message}`
|
|
2358
|
-
: `Final output durability failed: ${message}`;
|
|
1737
|
+
// The runner script is disposable once the child is terminal.
|
|
1738
|
+
if (task.runnerScriptFile !== undefined) {
|
|
1739
|
+
const runner = task.runnerScriptFile;
|
|
1740
|
+
await rm(runner, { force: true }).catch(() => undefined);
|
|
1741
|
+
task.runnerScriptFile = undefined;
|
|
2359
1742
|
}
|
|
2360
1743
|
|
|
2361
1744
|
task.endTime = this.now();
|