pi-background-tasks 0.7.3 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PUBLISHING.md +10 -7
- package/README.md +78 -6
- package/TESTING.md +13 -3
- package/TEST_PLAN.md +10 -5
- package/package.json +10 -8
- package/src/core/attested-pi-run.ts +19 -35
- package/src/core/common.ts +124 -8
- package/src/core/durable-fs.ts +400 -0
- package/src/core/fusion/artifacts.ts +20 -36
- package/src/core/fusion/budget.ts +372 -0
- package/src/core/fusion/config.ts +4 -36
- package/src/core/fusion/context.ts +508 -47
- package/src/core/fusion/orchestrator.ts +41 -4
- package/src/core/fusion/pi-child.ts +28 -6
- package/src/core/fusion/prompts.ts +21 -7
- package/src/core/fusion/types.ts +258 -4
- package/src/core/pi-launch.ts +225 -0
- package/src/core/registry.ts +507 -56
- package/src/core/windows-taskkill.ts +250 -0
- package/src/fusion-extension.ts +4 -1
package/src/core/registry.ts
CHANGED
|
@@ -50,12 +50,25 @@ import {
|
|
|
50
50
|
writeFileFsynced,
|
|
51
51
|
writeJsonAtomic,
|
|
52
52
|
} from './attested-pi-run.js';
|
|
53
|
+
import {
|
|
54
|
+
assertWindowsCommandLineWithinLimit,
|
|
55
|
+
resolvePiLaunch,
|
|
56
|
+
type PiLaunchSpec,
|
|
57
|
+
} from './pi-launch.js';
|
|
58
|
+
import {
|
|
59
|
+
runWindowsTaskkill,
|
|
60
|
+
type TaskkillOutcome,
|
|
61
|
+
type WindowsKillPhase,
|
|
62
|
+
type WindowsTaskkillOptions,
|
|
63
|
+
} from './windows-taskkill.js';
|
|
53
64
|
|
|
54
65
|
export const MAX_OUTPUT_BYTES = Number(process.env['PI_BG_MAX_OUTPUT_BYTES'] ?? 20 * 1024 * 1024);
|
|
55
66
|
export const KILL_GRACE_MS = 3000;
|
|
56
67
|
export const STOP_WAIT_MS = KILL_GRACE_MS + 1500;
|
|
57
68
|
export const MAX_RECENT_TASKS = 100;
|
|
58
69
|
const TELEMETRY_BUFFER_CHARS = 512 * 1024;
|
|
70
|
+
export const WIN32_CMD_PI_TELEMETRY_UNAVAILABLE_REASON =
|
|
71
|
+
'win32-cmd-cannot-safely-intercept-pi-argv';
|
|
59
72
|
|
|
60
73
|
export interface BackgroundTaskModelRegistry
|
|
61
74
|
extends Pick<ExtensionContext['modelRegistry'], 'getAll'> {
|
|
@@ -93,6 +106,19 @@ export type BackgroundTaskSpawn = (
|
|
|
93
106
|
) => BackgroundTaskChildProcess;
|
|
94
107
|
|
|
95
108
|
type KillProcessFn = (pid: number, signal?: NodeJS.Signals | number) => boolean;
|
|
109
|
+
type KillTreeFn = (
|
|
110
|
+
pid: number,
|
|
111
|
+
phase: WindowsKillPhase,
|
|
112
|
+
signal?: AbortSignal,
|
|
113
|
+
) => Promise<TaskkillOutcome>;
|
|
114
|
+
|
|
115
|
+
interface WindowsKillState {
|
|
116
|
+
softController?: AbortController | undefined;
|
|
117
|
+
softPromise?: Promise<void> | undefined;
|
|
118
|
+
forcePromise?: Promise<void> | undefined;
|
|
119
|
+
forceFailure?: Error | undefined;
|
|
120
|
+
forceFailureListeners?: Array<(error: Error) => void> | undefined;
|
|
121
|
+
}
|
|
96
122
|
|
|
97
123
|
export interface CompletionNotificationMessage {
|
|
98
124
|
customType: 'background-task-notification';
|
|
@@ -117,6 +143,7 @@ export interface BackgroundTaskRegistryOptions {
|
|
|
117
143
|
publishTerminal?: (task: BgTaskSnapshot) => void;
|
|
118
144
|
spawn?: BackgroundTaskSpawn;
|
|
119
145
|
killProcess?: KillProcessFn;
|
|
146
|
+
killTree?: KillTreeFn;
|
|
120
147
|
platform?: NodeJS.Platform;
|
|
121
148
|
env?: NodeJS.ProcessEnv;
|
|
122
149
|
makeTaskId?: () => string;
|
|
@@ -196,10 +223,15 @@ export function buildModelWindowIndex(
|
|
|
196
223
|
};
|
|
197
224
|
}
|
|
198
225
|
|
|
199
|
-
export function createPiTelemetryWrapperSource(
|
|
226
|
+
export function createPiTelemetryWrapperSource(
|
|
227
|
+
index: ModelWindowIndex,
|
|
228
|
+
launch: PiLaunchSpec = resolvePiLaunch(),
|
|
229
|
+
): string {
|
|
200
230
|
return `#!/usr/bin/env node
|
|
201
231
|
const { spawn } = require("node:child_process");
|
|
202
232
|
const index = ${JSON.stringify(index)};
|
|
233
|
+
const launch = ${JSON.stringify(launch)};
|
|
234
|
+
const WINDOWS_COMMAND_LINE_LIMIT = 32767;
|
|
203
235
|
|
|
204
236
|
const tokenUsage = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0 };
|
|
205
237
|
let costTotal = 0;
|
|
@@ -270,6 +302,42 @@ function currentToolUsage() {
|
|
|
270
302
|
return { total: toolUsage.total, failed: toolUsage.failed, byName: { ...toolUsage.byName } };
|
|
271
303
|
}
|
|
272
304
|
|
|
305
|
+
function renderWindowsArgument(value) {
|
|
306
|
+
if (value.length > 0 && !/[ \\t\"]/.test(value)) return value;
|
|
307
|
+
let rendered = "\\\"";
|
|
308
|
+
let backslashes = 0;
|
|
309
|
+
for (const char of value) {
|
|
310
|
+
if (char === "\\\\") {
|
|
311
|
+
backslashes += 1;
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
if (char === "\\\"") {
|
|
315
|
+
rendered += "\\\\".repeat(backslashes * 2 + 1);
|
|
316
|
+
rendered += "\\\"";
|
|
317
|
+
backslashes = 0;
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
if (backslashes > 0) {
|
|
321
|
+
rendered += "\\\\".repeat(backslashes);
|
|
322
|
+
backslashes = 0;
|
|
323
|
+
}
|
|
324
|
+
rendered += char;
|
|
325
|
+
}
|
|
326
|
+
if (backslashes > 0) rendered += "\\\\".repeat(backslashes * 2);
|
|
327
|
+
rendered += "\\\"";
|
|
328
|
+
return rendered;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function assertWindowsLimit(stage, args) {
|
|
332
|
+
if (process.platform !== "win32") return;
|
|
333
|
+
const measured = [launch.executable, ...launch.argvPrefix, ...args].map(renderWindowsArgument).join(" ").length + 1;
|
|
334
|
+
if (measured > WINDOWS_COMMAND_LINE_LIMIT) {
|
|
335
|
+
const error = new Error("pi_command_line_too_long: " + stage + " measured UTF-16 command line length " + String(measured) + " exceeds limit " + String(WINDOWS_COMMAND_LINE_LIMIT));
|
|
336
|
+
error.code = "pi_command_line_too_long";
|
|
337
|
+
throw error;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
273
341
|
function emitUnifiedTelemetry(payload) {
|
|
274
342
|
const out = { type: "background-task-telemetry", ...payload };
|
|
275
343
|
const tokens = currentTokenUsage();
|
|
@@ -407,34 +475,45 @@ function emitToolTelemetry() {
|
|
|
407
475
|
}
|
|
408
476
|
|
|
409
477
|
const parsed = parseInvocation(process.argv.slice(2));
|
|
410
|
-
|
|
478
|
+
let child;
|
|
411
479
|
let buffer = "";
|
|
480
|
+
try {
|
|
481
|
+
const childArgs = [...launch.argvPrefix, ...parsed.args];
|
|
482
|
+
assertWindowsLimit("telemetry-wrapper-pi", parsed.args);
|
|
483
|
+
child = spawn(launch.executable, childArgs, { stdio: ["ignore", "pipe", "pipe"], env: process.env, shell: false, windowsHide: true });
|
|
484
|
+
} catch (error) {
|
|
485
|
+
const message = error && typeof error.message === "string" ? error.message : String(error);
|
|
486
|
+
process.stderr.write("[pi-bg telemetry wrapper error: " + message + "]\\n");
|
|
487
|
+
process.exitCode = 1;
|
|
488
|
+
}
|
|
412
489
|
|
|
413
|
-
if (
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
490
|
+
if (child) {
|
|
491
|
+
if (!parsed.parseJson) {
|
|
492
|
+
child.stdout.pipe(process.stdout);
|
|
493
|
+
} else {
|
|
494
|
+
child.stdout.on("data", (chunk) => {
|
|
495
|
+
buffer += chunk.toString();
|
|
496
|
+
const lines = buffer.split("\\n");
|
|
497
|
+
buffer = lines.pop() || "";
|
|
498
|
+
for (const line of lines) processLine(line);
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
child.stderr.on("data", (chunk) => process.stderr.write(chunk));
|
|
502
|
+
child.on("error", (error) => {
|
|
503
|
+
process.stderr.write("[pi-bg telemetry wrapper error: " + error.message + "]\\n");
|
|
421
504
|
});
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
// pipe; signal termination is deferred through the same stdout barrier.
|
|
433
|
-
process.stdout.write("", () => {
|
|
434
|
-
if (signal) process.kill(process.pid, signal);
|
|
435
|
-
else process.exitCode = code ?? 0;
|
|
505
|
+
child.on("close", (code, signal) => {
|
|
506
|
+
if (parsed.parseJson && buffer.trim()) processLine(buffer);
|
|
507
|
+
// Never call process.exit() here: the final message telemetry may still be
|
|
508
|
+
// buffered on wrapper stdout, and forced exit can publish a stale context
|
|
509
|
+
// snapshot from the preceding assistant turn. exitCode lets Node drain the
|
|
510
|
+
// pipe; signal termination is deferred through the same stdout barrier.
|
|
511
|
+
process.stdout.write("", () => {
|
|
512
|
+
if (signal) process.kill(process.pid, signal);
|
|
513
|
+
else process.exitCode = code ?? 0;
|
|
514
|
+
});
|
|
436
515
|
});
|
|
437
|
-
}
|
|
516
|
+
}
|
|
438
517
|
|
|
439
518
|
function processLine(line) {
|
|
440
519
|
if (!line.trim()) return;
|
|
@@ -605,6 +684,7 @@ export class BackgroundTaskRegistry {
|
|
|
605
684
|
private shuttingDown = false;
|
|
606
685
|
private readonly spawn: BackgroundTaskSpawn;
|
|
607
686
|
private readonly killProcess: KillProcessFn;
|
|
687
|
+
private readonly killTree: KillTreeFn;
|
|
608
688
|
private readonly platform: NodeJS.Platform;
|
|
609
689
|
private readonly env: NodeJS.ProcessEnv;
|
|
610
690
|
private readonly makeTaskIdFn: () => string;
|
|
@@ -617,6 +697,7 @@ export class BackgroundTaskRegistry {
|
|
|
617
697
|
private readonly onChange: () => void;
|
|
618
698
|
private readonly sendCompletionNotification: CompletionNotificationSender;
|
|
619
699
|
private readonly publishTerminalSnapshot: (task: BgTaskSnapshot) => void;
|
|
700
|
+
private readonly windowsKillStates = new WeakMap<BgTask, WindowsKillState>();
|
|
620
701
|
|
|
621
702
|
constructor(options: BackgroundTaskRegistryOptions) {
|
|
622
703
|
this.spawn =
|
|
@@ -624,6 +705,11 @@ export class BackgroundTaskRegistry {
|
|
|
624
705
|
this.killProcess = options.killProcess ?? process.kill.bind(process);
|
|
625
706
|
this.platform = options.platform ?? process.platform;
|
|
626
707
|
this.env = options.env ?? process.env;
|
|
708
|
+
this.killTree = options.killTree ?? ((pid, phase, signal) => {
|
|
709
|
+
const taskkillOptions: WindowsTaskkillOptions =
|
|
710
|
+
signal === undefined ? { env: this.env } : { env: this.env, signal };
|
|
711
|
+
return runWindowsTaskkill(pid, phase, taskkillOptions);
|
|
712
|
+
});
|
|
627
713
|
this.makeTaskIdFn = options.makeTaskId ?? defaultTaskId;
|
|
628
714
|
this.now = options.now ?? Date.now;
|
|
629
715
|
this.maxOutputBytes = options.maxOutputBytes ?? MAX_OUTPUT_BYTES;
|
|
@@ -673,6 +759,14 @@ export class BackgroundTaskRegistry {
|
|
|
673
759
|
if (this.shuttingDown)
|
|
674
760
|
throw new Error('Cannot start a background task while Pi is shutting down');
|
|
675
761
|
|
|
762
|
+
const isAgent = options.isAgent ?? false;
|
|
763
|
+
const baseInvocation = shellInvocation(normalizedCommand, this.platform, this.env);
|
|
764
|
+
const piTelemetryRequested = isAgent && commandMayLaunchPiAgent(normalizedCommand, this.env);
|
|
765
|
+
const piTelemetryLaunch =
|
|
766
|
+
piTelemetryRequested && baseInvocation.dialect === 'posix'
|
|
767
|
+
? resolvePiLaunch({ platform: this.platform })
|
|
768
|
+
: undefined;
|
|
769
|
+
|
|
676
770
|
const dir = await this.ensureRuntimeDir(ctx);
|
|
677
771
|
const id = this.makeTaskIdFn();
|
|
678
772
|
const outputAbsPath = join(dir.abs, `${id}.output`);
|
|
@@ -688,7 +782,6 @@ export class BackgroundTaskRegistry {
|
|
|
688
782
|
normalizeTaskName(options.name) ??
|
|
689
783
|
normalizeTaskName(options.description) ??
|
|
690
784
|
deriveTaskNameFromCommand(normalizedCommand);
|
|
691
|
-
const isAgent = options.isAgent ?? false;
|
|
692
785
|
const trimmedDescription = options.description?.trim();
|
|
693
786
|
const description =
|
|
694
787
|
trimmedDescription && trimmedDescription.length > 0 ? trimmedDescription : undefined;
|
|
@@ -739,23 +832,32 @@ export class BackgroundTaskRegistry {
|
|
|
739
832
|
|
|
740
833
|
try {
|
|
741
834
|
let commandToSpawn = normalizedCommand;
|
|
742
|
-
if (
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
wrapperAbsPath,
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
835
|
+
if (piTelemetryRequested) {
|
|
836
|
+
if (baseInvocation.dialect === 'posix') {
|
|
837
|
+
if (piTelemetryLaunch === undefined) throw new Error('Pi telemetry launch spec was not resolved');
|
|
838
|
+
const wrapperAbsPath = join(dir.abs, `${id}.pi-telemetry-wrapper.cjs`);
|
|
839
|
+
await writeFile(
|
|
840
|
+
wrapperAbsPath,
|
|
841
|
+
createPiTelemetryWrapperSource(buildModelWindowIndex(ctx), piTelemetryLaunch),
|
|
842
|
+
'utf8',
|
|
843
|
+
);
|
|
844
|
+
commandToSpawn = `pi() { ${shellQuote(process.execPath)} ${shellQuote(wrapperAbsPath)} "$@"; }\n${normalizedCommand}`;
|
|
845
|
+
task.telemetryWrapped = true;
|
|
846
|
+
} else {
|
|
847
|
+
task.telemetryUnavailableReason = WIN32_CMD_PI_TELEMETRY_UNAVAILABLE_REASON;
|
|
848
|
+
}
|
|
751
849
|
}
|
|
752
|
-
const invocation =
|
|
850
|
+
const invocation =
|
|
851
|
+
commandToSpawn === normalizedCommand
|
|
852
|
+
? baseInvocation
|
|
853
|
+
: shellInvocation(commandToSpawn, this.platform, this.env);
|
|
753
854
|
const child = this.spawn(invocation.shell, invocation.args, {
|
|
754
855
|
cwd: ctx.cwd,
|
|
755
856
|
detached: this.platform !== 'win32',
|
|
756
857
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
757
858
|
env: this.env,
|
|
758
859
|
windowsHide: true,
|
|
860
|
+
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
|
|
759
861
|
});
|
|
760
862
|
|
|
761
863
|
task.child = child;
|
|
@@ -832,12 +934,20 @@ export class BackgroundTaskRegistry {
|
|
|
832
934
|
if (this.shuttingDown)
|
|
833
935
|
throw new Error('Cannot start an attested Pi task while Pi is shutting down');
|
|
834
936
|
|
|
937
|
+
const argv = buildAttestedPiArgv(request);
|
|
938
|
+
const attestedPiLaunch = resolvePiLaunch({ platform: this.platform });
|
|
939
|
+
assertWindowsCommandLineWithinLimit(
|
|
940
|
+
attestedPiLaunch,
|
|
941
|
+
argv.slice(1),
|
|
942
|
+
this.platform,
|
|
943
|
+
'attested-pi-run',
|
|
944
|
+
);
|
|
945
|
+
|
|
835
946
|
const dir = await this.ensureRuntimeDir(ctx);
|
|
836
947
|
const id = makeAttestedTaskId();
|
|
837
948
|
if (!ATTESTED_TASK_ID_PATTERN.test(id))
|
|
838
949
|
throw new Error('Generated attested task id is invalid');
|
|
839
950
|
const paths = makeAttestedTaskPaths(dir.abs, dir.display, id);
|
|
840
|
-
const argv = buildAttestedPiArgv(request);
|
|
841
951
|
const promptBytes = Buffer.from(request.prompt, 'utf8');
|
|
842
952
|
const reportAbsPath = await resolveReportPath(ctx.cwd, request.reportPath);
|
|
843
953
|
const auth = observePiOAuth(ctx, request.provider, request.model);
|
|
@@ -895,13 +1005,20 @@ export class BackgroundTaskRegistry {
|
|
|
895
1005
|
);
|
|
896
1006
|
await this.writeMetadata(task);
|
|
897
1007
|
|
|
898
|
-
const captured = spawnAndCapturePi(
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
1008
|
+
const captured = spawnAndCapturePi(
|
|
1009
|
+
this.spawn,
|
|
1010
|
+
argv,
|
|
1011
|
+
{
|
|
1012
|
+
cwd: ctx.cwd,
|
|
1013
|
+
detached: this.platform !== 'win32',
|
|
1014
|
+
shell: false,
|
|
1015
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
1016
|
+
env: attestedPiChildEnv(this.env),
|
|
1017
|
+
windowsHide: true,
|
|
1018
|
+
},
|
|
1019
|
+
this.platform,
|
|
1020
|
+
attestedPiLaunch,
|
|
1021
|
+
);
|
|
905
1022
|
task.child = captured.child;
|
|
906
1023
|
task.pid = captured.child.pid;
|
|
907
1024
|
await this.writeMetadata(task);
|
|
@@ -1011,11 +1128,21 @@ export class BackgroundTaskRegistry {
|
|
|
1011
1128
|
if (task.finalized) return;
|
|
1012
1129
|
task.finalized = true;
|
|
1013
1130
|
if (task.timeoutHandle) clearTimeout(task.timeoutHandle);
|
|
1131
|
+
if (task.killEscalationTimer !== undefined) {
|
|
1132
|
+
clearTimeout(task.killEscalationTimer);
|
|
1133
|
+
task.killEscalationTimer = undefined;
|
|
1134
|
+
}
|
|
1014
1135
|
let finalStatus = status;
|
|
1136
|
+
let finalError = error;
|
|
1137
|
+
const forceFailure = await this.awaitWindowsForceBeforeTerminal(task);
|
|
1138
|
+
if (forceFailure !== undefined) {
|
|
1139
|
+
finalStatus = 'failed';
|
|
1140
|
+
finalError = BackgroundTaskRegistry.appendTaskError(finalError, forceFailure.message);
|
|
1141
|
+
}
|
|
1015
1142
|
task.exitCode = exitCode;
|
|
1016
1143
|
task.signal = signal;
|
|
1017
1144
|
task.endTime = this.now();
|
|
1018
|
-
if (
|
|
1145
|
+
if (finalError) task.error = finalError;
|
|
1019
1146
|
|
|
1020
1147
|
const rawEvents = Buffer.concat(stdoutChunks);
|
|
1021
1148
|
const rawStderr = Buffer.concat(stderrChunks);
|
|
@@ -1119,7 +1246,12 @@ export class BackgroundTaskRegistry {
|
|
|
1119
1246
|
task.killKind = kind;
|
|
1120
1247
|
if (reason) task.error = reason;
|
|
1121
1248
|
this.requestKill(task, 'SIGTERM');
|
|
1122
|
-
const stopped =
|
|
1249
|
+
const stopped =
|
|
1250
|
+
this.platform === 'win32'
|
|
1251
|
+
? await this.waitForEndOrWindowsForceFailure(task, this.stopWaitMs)
|
|
1252
|
+
: await this.waitForEnd(task, this.stopWaitMs);
|
|
1253
|
+
const forceFailure = this.windowsKillStates.get(task)?.forceFailure;
|
|
1254
|
+
if (forceFailure !== undefined) throw forceFailure;
|
|
1123
1255
|
if (!stopped) {
|
|
1124
1256
|
throw new Error(
|
|
1125
1257
|
`Task ${task.id} did not exit within ${formatDuration(this.stopWaitMs)} after SIGTERM/SIGKILL`,
|
|
@@ -1401,6 +1533,260 @@ export class BackgroundTaskRegistry {
|
|
|
1401
1533
|
this.writeNotice(task, `${line}\n`);
|
|
1402
1534
|
}
|
|
1403
1535
|
|
|
1536
|
+
private getWindowsKillState(task: BgTask): WindowsKillState {
|
|
1537
|
+
let state = this.windowsKillStates.get(task);
|
|
1538
|
+
if (state === undefined) {
|
|
1539
|
+
state = {};
|
|
1540
|
+
this.windowsKillStates.set(task, state);
|
|
1541
|
+
}
|
|
1542
|
+
return state;
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
private static errorMessage(error: unknown): string {
|
|
1546
|
+
return error instanceof Error ? error.message : String(error);
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
private static appendTaskError(existing: string | undefined, next: string): string {
|
|
1550
|
+
if (existing === undefined || existing.length === 0) return next;
|
|
1551
|
+
if (existing.includes(next)) return existing;
|
|
1552
|
+
return `${existing}; ${next}`;
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
private static describeTaskkillOutcome(outcome: TaskkillOutcome): string {
|
|
1556
|
+
const exitCode = outcome.exitCode === null ? 'null' : String(outcome.exitCode);
|
|
1557
|
+
const signal = outcome.signal === null ? 'null' : outcome.signal;
|
|
1558
|
+
const stdout = outcome.stdout.length > 0 ? ` stdout=${JSON.stringify(outcome.stdout)}` : '';
|
|
1559
|
+
const stderr = outcome.stderr.length > 0 ? ` stderr=${JSON.stringify(outcome.stderr)}` : '';
|
|
1560
|
+
const stdoutTruncated = outcome.stdoutTruncated ? ' stdout_truncated=true' : '';
|
|
1561
|
+
const stderrTruncated = outcome.stderrTruncated ? ' stderr_truncated=true' : '';
|
|
1562
|
+
return `exit=${exitCode} signal=${signal}${stdout}${stderr}${stdoutTruncated}${stderrTruncated}`;
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
private isWindowsTaskkillTerminalRace(task: BgTask): boolean {
|
|
1566
|
+
return task.status !== 'running' || task.finalized === true;
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
private clearKillEscalationTimer(task: BgTask): void {
|
|
1570
|
+
if (task.killEscalationTimer !== undefined) {
|
|
1571
|
+
clearTimeout(task.killEscalationTimer);
|
|
1572
|
+
task.killEscalationTimer = undefined;
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
private recordWindowsTaskkillNotice(task: BgTask, message: string): void {
|
|
1577
|
+
this.writeNotice(task, `\n[background task Windows termination: ${message}]\n`);
|
|
1578
|
+
}
|
|
1579
|
+
|
|
1580
|
+
private recordWindowsSoftFailure(task: BgTask, pid: number, detail: string): void {
|
|
1581
|
+
const message =
|
|
1582
|
+
`Windows taskkill /T logical termination request failed for task ${task.id} pid ${String(pid)}: ` +
|
|
1583
|
+
`${detail}; force escalation remains scheduled`;
|
|
1584
|
+
task.error = BackgroundTaskRegistry.appendTaskError(task.error, message);
|
|
1585
|
+
this.recordWindowsTaskkillNotice(task, message);
|
|
1586
|
+
this.onChange();
|
|
1587
|
+
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1588
|
+
this.logger.error(
|
|
1589
|
+
`[background-tasks] failed to write Windows taskkill soft-failure metadata for ${task.id}:`,
|
|
1590
|
+
metadataError,
|
|
1591
|
+
);
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
private makeWindowsForceFailure(task: BgTask, pid: number, detail: string): Error {
|
|
1596
|
+
return new Error(
|
|
1597
|
+
`Windows taskkill /T /F force termination failed for task ${task.id} pid ${String(pid)}: ${detail}. Descendant processes may have leaked.`,
|
|
1598
|
+
);
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
private recordWindowsForceFailure(task: BgTask, error: Error): void {
|
|
1602
|
+
const state = this.getWindowsKillState(task);
|
|
1603
|
+
state.forceFailure = error;
|
|
1604
|
+
task.error = BackgroundTaskRegistry.appendTaskError(task.error, error.message);
|
|
1605
|
+
this.recordWindowsTaskkillNotice(task, error.message);
|
|
1606
|
+
this.onChange();
|
|
1607
|
+
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1608
|
+
this.logger.error(
|
|
1609
|
+
`[background-tasks] failed to write Windows taskkill force-failure metadata for ${task.id}:`,
|
|
1610
|
+
metadataError,
|
|
1611
|
+
);
|
|
1612
|
+
});
|
|
1613
|
+
const listeners = state.forceFailureListeners;
|
|
1614
|
+
if (listeners !== undefined) {
|
|
1615
|
+
delete state.forceFailureListeners;
|
|
1616
|
+
for (const listener of listeners) listener(error);
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
|
|
1620
|
+
private evaluateWindowsTaskkillOutcome(
|
|
1621
|
+
task: BgTask,
|
|
1622
|
+
pid: number,
|
|
1623
|
+
phase: WindowsKillPhase,
|
|
1624
|
+
outcome: TaskkillOutcome,
|
|
1625
|
+
): Error | undefined {
|
|
1626
|
+
if (outcome.exitCode === 0) return undefined;
|
|
1627
|
+
const detail = BackgroundTaskRegistry.describeTaskkillOutcome(outcome);
|
|
1628
|
+
if (outcome.exitCode === 128) {
|
|
1629
|
+
this.recordWindowsTaskkillNotice(
|
|
1630
|
+
task,
|
|
1631
|
+
`taskkill ${phase} reported process not found for pid ${String(pid)} (${detail}); treating as an already-exited race`,
|
|
1632
|
+
);
|
|
1633
|
+
return undefined;
|
|
1634
|
+
}
|
|
1635
|
+
if (this.isWindowsTaskkillTerminalRace(task)) {
|
|
1636
|
+
this.recordWindowsTaskkillNotice(
|
|
1637
|
+
task,
|
|
1638
|
+
`taskkill ${phase} finished after the task became terminal for pid ${String(pid)} (${detail}); treating as a terminal race`,
|
|
1639
|
+
);
|
|
1640
|
+
return undefined;
|
|
1641
|
+
}
|
|
1642
|
+
if (phase === 'terminate') {
|
|
1643
|
+
this.recordWindowsSoftFailure(task, pid, detail);
|
|
1644
|
+
return undefined;
|
|
1645
|
+
}
|
|
1646
|
+
return this.makeWindowsForceFailure(task, pid, detail);
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
private handleWindowsSoftException(
|
|
1650
|
+
task: BgTask,
|
|
1651
|
+
pid: number,
|
|
1652
|
+
error: unknown,
|
|
1653
|
+
state: WindowsKillState,
|
|
1654
|
+
): void {
|
|
1655
|
+
const message = BackgroundTaskRegistry.errorMessage(error);
|
|
1656
|
+
if (state.forcePromise !== undefined || this.isWindowsTaskkillTerminalRace(task)) return;
|
|
1657
|
+
this.recordWindowsSoftFailure(task, pid, message);
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
private startWindowsSoftKill(task: BgTask, pid: number): Promise<void> {
|
|
1661
|
+
const state = this.getWindowsKillState(task);
|
|
1662
|
+
if (state.softPromise !== undefined) return state.softPromise;
|
|
1663
|
+
const controller = new AbortController();
|
|
1664
|
+
state.softController = controller;
|
|
1665
|
+
|
|
1666
|
+
let launched: Promise<TaskkillOutcome>;
|
|
1667
|
+
try {
|
|
1668
|
+
launched = this.killTree(pid, 'terminate', controller.signal);
|
|
1669
|
+
} catch (error) {
|
|
1670
|
+
delete state.softController;
|
|
1671
|
+
throw new Error(
|
|
1672
|
+
`Could not kill task ${task.id}: Windows taskkill /T failed to start: ${BackgroundTaskRegistry.errorMessage(error)}`,
|
|
1673
|
+
);
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
const promise = launched
|
|
1677
|
+
.then((outcome) => {
|
|
1678
|
+
if (state.forcePromise !== undefined || this.isWindowsTaskkillTerminalRace(task)) return;
|
|
1679
|
+
const failure = this.evaluateWindowsTaskkillOutcome(task, pid, 'terminate', outcome);
|
|
1680
|
+
if (failure !== undefined) throw failure;
|
|
1681
|
+
})
|
|
1682
|
+
.catch((error: unknown) => {
|
|
1683
|
+
this.handleWindowsSoftException(task, pid, error, state);
|
|
1684
|
+
})
|
|
1685
|
+
.finally(() => {
|
|
1686
|
+
if (state.softController === controller) delete state.softController;
|
|
1687
|
+
});
|
|
1688
|
+
state.softPromise = promise;
|
|
1689
|
+
return promise;
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
private startWindowsForceKill(task: BgTask, pid: number): Promise<void> {
|
|
1693
|
+
const state = this.getWindowsKillState(task);
|
|
1694
|
+
if (state.forcePromise !== undefined) return state.forcePromise;
|
|
1695
|
+
|
|
1696
|
+
let resolveForce: (() => void) | undefined;
|
|
1697
|
+
let rejectForce: ((error: unknown) => void) | undefined;
|
|
1698
|
+
const forcePromise = new Promise<void>((resolve, reject) => {
|
|
1699
|
+
resolveForce = resolve;
|
|
1700
|
+
rejectForce = reject;
|
|
1701
|
+
});
|
|
1702
|
+
if (resolveForce === undefined || rejectForce === undefined) {
|
|
1703
|
+
throw new Error('Windows force termination promise could not be initialized');
|
|
1704
|
+
}
|
|
1705
|
+
const resolveForceReady = resolveForce;
|
|
1706
|
+
const rejectForceReady = rejectForce;
|
|
1707
|
+
state.forcePromise = forcePromise;
|
|
1708
|
+
void forcePromise.catch((error: unknown) => {
|
|
1709
|
+
this.logger.error(`[background-tasks] Windows force tree termination failed for ${task.id}:`, error);
|
|
1710
|
+
});
|
|
1711
|
+
|
|
1712
|
+
this.clearKillEscalationTimer(task);
|
|
1713
|
+
if (state.softController !== undefined && !state.softController.signal.aborted) {
|
|
1714
|
+
state.softController.abort();
|
|
1715
|
+
}
|
|
1716
|
+
|
|
1717
|
+
let launched: Promise<TaskkillOutcome>;
|
|
1718
|
+
try {
|
|
1719
|
+
launched = this.killTree(pid, 'force');
|
|
1720
|
+
} catch (error) {
|
|
1721
|
+
const failure = this.makeWindowsForceFailure(
|
|
1722
|
+
task,
|
|
1723
|
+
pid,
|
|
1724
|
+
`helper failed to start: ${BackgroundTaskRegistry.errorMessage(error)}`,
|
|
1725
|
+
);
|
|
1726
|
+
delete state.forcePromise;
|
|
1727
|
+
this.recordWindowsForceFailure(task, failure);
|
|
1728
|
+
rejectForceReady(failure);
|
|
1729
|
+
throw failure;
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
launched.then(
|
|
1733
|
+
(outcome) => {
|
|
1734
|
+
const failure = this.evaluateWindowsTaskkillOutcome(task, pid, 'force', outcome);
|
|
1735
|
+
if (failure !== undefined) {
|
|
1736
|
+
this.recordWindowsForceFailure(task, failure);
|
|
1737
|
+
rejectForceReady(failure);
|
|
1738
|
+
return;
|
|
1739
|
+
}
|
|
1740
|
+
resolveForceReady();
|
|
1741
|
+
},
|
|
1742
|
+
(error: unknown) => {
|
|
1743
|
+
if (this.isWindowsTaskkillTerminalRace(task)) {
|
|
1744
|
+
this.recordWindowsTaskkillNotice(
|
|
1745
|
+
task,
|
|
1746
|
+
`taskkill force rejected after the task became terminal for pid ${String(pid)} (${BackgroundTaskRegistry.errorMessage(error)}); treating as a terminal race`,
|
|
1747
|
+
);
|
|
1748
|
+
resolveForceReady();
|
|
1749
|
+
return;
|
|
1750
|
+
}
|
|
1751
|
+
const failure = this.makeWindowsForceFailure(task, pid, BackgroundTaskRegistry.errorMessage(error));
|
|
1752
|
+
this.recordWindowsForceFailure(task, failure);
|
|
1753
|
+
rejectForceReady(failure);
|
|
1754
|
+
},
|
|
1755
|
+
);
|
|
1756
|
+
|
|
1757
|
+
return forcePromise;
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
private requestWindowsKill(task: BgTask, pid: number, signal: NodeJS.Signals): void {
|
|
1761
|
+
if (signal === 'SIGKILL') {
|
|
1762
|
+
this.startWindowsForceKill(task, pid);
|
|
1763
|
+
task.killSignalSent = true;
|
|
1764
|
+
return;
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
this.startWindowsSoftKill(task, pid);
|
|
1768
|
+
task.killSignalSent = true;
|
|
1769
|
+
if (task.killEscalationTimer !== undefined) return;
|
|
1770
|
+
task.killEscalationTimer = setTimeout(() => {
|
|
1771
|
+
task.killEscalationTimer = undefined;
|
|
1772
|
+
if (task.status !== 'running') return;
|
|
1773
|
+
try {
|
|
1774
|
+
this.requestKill(task, 'SIGKILL');
|
|
1775
|
+
} catch (error) {
|
|
1776
|
+
task.error = BackgroundTaskRegistry.appendTaskError(
|
|
1777
|
+
task.error,
|
|
1778
|
+
`SIGKILL failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1779
|
+
);
|
|
1780
|
+
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
1781
|
+
this.logger.error(
|
|
1782
|
+
`[background-tasks] failed to write metadata for ${task.id}:`,
|
|
1783
|
+
metadataError,
|
|
1784
|
+
);
|
|
1785
|
+
});
|
|
1786
|
+
}
|
|
1787
|
+
}, this.killGraceMs).unref();
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1404
1790
|
private requestKill(task: BgTask, signal: NodeJS.Signals = 'SIGTERM'): void {
|
|
1405
1791
|
if (task.status !== 'running') {
|
|
1406
1792
|
throw new Error(`Task ${task.id} is ${task.status}, not running`);
|
|
@@ -1413,18 +1799,21 @@ export class BackgroundTaskRegistry {
|
|
|
1413
1799
|
}
|
|
1414
1800
|
if (task.killSignalSent && signal === 'SIGTERM') return;
|
|
1415
1801
|
|
|
1802
|
+
if (this.platform === 'win32') {
|
|
1803
|
+
this.requestWindowsKill(task, task.pid, signal);
|
|
1804
|
+
return;
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1416
1807
|
const errors: string[] = [];
|
|
1417
1808
|
let killed = false;
|
|
1418
1809
|
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
);
|
|
1427
|
-
}
|
|
1810
|
+
try {
|
|
1811
|
+
this.killProcess(-task.pid, signal);
|
|
1812
|
+
killed = true;
|
|
1813
|
+
} catch (error) {
|
|
1814
|
+
errors.push(
|
|
1815
|
+
`process group kill failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
1816
|
+
);
|
|
1428
1817
|
}
|
|
1429
1818
|
|
|
1430
1819
|
if (!killed) {
|
|
@@ -1441,7 +1830,13 @@ export class BackgroundTaskRegistry {
|
|
|
1441
1830
|
}
|
|
1442
1831
|
|
|
1443
1832
|
task.killSignalSent = true;
|
|
1444
|
-
|
|
1833
|
+
// SIGKILL is the terminal escalation; it must never schedule a further one.
|
|
1834
|
+
if (signal === 'SIGKILL') return;
|
|
1835
|
+
// Only one escalation timer may be outstanding. Concurrent stop requests
|
|
1836
|
+
// previously each scheduled their own, producing duplicate SIGKILLs.
|
|
1837
|
+
if (task.killEscalationTimer !== undefined) return;
|
|
1838
|
+
task.killEscalationTimer = setTimeout(() => {
|
|
1839
|
+
task.killEscalationTimer = undefined;
|
|
1445
1840
|
if (task.status !== 'running') return;
|
|
1446
1841
|
try {
|
|
1447
1842
|
this.requestKill(task, 'SIGKILL');
|
|
@@ -1473,6 +1868,53 @@ export class BackgroundTaskRegistry {
|
|
|
1473
1868
|
});
|
|
1474
1869
|
}
|
|
1475
1870
|
|
|
1871
|
+
private waitForEndOrWindowsForceFailure(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
1872
|
+
const state = this.getWindowsKillState(task);
|
|
1873
|
+
if (state.forceFailure !== undefined) return Promise.reject(state.forceFailure);
|
|
1874
|
+
if (task.status !== 'running') return Promise.resolve(true);
|
|
1875
|
+
return new Promise((resolve, reject) => {
|
|
1876
|
+
const cleanup = () => {
|
|
1877
|
+
clearTimeout(timeout);
|
|
1878
|
+
const waiterIndex = task.waiters.indexOf(done);
|
|
1879
|
+
if (waiterIndex >= 0) task.waiters.splice(waiterIndex, 1);
|
|
1880
|
+
const listeners = state.forceFailureListeners;
|
|
1881
|
+
if (listeners !== undefined) {
|
|
1882
|
+
const listenerIndex = listeners.indexOf(failed);
|
|
1883
|
+
if (listenerIndex >= 0) listeners.splice(listenerIndex, 1);
|
|
1884
|
+
if (listeners.length === 0) delete state.forceFailureListeners;
|
|
1885
|
+
}
|
|
1886
|
+
};
|
|
1887
|
+
const timeout = setTimeout(() => {
|
|
1888
|
+
cleanup();
|
|
1889
|
+
resolve(false);
|
|
1890
|
+
}, timeoutMs);
|
|
1891
|
+
const done = () => {
|
|
1892
|
+
cleanup();
|
|
1893
|
+
resolve(true);
|
|
1894
|
+
};
|
|
1895
|
+
const failed = (error: Error) => {
|
|
1896
|
+
cleanup();
|
|
1897
|
+
reject(error);
|
|
1898
|
+
};
|
|
1899
|
+
task.waiters.push(done);
|
|
1900
|
+
if (state.forceFailureListeners === undefined) state.forceFailureListeners = [];
|
|
1901
|
+
state.forceFailureListeners.push(failed);
|
|
1902
|
+
});
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
private async awaitWindowsForceBeforeTerminal(task: BgTask): Promise<Error | undefined> {
|
|
1906
|
+
const state = this.windowsKillStates.get(task);
|
|
1907
|
+
if (state === undefined) return undefined;
|
|
1908
|
+
const forcePromise = state.forcePromise;
|
|
1909
|
+
if (forcePromise === undefined) return state.forceFailure;
|
|
1910
|
+
try {
|
|
1911
|
+
await forcePromise;
|
|
1912
|
+
} catch (error) {
|
|
1913
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
1914
|
+
}
|
|
1915
|
+
return state.forceFailure;
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1476
1918
|
private publishTerminal(task: BgTask): void {
|
|
1477
1919
|
if (task.terminalPublished || task.terminalPublishInFlight) return;
|
|
1478
1920
|
task.terminalPublishInFlight = true;
|
|
@@ -1572,8 +2014,17 @@ export class BackgroundTaskRegistry {
|
|
|
1572
2014
|
if (task.finalized) return;
|
|
1573
2015
|
task.finalized = true;
|
|
1574
2016
|
if (task.timeoutHandle) clearTimeout(task.timeoutHandle);
|
|
2017
|
+
if (task.killEscalationTimer !== undefined) {
|
|
2018
|
+
clearTimeout(task.killEscalationTimer);
|
|
2019
|
+
task.killEscalationTimer = undefined;
|
|
2020
|
+
}
|
|
1575
2021
|
let finalStatus = status;
|
|
1576
2022
|
let finalError = error;
|
|
2023
|
+
const forceFailure = await this.awaitWindowsForceBeforeTerminal(task);
|
|
2024
|
+
if (forceFailure !== undefined) {
|
|
2025
|
+
finalStatus = 'failed';
|
|
2026
|
+
finalError = BackgroundTaskRegistry.appendTaskError(finalError, forceFailure.message);
|
|
2027
|
+
}
|
|
1577
2028
|
task.exitCode = exitCode;
|
|
1578
2029
|
task.signal = signal ?? null;
|
|
1579
2030
|
|