pi-background-tasks 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BACKGROUND-TASKS-INSTRUCTIONS.md +1 -1
- package/README.md +70 -25
- package/TESTING.md +42 -2
- package/TEST_PLAN.md +7 -4
- package/dist/extensions/anthropic-attribution-child.js +4 -0
- package/dist/extensions/anthropic-attribution-child.js.map +1 -0
- package/dist/extensions/anthropic-attribution.js +107 -0
- package/dist/extensions/anthropic-attribution.js.map +1 -0
- package/dist/extensions/background-tasks.js +2 -0
- package/dist/extensions/background-tasks.js.map +1 -0
- package/dist/extensions/delegate-child.js +2 -0
- package/dist/extensions/delegate-child.js.map +1 -0
- package/dist/extensions/fusion-child.js +2 -0
- package/dist/extensions/fusion-child.js.map +1 -0
- package/dist/package.json +5 -0
- package/dist/src/core/anthropic-attribution-path.js +24 -0
- package/dist/src/core/anthropic-attribution-path.js.map +1 -0
- package/dist/src/core/anthropic-attribution.js +2247 -0
- package/dist/src/core/anthropic-attribution.js.map +1 -0
- package/dist/src/core/attested-pi-contract.js +5 -0
- package/dist/src/core/attested-pi-contract.js.map +1 -0
- package/dist/src/core/attested-pi-run.js +749 -0
- package/dist/src/core/attested-pi-run.js.map +1 -0
- package/dist/src/core/canonical-json.js +19 -0
- package/dist/src/core/canonical-json.js.map +1 -0
- package/dist/src/core/common.js +787 -0
- package/dist/src/core/common.js.map +1 -0
- package/dist/src/core/config.js +78 -0
- package/dist/src/core/config.js.map +1 -0
- package/dist/src/core/context/parent-snapshot.js +75 -0
- package/dist/src/core/context/parent-snapshot.js.map +1 -0
- package/dist/src/core/context/token-budget.js +631 -0
- package/dist/src/core/context/token-budget.js.map +1 -0
- package/dist/src/core/context/visible-conversation-v2.js +390 -0
- package/dist/src/core/context/visible-conversation-v2.js.map +1 -0
- package/dist/src/core/delegate/artifacts.js +357 -0
- package/dist/src/core/delegate/artifacts.js.map +1 -0
- package/dist/src/core/delegate/budget.js +266 -0
- package/dist/src/core/delegate/budget.js.map +1 -0
- package/dist/src/core/delegate/facade-contract.js +7 -0
- package/dist/src/core/delegate/facade-contract.js.map +1 -0
- package/dist/src/core/delegate/hook-contract-evidence.json +18 -0
- package/dist/src/core/delegate/hook-contract.js +124 -0
- package/dist/src/core/delegate/hook-contract.js.map +1 -0
- package/dist/src/core/delegate/launch.js +366 -0
- package/dist/src/core/delegate/launch.js.map +1 -0
- package/dist/src/core/delegate/result-package.js +293 -0
- package/dist/src/core/delegate/result-package.js.map +1 -0
- package/dist/src/core/delegate/runner.js +398 -0
- package/dist/src/core/delegate/runner.js.map +1 -0
- package/dist/src/core/delegate/seed.js +334 -0
- package/dist/src/core/delegate/seed.js.map +1 -0
- package/dist/src/core/delegate/types.js +103 -0
- package/dist/src/core/delegate/types.js.map +1 -0
- package/dist/src/core/durable-fs.js +372 -0
- package/dist/src/core/durable-fs.js.map +1 -0
- package/dist/src/core/extension-api.js +382 -0
- package/dist/src/core/extension-api.js.map +1 -0
- package/dist/src/core/fusion/artifacts.js +654 -0
- package/dist/src/core/fusion/artifacts.js.map +1 -0
- package/dist/src/core/fusion/budget.js +949 -0
- package/dist/src/core/fusion/budget.js.map +1 -0
- package/dist/src/core/fusion/child-protocol.js +176 -0
- package/dist/src/core/fusion/child-protocol.js.map +1 -0
- package/dist/src/core/fusion/claude-cache.js +156 -0
- package/dist/src/core/fusion/claude-cache.js.map +1 -0
- package/dist/src/core/fusion/clean-context.js +64 -0
- package/dist/src/core/fusion/clean-context.js.map +1 -0
- package/dist/src/core/fusion/config.js +349 -0
- package/dist/src/core/fusion/config.js.map +1 -0
- package/dist/src/core/fusion/context.js +184 -0
- package/dist/src/core/fusion/context.js.map +1 -0
- package/dist/src/core/fusion/evaluation.js +696 -0
- package/dist/src/core/fusion/evaluation.js.map +1 -0
- package/dist/src/core/fusion/facade-contract.js +3 -0
- package/dist/src/core/fusion/facade-contract.js.map +1 -0
- package/dist/src/core/fusion/orchestrator.js +867 -0
- package/dist/src/core/fusion/orchestrator.js.map +1 -0
- package/dist/src/core/fusion/output-contract.js +27 -0
- package/dist/src/core/fusion/output-contract.js.map +1 -0
- package/dist/src/core/fusion/pi-child.js +1741 -0
- package/dist/src/core/fusion/pi-child.js.map +1 -0
- package/dist/src/core/fusion/prompts.js +260 -0
- package/dist/src/core/fusion/prompts.js.map +1 -0
- package/dist/src/core/fusion/result-package.js +857 -0
- package/dist/src/core/fusion/result-package.js.map +1 -0
- package/dist/src/core/fusion/source-policy.js +259 -0
- package/dist/src/core/fusion/source-policy.js.map +1 -0
- package/dist/src/core/fusion/types.js +207 -0
- package/dist/src/core/fusion/types.js.map +1 -0
- package/dist/src/core/fusion/web-fetch.js +777 -0
- package/dist/src/core/fusion/web-fetch.js.map +1 -0
- package/dist/src/core/fusion/workflows.js +126 -0
- package/dist/src/core/fusion/workflows.js.map +1 -0
- package/dist/src/core/lazy-module.js +181 -0
- package/dist/src/core/lazy-module.js.map +1 -0
- package/dist/src/core/pi-launch.js +431 -0
- package/dist/src/core/pi-launch.js.map +1 -0
- package/dist/src/core/registry.js +3331 -0
- package/dist/src/core/registry.js.map +1 -0
- package/dist/src/core/reload-shell-owner.js +1396 -0
- package/dist/src/core/reload-shell-owner.js.map +1 -0
- package/dist/src/core/shell-policy.js +80 -0
- package/dist/src/core/shell-policy.js.map +1 -0
- package/dist/src/core/task-durable.js +51 -0
- package/dist/src/core/task-durable.js.map +1 -0
- package/dist/src/core/update-check.js +92 -0
- package/dist/src/core/update-check.js.map +1 -0
- package/dist/src/core/windows-taskkill.js +185 -0
- package/dist/src/core/windows-taskkill.js.map +1 -0
- package/dist/src/delegate-child-extension.js +776 -0
- package/dist/src/delegate-child-extension.js.map +1 -0
- package/dist/src/delegate-extension.js +746 -0
- package/dist/src/delegate-extension.js.map +1 -0
- package/dist/src/extension.js +953 -0
- package/dist/src/extension.js.map +1 -0
- package/dist/src/fusion-child-extension.js +760 -0
- package/dist/src/fusion-child-extension.js.map +1 -0
- package/dist/src/fusion-extension.js +1030 -0
- package/dist/src/fusion-extension.js.map +1 -0
- package/dist/src/ui/background-tasks-manager.js +689 -0
- package/dist/src/ui/background-tasks-manager.js.map +1 -0
- package/dist/src/ui/fusion-model-selector.js +277 -0
- package/dist/src/ui/fusion-model-selector.js.map +1 -0
- package/docs/INDEX.md +35 -33
- package/docs/api/eventbus-v1.md +21 -5
- package/docs/choose-a-workflow.md +2 -0
- package/docs/commands/bg-clear.md +3 -3
- package/docs/commands/bg-update.md +3 -3
- package/docs/commands/bg.md +18 -6
- package/docs/commands/claude-cache.md +4 -4
- package/docs/commands/fusion-models.md +49 -4
- package/docs/commands/fusion.md +3 -3
- package/docs/commands/jobs.md +3 -3
- package/docs/commands/kill.md +3 -3
- package/docs/commands/logs.md +3 -3
- package/docs/commands/task-manager.md +5 -5
- package/docs/concepts/completion-delivery.md +6 -2
- package/docs/getting-started.md +7 -1
- package/docs/manifest.json +228 -42
- package/docs/operations/configuration.md +58 -5
- package/docs/operations/releasing.md +4 -2
- package/docs/operations/testing.md +22 -0
- package/docs/operations/troubleshooting.md +6 -2
- package/docs/read-before-edit.md +44 -31
- package/docs/reference/runtime-contracts.md +87 -51
- package/docs/reference/shortcuts-and-dock.md +28 -14
- package/docs/subsystems/anthropic-attribution.md +23 -11
- package/docs/subsystems/attested-pi-runs.md +12 -7
- package/docs/subsystems/background-task-runtime.md +53 -11
- package/docs/subsystems/child-launch-durability-and-safety.md +28 -9
- package/docs/subsystems/delegation.md +16 -3
- package/docs/subsystems/docs-freshness-gate.md +14 -7
- package/docs/subsystems/fusion.md +36 -11
- package/docs/subsystems/host-ui-and-telemetry.md +22 -9
- package/docs/tools/bg_delegate.md +3 -1
- package/docs/tools/bg_kill.md +3 -1
- package/docs/tools/bg_logs.md +3 -1
- package/docs/tools/bg_result.md +5 -3
- package/docs/tools/bg_run.md +31 -5
- package/docs/tools/bg_run_pi_attested.md +3 -1
- package/docs/tools/bg_status.md +3 -1
- package/docs/tools/fusion_investigate.md +3 -1
- package/docs/tools/fusion_reason.md +3 -1
- package/docs/tools/fusion_research.md +3 -1
- package/docs/tools/fusion_validate.md +3 -1
- package/extensions/anthropic-attribution-child.ts +3 -0
- package/extensions/anthropic-attribution.ts +166 -1
- package/package.json +12 -10
- package/src/core/anthropic-attribution-path.ts +5 -4
- package/src/core/anthropic-attribution.ts +293 -113
- package/src/core/attested-pi-contract.ts +4 -0
- package/src/core/attested-pi-run.ts +468 -81
- package/src/core/canonical-json.ts +21 -0
- package/src/core/common.ts +450 -40
- package/src/core/config.ts +121 -0
- package/src/core/context/visible-conversation-v2.ts +3 -6
- package/src/core/delegate/artifacts.ts +13 -9
- package/src/core/delegate/budget.ts +13 -6
- package/src/core/delegate/facade-contract.ts +6 -0
- package/src/core/delegate/result-package.ts +32 -14
- package/src/core/delegate/runner.ts +129 -58
- package/src/core/delegate/seed.ts +5 -7
- package/src/core/durable-fs.ts +178 -30
- package/src/core/extension-api.ts +36 -8
- package/src/core/fusion/artifacts.ts +17 -16
- package/src/core/fusion/clean-context.ts +8 -5
- package/src/core/fusion/config.ts +2 -1
- package/src/core/fusion/context.ts +2 -6
- package/src/core/fusion/facade-contract.ts +2 -0
- package/src/core/fusion/orchestrator.ts +3 -3
- package/src/core/fusion/prompts.ts +1 -1
- package/src/core/fusion/result-package.ts +259 -72
- package/src/core/fusion/source-policy.ts +43 -17
- package/src/core/lazy-module.ts +215 -0
- package/src/core/pi-launch.ts +390 -65
- package/src/core/registry.ts +2010 -381
- package/src/core/reload-shell-owner.ts +1662 -0
- package/src/core/shell-policy.ts +134 -0
- package/src/core/task-durable.ts +67 -0
- package/src/delegate-extension.ts +404 -144
- package/src/extension.ts +387 -114
- package/src/fusion-extension.ts +202 -96
package/src/core/registry.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { spawn as nodeSpawn, type SpawnOptions } from 'node:child_process';
|
|
2
2
|
import { randomBytes } from 'node:crypto';
|
|
3
3
|
import { createWriteStream, existsSync } from 'node:fs';
|
|
4
|
-
import { mkdir, realpath, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { mkdir, realpath, rm, writeFile } from 'node:fs/promises';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import type { Api, Model } from '@earendil-works/pi-ai';
|
|
7
7
|
import type { ExtensionContext } from '@earendil-works/pi-coding-agent';
|
|
@@ -16,8 +16,11 @@ import {
|
|
|
16
16
|
normalizeTaskName,
|
|
17
17
|
parseAgentActivity,
|
|
18
18
|
parseJsonText,
|
|
19
|
+
rejectSurvivalForTaskKind,
|
|
20
|
+
resolveShellPolicy,
|
|
19
21
|
sanitizePathSegment,
|
|
20
|
-
|
|
22
|
+
shellInvocationForPolicy,
|
|
23
|
+
shellPolicySnapshot,
|
|
21
24
|
shellQuote,
|
|
22
25
|
snapshot,
|
|
23
26
|
taskDisplayName,
|
|
@@ -26,6 +29,13 @@ import {
|
|
|
26
29
|
type BgTaskSnapshot,
|
|
27
30
|
type JsonObject,
|
|
28
31
|
type KillKind,
|
|
32
|
+
type ReloadShellActivationClaimV1,
|
|
33
|
+
type ReloadShellActivationLeaseV1,
|
|
34
|
+
type ReloadShellHostAdapterV1,
|
|
35
|
+
type ReloadShellIdentityV1,
|
|
36
|
+
type ReloadShellOwnerHubV1,
|
|
37
|
+
type ReloadableShellExecutionV1,
|
|
38
|
+
type ResolvedShellPolicy,
|
|
29
39
|
type StartAttestedPiTaskOptions,
|
|
30
40
|
type StartDelegateTaskOptions,
|
|
31
41
|
type StartManagedTaskOptions,
|
|
@@ -34,31 +44,30 @@ import {
|
|
|
34
44
|
type TaskStatus,
|
|
35
45
|
type TaskTokenUsage,
|
|
36
46
|
type TaskToolUsage,
|
|
47
|
+
type TerminalPublicationAbandonReason,
|
|
48
|
+
ReloadSurvivalError,
|
|
37
49
|
} from './common.js';
|
|
38
50
|
import {
|
|
51
|
+
ATTESTED_GIT_KILL_GRACE_MS,
|
|
52
|
+
ATTESTED_GIT_MAX_OUTPUT_BYTES,
|
|
39
53
|
ATTESTED_TASK_ID_PATTERN,
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
gitRepoRoot,
|
|
46
|
-
makeAttestedTaskId,
|
|
47
|
-
makeAttestedTaskPaths,
|
|
48
|
-
observePiOAuth,
|
|
49
|
-
parsePiJsonEvents,
|
|
50
|
-
resolveReportPath,
|
|
51
|
-
spawnAndCapturePi,
|
|
52
|
-
writeFileFsynced,
|
|
53
|
-
writeJsonAtomic,
|
|
54
|
-
} from './attested-pi-run.js';
|
|
54
|
+
} from './attested-pi-contract.js';
|
|
55
|
+
import type { AttestedGitSpawn, GitCommandOptions } from './attested-pi-run.js';
|
|
56
|
+
import { closeAndFsyncOutputStream, writeFileFsynced, writeJsonAtomic } from './task-durable.js';
|
|
57
|
+
|
|
58
|
+
type AttestedPiRuntime = typeof import('./attested-pi-run.js');
|
|
55
59
|
import {
|
|
56
60
|
assertWindowsCommandLineWithinLimit,
|
|
57
61
|
piLaunchArgv,
|
|
58
62
|
resolvePiLaunch,
|
|
59
63
|
type PiLaunchSpec,
|
|
60
64
|
} from './pi-launch.js';
|
|
65
|
+
import { BackgroundTaskExtensionServiceClosedError } from './extension-api.js';
|
|
61
66
|
import { resolveAnthropicAttributionExtensionPath } from './anthropic-attribution-path.js';
|
|
67
|
+
import {
|
|
68
|
+
createReloadableShellExecutionV1,
|
|
69
|
+
RELOAD_SHELL_OWNER_PROTOCOL,
|
|
70
|
+
} from './reload-shell-owner.js';
|
|
62
71
|
import {
|
|
63
72
|
runWindowsTaskkill,
|
|
64
73
|
type TaskkillOutcome,
|
|
@@ -70,9 +79,57 @@ export const MAX_OUTPUT_BYTES = Number(process.env['PI_BG_MAX_OUTPUT_BYTES'] ??
|
|
|
70
79
|
export const KILL_GRACE_MS = 3000;
|
|
71
80
|
export const STOP_WAIT_MS = KILL_GRACE_MS + 1500;
|
|
72
81
|
export const MAX_RECENT_TASKS = 100;
|
|
82
|
+
export const TERMINAL_PUBLICATION_MAX_ATTEMPTS = 3;
|
|
83
|
+
export const TERMINAL_PUBLICATION_RETRY_MS = 100;
|
|
84
|
+
export const TASK_ADMISSION_TIMEOUT_MS = 30_000;
|
|
85
|
+
const TERMINAL_PUBLICATION_DIAGNOSTIC_CHARS = 500;
|
|
73
86
|
const TELEMETRY_BUFFER_CHARS = 512 * 1024;
|
|
87
|
+
|
|
88
|
+
export type TerminalPublicationClosureReason = Extract<
|
|
89
|
+
TerminalPublicationAbandonReason,
|
|
90
|
+
'registry_shutdown' | 'publisher_closed'
|
|
91
|
+
>;
|
|
92
|
+
|
|
93
|
+
type TerminalPublicationGateOutcome =
|
|
94
|
+
| { readonly kind: 'released' }
|
|
95
|
+
| { readonly kind: 'rejected'; readonly error: unknown }
|
|
96
|
+
| { readonly kind: 'closed'; readonly reason: TerminalPublicationAbandonReason };
|
|
97
|
+
|
|
98
|
+
interface TerminalPublicationAbandonSignal {
|
|
99
|
+
readonly promise: Promise<TerminalPublicationAbandonReason>;
|
|
100
|
+
readonly resolve: (reason: TerminalPublicationAbandonReason) => void;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export class BackgroundTaskAdmissionClosedError extends Error {
|
|
104
|
+
readonly code = 'pi_background_tasks_admission_closed';
|
|
105
|
+
|
|
106
|
+
constructor(kind: string) {
|
|
107
|
+
super(`Cannot start ${kind} after background task admissions have closed`);
|
|
108
|
+
this.name = 'BackgroundTaskAdmissionClosedError';
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export class BackgroundTaskAdmissionTimeoutError extends Error {
|
|
113
|
+
readonly code = 'pi_background_tasks_admission_timeout';
|
|
114
|
+
|
|
115
|
+
constructor(kind: string, timeoutMs: number) {
|
|
116
|
+
super(`Timed out while preparing ${kind} after ${String(timeoutMs)}ms`);
|
|
117
|
+
this.name = 'BackgroundTaskAdmissionTimeoutError';
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
interface TaskAdmission {
|
|
122
|
+
readonly kind: string;
|
|
123
|
+
readonly controller: AbortController;
|
|
124
|
+
readonly deadlineAt: number;
|
|
125
|
+
readonly timeoutMs: number;
|
|
126
|
+
timeoutHandle: NodeJS.Timeout | undefined;
|
|
127
|
+
released: boolean;
|
|
128
|
+
}
|
|
74
129
|
export const WIN32_CMD_PI_TELEMETRY_UNAVAILABLE_REASON =
|
|
75
130
|
'win32-cmd-cannot-safely-intercept-pi-argv';
|
|
131
|
+
export const NON_POSIX_SHELL_PI_TELEMETRY_UNAVAILABLE_REASON =
|
|
132
|
+
'user-non-posix-shell-cannot-safely-intercept-pi-argv';
|
|
76
133
|
|
|
77
134
|
export interface BackgroundTaskModelRegistry
|
|
78
135
|
extends Pick<ExtensionContext['modelRegistry'], 'getAll'> {
|
|
@@ -123,6 +180,19 @@ type KillTreeFn = (
|
|
|
123
180
|
signal?: AbortSignal,
|
|
124
181
|
) => Promise<TaskkillOutcome>;
|
|
125
182
|
|
|
183
|
+
interface PosixProcessGroupKillState {
|
|
184
|
+
readonly groupId: number;
|
|
185
|
+
readonly completion: Promise<void>;
|
|
186
|
+
readonly resolveCompletion: () => void;
|
|
187
|
+
readonly deadlineAt: number;
|
|
188
|
+
forceAttempted: boolean;
|
|
189
|
+
settled: boolean;
|
|
190
|
+
failure?: Error | undefined;
|
|
191
|
+
lastProbeError?: Error | undefined;
|
|
192
|
+
verificationTimer?: NodeJS.Timeout | undefined;
|
|
193
|
+
failureListeners?: Array<(error: Error) => void> | undefined;
|
|
194
|
+
}
|
|
195
|
+
|
|
126
196
|
interface WindowsKillState {
|
|
127
197
|
softController?: AbortController | undefined;
|
|
128
198
|
softPromise?: Promise<void> | undefined;
|
|
@@ -157,13 +227,19 @@ export interface BackgroundTaskRegistryOptions {
|
|
|
157
227
|
killTree?: KillTreeFn;
|
|
158
228
|
platform?: NodeJS.Platform;
|
|
159
229
|
env?: NodeJS.ProcessEnv;
|
|
230
|
+
shellPolicy?: ResolvedShellPolicy;
|
|
160
231
|
makeTaskId?: () => string;
|
|
161
232
|
now?: () => number;
|
|
162
233
|
maxOutputBytes?: number;
|
|
163
234
|
maxRecentTasks?: number;
|
|
164
235
|
killGraceMs?: number;
|
|
165
236
|
stopWaitMs?: number;
|
|
237
|
+
taskAdmissionTimeoutMs?: number;
|
|
238
|
+
attestedGitKillGraceMs?: number;
|
|
239
|
+
attestedGitMaxOutputBytes?: number;
|
|
240
|
+
attestedGitSpawn?: AttestedGitSpawn;
|
|
166
241
|
logger?: Pick<Console, 'error'>;
|
|
242
|
+
reloadShellOwner?: ReloadShellOwnerHubV1;
|
|
167
243
|
}
|
|
168
244
|
|
|
169
245
|
interface RuntimeDir {
|
|
@@ -719,34 +795,64 @@ export class BackgroundTaskRegistry {
|
|
|
719
795
|
private readonly tasks = new Map<string, BgTask>();
|
|
720
796
|
private runtimeDir: RuntimeDir | undefined;
|
|
721
797
|
private shuttingDown = false;
|
|
798
|
+
private taskAdmissionsClosed = false;
|
|
799
|
+
private readonly activeTaskAdmissions = new Set<TaskAdmission>();
|
|
800
|
+
private readonly taskAdmissionDrainWaiters = new Set<() => void>();
|
|
801
|
+
private terminalPublicationClosed = false;
|
|
802
|
+
private terminalPublicationCloseReason: TerminalPublicationClosureReason | undefined;
|
|
803
|
+
private readonly terminalPublicationClosedSignal: Promise<TerminalPublicationClosureReason>;
|
|
804
|
+
private resolveTerminalPublicationClosedSignal: (
|
|
805
|
+
reason: TerminalPublicationClosureReason,
|
|
806
|
+
) => void = () => {};
|
|
722
807
|
private readonly spawn: BackgroundTaskSpawn;
|
|
723
808
|
private readonly killProcess: KillProcessFn;
|
|
724
809
|
private readonly killTree: KillTreeFn;
|
|
725
810
|
private readonly platform: NodeJS.Platform;
|
|
726
811
|
private readonly env: NodeJS.ProcessEnv;
|
|
812
|
+
private shellPolicy: ResolvedShellPolicy | undefined;
|
|
813
|
+
private readonly shellPolicyEnv: NodeJS.ProcessEnv | undefined;
|
|
727
814
|
private readonly makeTaskIdFn: () => string;
|
|
728
815
|
private readonly now: () => number;
|
|
729
816
|
private readonly maxOutputBytes: number;
|
|
730
817
|
private readonly maxRecentTasks: number;
|
|
731
818
|
private readonly killGraceMs: number;
|
|
732
819
|
private readonly stopWaitMs: number;
|
|
820
|
+
private readonly taskAdmissionTimeoutMs: number;
|
|
821
|
+
private readonly attestedGitKillGraceMs: number;
|
|
822
|
+
private readonly attestedGitMaxOutputBytes: number;
|
|
823
|
+
private readonly attestedGitSpawn: AttestedGitSpawn | undefined;
|
|
824
|
+
private attestedRuntimePromise: Promise<AttestedPiRuntime> | undefined;
|
|
733
825
|
private readonly logger: Pick<Console, 'error'>;
|
|
734
826
|
private readonly onChange: () => void;
|
|
735
827
|
private readonly sendCompletionNotification: CompletionNotificationSender;
|
|
736
828
|
private readonly publishTerminalSnapshot: (task: BgTaskSnapshot) => void;
|
|
829
|
+
private readonly posixProcessGroupKillStates = new WeakMap<BgTask, PosixProcessGroupKillState>();
|
|
737
830
|
private readonly windowsKillStates = new WeakMap<BgTask, WindowsKillState>();
|
|
831
|
+
private readonly terminalPublicationAbandonSignals = new WeakMap<
|
|
832
|
+
BgTask,
|
|
833
|
+
TerminalPublicationAbandonSignal
|
|
834
|
+
>();
|
|
835
|
+
private readonly reloadShellOwner: ReloadShellOwnerHubV1 | undefined;
|
|
836
|
+
private reloadShellLease: ReloadShellActivationLeaseV1 | undefined;
|
|
837
|
+
private reloadShellIdentity: ReloadShellIdentityV1 | undefined;
|
|
738
838
|
|
|
739
839
|
constructor(options: BackgroundTaskRegistryOptions) {
|
|
840
|
+
this.terminalPublicationClosedSignal = new Promise((resolve) => {
|
|
841
|
+
this.resolveTerminalPublicationClosedSignal = resolve;
|
|
842
|
+
});
|
|
740
843
|
this.spawn =
|
|
741
844
|
options.spawn ?? ((command, args, spawnOptions) => nodeSpawn(command, args, spawnOptions));
|
|
742
845
|
this.killProcess = options.killProcess ?? process.kill.bind(process);
|
|
743
846
|
this.platform = options.platform ?? process.platform;
|
|
744
847
|
this.env = options.env ?? process.env;
|
|
848
|
+
this.shellPolicy = options.shellPolicy;
|
|
849
|
+
this.shellPolicyEnv = options.shellPolicy === undefined ? { ...this.env } : undefined;
|
|
850
|
+
const taskkillEnv = this.env;
|
|
745
851
|
this.killTree =
|
|
746
852
|
options.killTree ??
|
|
747
853
|
((pid, phase, signal) => {
|
|
748
854
|
const taskkillOptions: WindowsTaskkillOptions =
|
|
749
|
-
signal === undefined ? { env:
|
|
855
|
+
signal === undefined ? { env: taskkillEnv } : { env: taskkillEnv, signal };
|
|
750
856
|
return runWindowsTaskkill(pid, phase, taskkillOptions);
|
|
751
857
|
});
|
|
752
858
|
this.makeTaskIdFn = options.makeTaskId ?? defaultTaskId;
|
|
@@ -755,18 +861,214 @@ export class BackgroundTaskRegistry {
|
|
|
755
861
|
this.maxRecentTasks = options.maxRecentTasks ?? MAX_RECENT_TASKS;
|
|
756
862
|
this.killGraceMs = options.killGraceMs ?? KILL_GRACE_MS;
|
|
757
863
|
this.stopWaitMs = options.stopWaitMs ?? STOP_WAIT_MS;
|
|
864
|
+
this.taskAdmissionTimeoutMs = BackgroundTaskRegistry.positiveTimeout(
|
|
865
|
+
options.taskAdmissionTimeoutMs,
|
|
866
|
+
TASK_ADMISSION_TIMEOUT_MS,
|
|
867
|
+
'taskAdmissionTimeoutMs',
|
|
868
|
+
);
|
|
869
|
+
this.attestedGitKillGraceMs = BackgroundTaskRegistry.positiveTimeout(
|
|
870
|
+
options.attestedGitKillGraceMs,
|
|
871
|
+
ATTESTED_GIT_KILL_GRACE_MS,
|
|
872
|
+
'attestedGitKillGraceMs',
|
|
873
|
+
);
|
|
874
|
+
this.attestedGitMaxOutputBytes = BackgroundTaskRegistry.positiveTimeout(
|
|
875
|
+
options.attestedGitMaxOutputBytes,
|
|
876
|
+
ATTESTED_GIT_MAX_OUTPUT_BYTES,
|
|
877
|
+
'attestedGitMaxOutputBytes',
|
|
878
|
+
);
|
|
879
|
+
this.attestedGitSpawn = options.attestedGitSpawn;
|
|
758
880
|
this.logger = options.logger ?? console;
|
|
759
881
|
this.onChange = options.onChange ?? noopOnChange;
|
|
760
882
|
this.sendCompletionNotification = options.sendCompletionNotification;
|
|
761
883
|
this.publishTerminalSnapshot = options.publishTerminal ?? noopOnChange;
|
|
884
|
+
this.reloadShellOwner = options.reloadShellOwner;
|
|
762
885
|
}
|
|
763
886
|
|
|
764
887
|
isShuttingDown(): boolean {
|
|
765
888
|
return this.shuttingDown;
|
|
766
889
|
}
|
|
767
890
|
|
|
891
|
+
private resolvedShellPolicy(): ResolvedShellPolicy {
|
|
892
|
+
const existing = this.shellPolicy;
|
|
893
|
+
if (existing !== undefined) return existing;
|
|
894
|
+
const resolved = resolveShellPolicy(
|
|
895
|
+
this.platform,
|
|
896
|
+
this.shellPolicyEnv ?? this.env,
|
|
897
|
+
process.cwd(),
|
|
898
|
+
);
|
|
899
|
+
this.shellPolicy = resolved;
|
|
900
|
+
return resolved;
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
private loadAttestedRuntime(): Promise<AttestedPiRuntime> {
|
|
904
|
+
const existing = this.attestedRuntimePromise;
|
|
905
|
+
if (existing !== undefined) return existing;
|
|
906
|
+
const loading = import('./attested-pi-run.js');
|
|
907
|
+
this.attestedRuntimePromise = loading;
|
|
908
|
+
return loading;
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
private static positiveTimeout(
|
|
912
|
+
value: number | undefined,
|
|
913
|
+
fallback: number,
|
|
914
|
+
label: string,
|
|
915
|
+
): number {
|
|
916
|
+
const candidate = value ?? fallback;
|
|
917
|
+
if (!Number.isFinite(candidate) || candidate <= 0) {
|
|
918
|
+
throw new Error(`${label} must be a positive finite number`);
|
|
919
|
+
}
|
|
920
|
+
return Math.max(1, Math.floor(candidate));
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
private beginTaskAdmission(kind: string): TaskAdmission {
|
|
924
|
+
this.assertTaskAdmissionOpen(kind);
|
|
925
|
+
const controller = new AbortController();
|
|
926
|
+
const admission: TaskAdmission = {
|
|
927
|
+
kind,
|
|
928
|
+
controller,
|
|
929
|
+
deadlineAt: Date.now() + this.taskAdmissionTimeoutMs,
|
|
930
|
+
timeoutMs: this.taskAdmissionTimeoutMs,
|
|
931
|
+
timeoutHandle: undefined,
|
|
932
|
+
released: false,
|
|
933
|
+
};
|
|
934
|
+
admission.timeoutHandle = setTimeout(() => {
|
|
935
|
+
if (admission.released || admission.controller.signal.aborted) return;
|
|
936
|
+
admission.controller.abort(
|
|
937
|
+
new BackgroundTaskAdmissionTimeoutError(admission.kind, admission.timeoutMs),
|
|
938
|
+
);
|
|
939
|
+
}, admission.timeoutMs);
|
|
940
|
+
this.activeTaskAdmissions.add(admission);
|
|
941
|
+
return admission;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
private releaseTaskAdmission(admission: TaskAdmission): void {
|
|
945
|
+
if (admission.released) return;
|
|
946
|
+
admission.released = true;
|
|
947
|
+
if (admission.timeoutHandle !== undefined) {
|
|
948
|
+
clearTimeout(admission.timeoutHandle);
|
|
949
|
+
admission.timeoutHandle = undefined;
|
|
950
|
+
}
|
|
951
|
+
this.activeTaskAdmissions.delete(admission);
|
|
952
|
+
if (this.activeTaskAdmissions.size !== 0) return;
|
|
953
|
+
for (const resolve of this.taskAdmissionDrainWaiters) resolve();
|
|
954
|
+
this.taskAdmissionDrainWaiters.clear();
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
private taskAdmissionError(admission: TaskAdmission): Error {
|
|
958
|
+
const reason = admission.controller.signal.reason;
|
|
959
|
+
if (reason instanceof Error) return reason;
|
|
960
|
+
if (this.shuttingDown || this.taskAdmissionsClosed) {
|
|
961
|
+
return new BackgroundTaskAdmissionClosedError(admission.kind);
|
|
962
|
+
}
|
|
963
|
+
return new BackgroundTaskAdmissionTimeoutError(admission.kind, admission.timeoutMs);
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
private surfacedTaskAdmissionError(admission: TaskAdmission, ...details: unknown[]): Error {
|
|
967
|
+
const primary = this.taskAdmissionError(admission);
|
|
968
|
+
const meaningful = details.filter((detail) => {
|
|
969
|
+
if (detail === undefined || detail === primary) return false;
|
|
970
|
+
if (typeof detail !== 'object' || detail === null) return true;
|
|
971
|
+
return (
|
|
972
|
+
Reflect.get(detail, 'name') !== 'AbortError' &&
|
|
973
|
+
Reflect.get(detail, 'code') !== Reflect.get(primary, 'code')
|
|
974
|
+
);
|
|
975
|
+
});
|
|
976
|
+
if (meaningful.length === 0) return primary;
|
|
977
|
+
return new AggregateError(
|
|
978
|
+
[primary, ...meaningful],
|
|
979
|
+
`${primary.message}; admission cancellation or cleanup reported additional failures: ${meaningful.map(BackgroundTaskRegistry.errorMessage).join('; ')}`,
|
|
980
|
+
);
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
private assertTaskAdmissionOpen(kind: string, admission?: TaskAdmission): void {
|
|
984
|
+
if (admission?.controller.signal.aborted === true) throw this.taskAdmissionError(admission);
|
|
985
|
+
if (this.shuttingDown || this.taskAdmissionsClosed) {
|
|
986
|
+
throw new BackgroundTaskAdmissionClosedError(kind);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
|
|
990
|
+
private async awaitTaskAdmissionBoundary<T>(
|
|
991
|
+
promise: Promise<T>,
|
|
992
|
+
admission: TaskAdmission,
|
|
993
|
+
): Promise<T> {
|
|
994
|
+
try {
|
|
995
|
+
const value = await promise;
|
|
996
|
+
this.assertTaskAdmissionOpen(admission.kind, admission);
|
|
997
|
+
return value;
|
|
998
|
+
} catch (error) {
|
|
999
|
+
if (admission.controller.signal.aborted && typeof error === 'object' && error !== null) {
|
|
1000
|
+
const isPlainAbort = Reflect.get(error, 'name') === 'AbortError';
|
|
1001
|
+
const isCleanDurableCancellation =
|
|
1002
|
+
Reflect.get(error, 'code') === 'durable_file_cancelled' &&
|
|
1003
|
+
Reflect.get(error, 'renameCompleted') !== true &&
|
|
1004
|
+
Array.isArray(Reflect.get(error, 'cleanupFailures')) &&
|
|
1005
|
+
(Reflect.get(error, 'cleanupFailures') as unknown[]).length === 0;
|
|
1006
|
+
if (isPlainAbort || isCleanDurableCancellation) {
|
|
1007
|
+
throw this.taskAdmissionError(admission);
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
throw error;
|
|
1011
|
+
}
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
closeTaskAdmissions(): void {
|
|
1015
|
+
if (this.taskAdmissionsClosed) return;
|
|
1016
|
+
this.taskAdmissionsClosed = true;
|
|
1017
|
+
for (const admission of this.activeTaskAdmissions) {
|
|
1018
|
+
if (!admission.controller.signal.aborted) {
|
|
1019
|
+
admission.controller.abort(new BackgroundTaskAdmissionClosedError(admission.kind));
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
waitForTaskAdmissions(): Promise<void> {
|
|
1025
|
+
if (this.activeTaskAdmissions.size === 0) return Promise.resolve();
|
|
1026
|
+
return new Promise((resolve) => {
|
|
1027
|
+
this.taskAdmissionDrainWaiters.add(resolve);
|
|
1028
|
+
});
|
|
1029
|
+
}
|
|
1030
|
+
|
|
768
1031
|
setShuttingDown(value: boolean): void {
|
|
769
|
-
|
|
1032
|
+
if (value) {
|
|
1033
|
+
this.shuttingDown = true;
|
|
1034
|
+
this.closeTaskAdmissions();
|
|
1035
|
+
this.closeTerminalPublication('registry_shutdown');
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
// Publication and admission closure belong to one extension activation and
|
|
1039
|
+
// are one-way. Pi session replacement creates a fresh registry; an old
|
|
1040
|
+
// registry must not be reopened by a late lifecycle continuation.
|
|
1041
|
+
if (!this.terminalPublicationClosed && !this.taskAdmissionsClosed) this.shuttingDown = false;
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
closeTerminalPublication(reason: TerminalPublicationClosureReason): void {
|
|
1045
|
+
if (!this.terminalPublicationClosed) {
|
|
1046
|
+
this.terminalPublicationClosed = true;
|
|
1047
|
+
this.terminalPublicationCloseReason = reason;
|
|
1048
|
+
this.resolveTerminalPublicationClosedSignal(reason);
|
|
1049
|
+
}
|
|
1050
|
+
const effectiveReason = this.terminalPublicationCloseReason ?? reason;
|
|
1051
|
+
for (const task of this.tasks.values()) {
|
|
1052
|
+
if (task.terminalPublicationState === 'pending' && task.terminalEmitInFlight === true) {
|
|
1053
|
+
// A synchronous listener can close the service while emit() is still on
|
|
1054
|
+
// the stack. Dispose queued work now, but let the emitter's return/throw
|
|
1055
|
+
// settle the in-flight attempt exactly once.
|
|
1056
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
1057
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
1058
|
+
task.terminalPublishRetryHandle = undefined;
|
|
1059
|
+
}
|
|
1060
|
+
task.terminalPublicationGate = undefined;
|
|
1061
|
+
continue;
|
|
1062
|
+
}
|
|
1063
|
+
const shouldLog =
|
|
1064
|
+
task.terminalPublicationState === 'pending' &&
|
|
1065
|
+
task.status !== 'running' &&
|
|
1066
|
+
(task.terminalPublishAttempts > 0 ||
|
|
1067
|
+
task.terminalPublishRetryHandle !== undefined ||
|
|
1068
|
+
task.terminalPublicationGate !== undefined);
|
|
1069
|
+
this.abandonTerminalPublication(task, effectiveReason, undefined, shouldLog);
|
|
1070
|
+
}
|
|
1071
|
+
this.pruneOldTasks();
|
|
770
1072
|
}
|
|
771
1073
|
|
|
772
1074
|
allTasks(): BgTask[] {
|
|
@@ -777,6 +1079,186 @@ export class BackgroundTaskRegistry {
|
|
|
777
1079
|
return snapshot(task);
|
|
778
1080
|
}
|
|
779
1081
|
|
|
1082
|
+
hasCurrentReloadLease(): boolean {
|
|
1083
|
+
const lease = this.reloadShellLease;
|
|
1084
|
+
return lease !== undefined && this.reloadShellOwner?.isCurrentLease(lease) === true;
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
async stageReloadActivation(
|
|
1088
|
+
claim: ReloadShellActivationClaimV1,
|
|
1089
|
+
): Promise<ReloadShellHostAdapterV1> {
|
|
1090
|
+
if (claim.protocol !== RELOAD_SHELL_OWNER_PROTOCOL) {
|
|
1091
|
+
throw new ReloadSurvivalError(
|
|
1092
|
+
'pi_bg_reload_owner_protocol_incompatible',
|
|
1093
|
+
'activation claim does not use the supported reload shell owner protocol',
|
|
1094
|
+
);
|
|
1095
|
+
}
|
|
1096
|
+
const staged: ReloadableShellExecutionV1[] = [];
|
|
1097
|
+
const ids = new Set<string>();
|
|
1098
|
+
for (const execution of claim.executions) {
|
|
1099
|
+
if (
|
|
1100
|
+
execution.protocol !== RELOAD_SHELL_OWNER_PROTOCOL ||
|
|
1101
|
+
execution.task.reloadExecution !== execution ||
|
|
1102
|
+
execution.task.surviveReload !== true
|
|
1103
|
+
) {
|
|
1104
|
+
throw new ReloadSurvivalError(
|
|
1105
|
+
'pi_bg_reload_owner_protocol_incompatible',
|
|
1106
|
+
'activation claim contains an incompatible reload shell execution',
|
|
1107
|
+
);
|
|
1108
|
+
}
|
|
1109
|
+
if (ids.has(execution.task.id) || this.tasks.has(execution.task.id)) {
|
|
1110
|
+
throw new ReloadSurvivalError(
|
|
1111
|
+
'pi_bg_reload_owner_activation_conflict',
|
|
1112
|
+
`claimed task id ${execution.task.id} conflicts with the fresh registry`,
|
|
1113
|
+
);
|
|
1114
|
+
}
|
|
1115
|
+
ids.add(execution.task.id);
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
try {
|
|
1119
|
+
for (const execution of claim.executions) {
|
|
1120
|
+
const task = execution.task;
|
|
1121
|
+
task.reloadHostDeliveryInFlight = false;
|
|
1122
|
+
task.reloadHostDeliverySettled = false;
|
|
1123
|
+
task.reloadHostNotificationSettled = false;
|
|
1124
|
+
execution.updateLeaseAudit(claim.generation, (task.reloadSurvival?.handoffCount ?? 0) + 1);
|
|
1125
|
+
this.tasks.set(task.id, task);
|
|
1126
|
+
staged.push(execution);
|
|
1127
|
+
}
|
|
1128
|
+
await Promise.all(staged.map(async (execution) => this.writeMetadata(execution.task)));
|
|
1129
|
+
} catch (error) {
|
|
1130
|
+
for (const execution of staged) this.tasks.delete(execution.task.id);
|
|
1131
|
+
throw error;
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
let boundLease: ReloadShellActivationLeaseV1 | undefined;
|
|
1135
|
+
return {
|
|
1136
|
+
activationNonce: claim.activationNonce,
|
|
1137
|
+
onBound: (lease) => {
|
|
1138
|
+
if (
|
|
1139
|
+
lease.activationNonce !== claim.activationNonce ||
|
|
1140
|
+
lease.generation !== claim.generation ||
|
|
1141
|
+
lease.identityKey !== claim.identityKey
|
|
1142
|
+
) {
|
|
1143
|
+
throw new ReloadSurvivalError(
|
|
1144
|
+
'pi_bg_reload_owner_stale_claim',
|
|
1145
|
+
'committed lease does not match its staged activation claim',
|
|
1146
|
+
);
|
|
1147
|
+
}
|
|
1148
|
+
boundLease = lease;
|
|
1149
|
+
this.reloadShellLease = lease;
|
|
1150
|
+
this.reloadShellIdentity = claim.identity;
|
|
1151
|
+
},
|
|
1152
|
+
onChanged: (execution) => {
|
|
1153
|
+
const lease = boundLease;
|
|
1154
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
1155
|
+
this.onChange();
|
|
1156
|
+
},
|
|
1157
|
+
onTerminal: (execution) => {
|
|
1158
|
+
const lease = boundLease;
|
|
1159
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
1160
|
+
void this.deliverReloadTerminal(execution, lease);
|
|
1161
|
+
},
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
abortReloadActivation(claim: ReloadShellActivationClaimV1): void {
|
|
1166
|
+
for (const execution of claim.executions) {
|
|
1167
|
+
const task = execution.task;
|
|
1168
|
+
if (this.tasks.get(task.id) !== task) continue;
|
|
1169
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
1170
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
1171
|
+
task.terminalPublishRetryHandle = undefined;
|
|
1172
|
+
}
|
|
1173
|
+
task.terminalPublicationGate = undefined;
|
|
1174
|
+
task.terminalPublishInFlight = false;
|
|
1175
|
+
this.tasks.delete(task.id);
|
|
1176
|
+
}
|
|
1177
|
+
if (this.reloadShellLease?.activationNonce === claim.activationNonce) {
|
|
1178
|
+
this.reloadShellLease = undefined;
|
|
1179
|
+
this.reloadShellIdentity = undefined;
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
|
|
1183
|
+
prepareReloadHandoff(lease: ReloadShellActivationLeaseV1): readonly BgTask[] {
|
|
1184
|
+
if (this.reloadShellOwner === undefined || this.reloadShellLease !== lease) {
|
|
1185
|
+
throw new ReloadSurvivalError(
|
|
1186
|
+
'pi_bg_reload_owner_stale_claim',
|
|
1187
|
+
'registry does not own the requested reload activation lease',
|
|
1188
|
+
);
|
|
1189
|
+
}
|
|
1190
|
+
const executions = this.reloadShellOwner.beginReloadHandoff(lease);
|
|
1191
|
+
const tasks: BgTask[] = [];
|
|
1192
|
+
for (const execution of executions) {
|
|
1193
|
+
const task = execution.task;
|
|
1194
|
+
if (this.tasks.get(task.id) !== task) {
|
|
1195
|
+
throw new ReloadSurvivalError(
|
|
1196
|
+
'pi_bg_reload_owner_stale_claim',
|
|
1197
|
+
`registry no longer owns survivor ${task.id}`,
|
|
1198
|
+
);
|
|
1199
|
+
}
|
|
1200
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
1201
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
1202
|
+
task.terminalPublishRetryHandle = undefined;
|
|
1203
|
+
}
|
|
1204
|
+
task.terminalPublicationGate = undefined;
|
|
1205
|
+
task.terminalPublishInFlight = false;
|
|
1206
|
+
task.reloadHostDeliveryInFlight = false;
|
|
1207
|
+
task.reloadHostDeliverySettled = false;
|
|
1208
|
+
task.reloadHostNotificationSettled = false;
|
|
1209
|
+
this.tasks.delete(task.id);
|
|
1210
|
+
tasks.push(task);
|
|
1211
|
+
}
|
|
1212
|
+
this.reloadShellLease = undefined;
|
|
1213
|
+
this.reloadShellIdentity = undefined;
|
|
1214
|
+
return Object.freeze(tasks);
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
async waitForReloadHostSettlement(timeoutMs = this.stopWaitMs): Promise<void> {
|
|
1218
|
+
const deadline = Date.now() + timeoutMs;
|
|
1219
|
+
while (true) {
|
|
1220
|
+
const unsettled = [...this.tasks.values()].filter(
|
|
1221
|
+
(task) =>
|
|
1222
|
+
task.reloadExecution?.phase === 'terminal' &&
|
|
1223
|
+
(task.reloadHostNotificationSettled !== true ||
|
|
1224
|
+
task.terminalPublicationState === 'pending'),
|
|
1225
|
+
);
|
|
1226
|
+
if (unsettled.length === 0) return;
|
|
1227
|
+
const remaining = deadline - Date.now();
|
|
1228
|
+
if (remaining <= 0) {
|
|
1229
|
+
throw new Error(
|
|
1230
|
+
`Timed out waiting for reload shell host settlement: ${unsettled.map((task) => task.id).join(', ')}`,
|
|
1231
|
+
);
|
|
1232
|
+
}
|
|
1233
|
+
await new Promise<void>((resolve) => setTimeout(resolve, Math.min(10, remaining)));
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
releaseReloadActivation(lease: ReloadShellActivationLeaseV1): void {
|
|
1238
|
+
if (this.reloadShellOwner === undefined) return;
|
|
1239
|
+
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) return;
|
|
1240
|
+
this.reloadShellOwner.releaseActivation(lease);
|
|
1241
|
+
this.reloadShellLease = undefined;
|
|
1242
|
+
this.reloadShellIdentity = undefined;
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
currentReloadLease(): ReloadShellActivationLeaseV1 | undefined {
|
|
1246
|
+
return this.hasCurrentReloadLease() ? this.reloadShellLease : undefined;
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
private ownsReloadExecution(
|
|
1250
|
+
execution: ReloadableShellExecutionV1,
|
|
1251
|
+
lease: ReloadShellActivationLeaseV1 | undefined,
|
|
1252
|
+
): lease is ReloadShellActivationLeaseV1 {
|
|
1253
|
+
return (
|
|
1254
|
+
lease !== undefined &&
|
|
1255
|
+
this.reloadShellLease === lease &&
|
|
1256
|
+
this.reloadShellOwner?.isCurrentLease(lease) === true &&
|
|
1257
|
+
this.tasks.get(execution.task.id) === execution.task &&
|
|
1258
|
+
execution.task.reloadExecution === execution
|
|
1259
|
+
);
|
|
1260
|
+
}
|
|
1261
|
+
|
|
780
1262
|
async ensureRuntimeDir(ctx: BackgroundTaskContext): Promise<RuntimeDir> {
|
|
781
1263
|
if (this.runtimeDir) return this.runtimeDir;
|
|
782
1264
|
const sessionId = sanitizePathSegment(ctx.sessionId ?? `session-${String(process.pid)}`);
|
|
@@ -788,25 +1270,187 @@ export class BackgroundTaskRegistry {
|
|
|
788
1270
|
return this.runtimeDir;
|
|
789
1271
|
}
|
|
790
1272
|
|
|
1273
|
+
private async destroyTaskStream(task: BgTask): Promise<void> {
|
|
1274
|
+
const stream = task.stream;
|
|
1275
|
+
if (stream === undefined || stream.closed) return;
|
|
1276
|
+
await new Promise<void>((resolve) => {
|
|
1277
|
+
const closed = () => {
|
|
1278
|
+
stream.off('close', closed);
|
|
1279
|
+
resolve();
|
|
1280
|
+
};
|
|
1281
|
+
stream.once('close', closed);
|
|
1282
|
+
if (!stream.destroyed) stream.destroy();
|
|
1283
|
+
if (stream.closed) closed();
|
|
1284
|
+
});
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
private async discardUnspawnedTask(task: BgTask, paths: readonly string[]): Promise<void> {
|
|
1288
|
+
this.tasks.delete(task.id);
|
|
1289
|
+
task.finalized = true;
|
|
1290
|
+
task.status = 'failed';
|
|
1291
|
+
if (task.timeoutHandle !== undefined) clearTimeout(task.timeoutHandle);
|
|
1292
|
+
if (task.killEscalationTimer !== undefined) clearTimeout(task.killEscalationTimer);
|
|
1293
|
+
await this.destroyTaskStream(task);
|
|
1294
|
+
const removals = await Promise.allSettled(paths.map((path) => rm(path, { force: true })));
|
|
1295
|
+
const failures: Error[] = [];
|
|
1296
|
+
for (let index = 0; index < removals.length; index++) {
|
|
1297
|
+
const result = removals[index];
|
|
1298
|
+
if (result?.status !== 'rejected') continue;
|
|
1299
|
+
const path = paths[index] ?? '<unknown admission artifact>';
|
|
1300
|
+
failures.push(
|
|
1301
|
+
new Error(
|
|
1302
|
+
`Failed to remove interrupted admission artifact ${path}: ${BackgroundTaskRegistry.errorMessage(result.reason)}`,
|
|
1303
|
+
),
|
|
1304
|
+
);
|
|
1305
|
+
}
|
|
1306
|
+
if (failures.length > 0) {
|
|
1307
|
+
throw new AggregateError(failures, 'Interrupted task admission artifact cleanup failed');
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
private attestedGitOptions(admission?: TaskAdmission): GitCommandOptions {
|
|
1312
|
+
return {
|
|
1313
|
+
signal: admission?.controller.signal,
|
|
1314
|
+
deadlineAt: admission?.deadlineAt ?? Date.now() + this.taskAdmissionTimeoutMs,
|
|
1315
|
+
killProcess: this.killProcess,
|
|
1316
|
+
killTree: this.killTree,
|
|
1317
|
+
platform: this.platform,
|
|
1318
|
+
env: this.env,
|
|
1319
|
+
killGraceMs: this.attestedGitKillGraceMs,
|
|
1320
|
+
maxOutputBytes: this.attestedGitMaxOutputBytes,
|
|
1321
|
+
...(this.attestedGitSpawn === undefined ? {} : { spawn: this.attestedGitSpawn }),
|
|
1322
|
+
};
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
private captureSpawnedChild(task: BgTask, child: BackgroundTaskChildProcess): void {
|
|
1326
|
+
task.child = child;
|
|
1327
|
+
task.pid = child.pid;
|
|
1328
|
+
if (
|
|
1329
|
+
this.platform !== 'win32' &&
|
|
1330
|
+
child.pid !== undefined &&
|
|
1331
|
+
Number.isSafeInteger(child.pid) &&
|
|
1332
|
+
child.pid > 0
|
|
1333
|
+
) {
|
|
1334
|
+
// Capture detached-group ownership once, directly from this spawn. Never
|
|
1335
|
+
// reconstruct signal authority from mutable task metadata or a later PID.
|
|
1336
|
+
task.ownedPosixProcessGroupId = child.pid;
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
private bindOwnedTaskToAdmission(task: BgTask, admission: TaskAdmission): () => void {
|
|
1341
|
+
const cancelOwnedTask = (): void => {
|
|
1342
|
+
this.stopOwnedTaskAfterAdmissionCancellation(task, this.taskAdmissionError(admission));
|
|
1343
|
+
};
|
|
1344
|
+
admission.controller.signal.addEventListener('abort', cancelOwnedTask, { once: true });
|
|
1345
|
+
if (admission.controller.signal.aborted) cancelOwnedTask();
|
|
1346
|
+
return () => {
|
|
1347
|
+
admission.controller.signal.removeEventListener('abort', cancelOwnedTask);
|
|
1348
|
+
};
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
private stopOwnedTaskAfterAdmissionCancellation(task: BgTask, error: Error): void {
|
|
1352
|
+
if (task.status !== 'running') return;
|
|
1353
|
+
task.killKind = error instanceof BackgroundTaskAdmissionTimeoutError ? 'timeout' : 'shutdown';
|
|
1354
|
+
task.error = error.message;
|
|
1355
|
+
try {
|
|
1356
|
+
this.requestKill(task, 'SIGTERM');
|
|
1357
|
+
} catch (killError) {
|
|
1358
|
+
this.logger.error(
|
|
1359
|
+
`[background-tasks] failed to stop ${task.id} after admission cancellation:`,
|
|
1360
|
+
killError,
|
|
1361
|
+
);
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
|
|
791
1365
|
async startTask(
|
|
792
1366
|
ctx: BackgroundTaskContext,
|
|
793
1367
|
command: string,
|
|
794
1368
|
options: StartTaskOptions = {},
|
|
1369
|
+
): Promise<BgTask> {
|
|
1370
|
+
const hasSurvival = Object.prototype.hasOwnProperty.call(options, 'surviveReload');
|
|
1371
|
+
if (hasSurvival && typeof options.surviveReload !== 'boolean') {
|
|
1372
|
+
throw new ReloadSurvivalError(
|
|
1373
|
+
'pi_bg_survive_reload_invalid',
|
|
1374
|
+
'surviveReload must be a boolean when present',
|
|
1375
|
+
);
|
|
1376
|
+
}
|
|
1377
|
+
const surviveReload = options.surviveReload === true;
|
|
1378
|
+
if (surviveReload && options.isAgent === true) {
|
|
1379
|
+
throw new ReloadSurvivalError(
|
|
1380
|
+
'pi_bg_survive_reload_requires_non_agent',
|
|
1381
|
+
'surviveReload requires isAgent:false',
|
|
1382
|
+
);
|
|
1383
|
+
}
|
|
1384
|
+
const reloadLease = surviveReload ? this.currentReloadLease() : undefined;
|
|
1385
|
+
if (surviveReload && reloadLease === undefined) {
|
|
1386
|
+
throw new ReloadSurvivalError(
|
|
1387
|
+
'pi_bg_reload_owner_unavailable',
|
|
1388
|
+
'no successfully bound same-process reload owner activation is available',
|
|
1389
|
+
);
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
const admission = this.beginTaskAdmission('a background task');
|
|
1393
|
+
try {
|
|
1394
|
+
if (surviveReload && reloadLease !== undefined) {
|
|
1395
|
+
return await this.startReloadableTaskAdmitted(
|
|
1396
|
+
ctx,
|
|
1397
|
+
command,
|
|
1398
|
+
options,
|
|
1399
|
+
admission,
|
|
1400
|
+
reloadLease,
|
|
1401
|
+
);
|
|
1402
|
+
}
|
|
1403
|
+
return await this.startTaskAdmitted(ctx, command, options, admission);
|
|
1404
|
+
} finally {
|
|
1405
|
+
this.releaseTaskAdmission(admission);
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
private async startReloadableTaskAdmitted(
|
|
1410
|
+
ctx: BackgroundTaskContext,
|
|
1411
|
+
command: string,
|
|
1412
|
+
options: StartTaskOptions,
|
|
1413
|
+
admission: TaskAdmission,
|
|
1414
|
+
lease: ReloadShellActivationLeaseV1,
|
|
795
1415
|
): Promise<BgTask> {
|
|
796
1416
|
const normalizedCommand = command.trim();
|
|
797
1417
|
if (!normalizedCommand) throw new Error('Background command is empty');
|
|
798
|
-
if (
|
|
799
|
-
throw new
|
|
1418
|
+
if (options.isAgent === true) {
|
|
1419
|
+
throw new ReloadSurvivalError(
|
|
1420
|
+
'pi_bg_survive_reload_requires_non_agent',
|
|
1421
|
+
'surviveReload requires isAgent:false',
|
|
1422
|
+
);
|
|
1423
|
+
}
|
|
1424
|
+
if (
|
|
1425
|
+
this.reloadShellOwner === undefined ||
|
|
1426
|
+
this.reloadShellIdentity === undefined ||
|
|
1427
|
+
this.reloadShellLease !== lease ||
|
|
1428
|
+
!this.reloadShellOwner.isCurrentLease(lease)
|
|
1429
|
+
) {
|
|
1430
|
+
throw new ReloadSurvivalError(
|
|
1431
|
+
'pi_bg_reload_owner_unavailable',
|
|
1432
|
+
'reload owner activation became unavailable before launch',
|
|
1433
|
+
);
|
|
1434
|
+
}
|
|
1435
|
+
if (ctx.sessionId !== this.reloadShellIdentity.sessionId) {
|
|
1436
|
+
throw new ReloadSurvivalError(
|
|
1437
|
+
'pi_bg_reload_owner_stale_claim',
|
|
1438
|
+
'launch context session id does not match the bound reload owner identity',
|
|
1439
|
+
);
|
|
1440
|
+
}
|
|
1441
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
800
1442
|
|
|
801
|
-
const
|
|
802
|
-
const
|
|
803
|
-
const
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
1443
|
+
const shellPolicy = this.resolvedShellPolicy();
|
|
1444
|
+
const invocation = shellInvocationForPolicy(normalizedCommand, shellPolicy);
|
|
1445
|
+
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
1446
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1447
|
+
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) {
|
|
1448
|
+
throw new ReloadSurvivalError(
|
|
1449
|
+
'pi_bg_reload_owner_stale_claim',
|
|
1450
|
+
'reload owner activation changed during task preflight',
|
|
1451
|
+
);
|
|
1452
|
+
}
|
|
808
1453
|
|
|
809
|
-
const dir = await this.ensureRuntimeDir(ctx);
|
|
810
1454
|
const id = this.makeTaskIdFn();
|
|
811
1455
|
const outputAbsPath = join(dir.abs, `${id}.output`);
|
|
812
1456
|
const metadataAbsPath = join(dir.abs, `${id}.json`);
|
|
@@ -824,7 +1468,6 @@ export class BackgroundTaskRegistry {
|
|
|
824
1468
|
const trimmedDescription = options.description?.trim();
|
|
825
1469
|
const description =
|
|
826
1470
|
trimmedDescription && trimmedDescription.length > 0 ? trimmedDescription : undefined;
|
|
827
|
-
|
|
828
1471
|
const task: BgTask = {
|
|
829
1472
|
id,
|
|
830
1473
|
name: taskName,
|
|
@@ -839,69 +1482,269 @@ export class BackgroundTaskRegistry {
|
|
|
839
1482
|
exitCode: undefined,
|
|
840
1483
|
pid: undefined,
|
|
841
1484
|
bytesWritten: 0,
|
|
842
|
-
isAgent,
|
|
1485
|
+
isAgent: false,
|
|
1486
|
+
surviveReload: true,
|
|
843
1487
|
notified: false,
|
|
844
1488
|
notifyOnCompletion: options.notifyOnCompletion ?? true,
|
|
845
1489
|
triggerOnCompletion: options.triggerOnCompletion ?? false,
|
|
846
1490
|
timeoutSeconds,
|
|
1491
|
+
terminalPublished: false,
|
|
1492
|
+
terminalPublicationState: 'pending',
|
|
1493
|
+
terminalPublishAttempts: 0,
|
|
847
1494
|
terminalPublicationGate: options.terminalPublicationGate,
|
|
1495
|
+
shellPolicy: shellPolicySnapshot(shellPolicy),
|
|
848
1496
|
waiters: [],
|
|
849
1497
|
};
|
|
1498
|
+
const launchNonce = randomBytes(16).toString('hex');
|
|
1499
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
850
1500
|
this.tasks.set(id, task);
|
|
851
1501
|
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
1502
|
+
let execution: ReloadableShellExecutionV1 | undefined;
|
|
1503
|
+
let registered = false;
|
|
1504
|
+
let committed = false;
|
|
1505
|
+
let abortListener: (() => void) | undefined;
|
|
1506
|
+
try {
|
|
1507
|
+
execution = createReloadableShellExecutionV1({
|
|
1508
|
+
task,
|
|
1509
|
+
identity: this.reloadShellIdentity,
|
|
1510
|
+
lease,
|
|
1511
|
+
launchNonce,
|
|
1512
|
+
invocation,
|
|
1513
|
+
spawn: this.spawn,
|
|
1514
|
+
killProcess: this.killProcess,
|
|
1515
|
+
killTree: this.killTree,
|
|
1516
|
+
platform: this.platform,
|
|
1517
|
+
env: this.env,
|
|
1518
|
+
maxOutputBytes: this.maxOutputBytes,
|
|
1519
|
+
killGraceMs: this.killGraceMs,
|
|
1520
|
+
stopWaitMs: this.stopWaitMs,
|
|
1521
|
+
now: this.now,
|
|
1522
|
+
logger: this.logger,
|
|
1523
|
+
});
|
|
1524
|
+
this.reloadShellOwner.registerExecution(lease, execution);
|
|
1525
|
+
registered = true;
|
|
1526
|
+
abortListener = () => {
|
|
1527
|
+
if (execution === undefined) return;
|
|
1528
|
+
const error = this.taskAdmissionError(admission);
|
|
1529
|
+
execution.failAdmission(error);
|
|
1530
|
+
const kind: KillKind =
|
|
1531
|
+
error instanceof BackgroundTaskAdmissionTimeoutError ? 'timeout' : 'shutdown';
|
|
1532
|
+
void execution.requestStop(kind, error.message).catch((stopError: unknown) => {
|
|
1533
|
+
this.logger.error(
|
|
1534
|
+
`[background-tasks] failed to stop reloadable task ${task.id} after admission cancellation:`,
|
|
1535
|
+
stopError,
|
|
867
1536
|
);
|
|
1537
|
+
});
|
|
1538
|
+
};
|
|
1539
|
+
admission.controller.signal.addEventListener('abort', abortListener, { once: true });
|
|
1540
|
+
if (admission.controller.signal.aborted) abortListener();
|
|
1541
|
+
|
|
1542
|
+
await this.awaitTaskAdmissionBoundary(
|
|
1543
|
+
execution.commitInitialMetadata(admission.controller.signal),
|
|
1544
|
+
admission,
|
|
1545
|
+
);
|
|
1546
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1547
|
+
if (this.reloadShellLease !== lease || !this.reloadShellOwner.isCurrentLease(lease)) {
|
|
1548
|
+
throw new ReloadSurvivalError(
|
|
1549
|
+
'pi_bg_reload_owner_stale_claim',
|
|
1550
|
+
'reload owner activation changed before admission commit',
|
|
1551
|
+
);
|
|
1552
|
+
}
|
|
1553
|
+
this.reloadShellOwner.markAdmissionCommitted(lease, execution);
|
|
1554
|
+
committed = true;
|
|
1555
|
+
this.onChange();
|
|
1556
|
+
return task;
|
|
1557
|
+
} catch (error) {
|
|
1558
|
+
const primary = admission.controller.signal.aborted
|
|
1559
|
+
? this.taskAdmissionError(admission)
|
|
1560
|
+
: error instanceof Error
|
|
1561
|
+
? error
|
|
1562
|
+
: new Error(String(error));
|
|
1563
|
+
execution?.failAdmission(primary);
|
|
1564
|
+
let cleanupError: unknown;
|
|
1565
|
+
if (execution !== undefined) {
|
|
1566
|
+
try {
|
|
1567
|
+
if (task.status === 'running') {
|
|
1568
|
+
await execution.requestStop(
|
|
1569
|
+
admission.controller.signal.aborted &&
|
|
1570
|
+
primary instanceof BackgroundTaskAdmissionTimeoutError
|
|
1571
|
+
? 'timeout'
|
|
1572
|
+
: 'shutdown',
|
|
1573
|
+
primary.message,
|
|
1574
|
+
);
|
|
1575
|
+
}
|
|
1576
|
+
} catch (stopError) {
|
|
1577
|
+
cleanupError = stopError;
|
|
868
1578
|
}
|
|
869
1579
|
}
|
|
870
|
-
|
|
1580
|
+
if (registered && !committed && execution !== undefined) {
|
|
1581
|
+
if (this.reloadShellOwner.isCurrentLease(lease)) {
|
|
1582
|
+
this.reloadShellOwner.releaseExecution(lease, execution);
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
this.tasks.delete(task.id);
|
|
1586
|
+
if (execution === undefined) {
|
|
1587
|
+
const removals = await Promise.allSettled([
|
|
1588
|
+
rm(outputAbsPath, { force: true }),
|
|
1589
|
+
rm(metadataAbsPath, { force: true }),
|
|
1590
|
+
]);
|
|
1591
|
+
const removalFailure = removals.find((result) => result.status === 'rejected');
|
|
1592
|
+
if (removalFailure?.status === 'rejected') cleanupError = removalFailure.reason;
|
|
1593
|
+
}
|
|
1594
|
+
if (admission.controller.signal.aborted) {
|
|
1595
|
+
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
1596
|
+
}
|
|
1597
|
+
if (cleanupError !== undefined) {
|
|
1598
|
+
throw new AggregateError(
|
|
1599
|
+
[primary, cleanupError],
|
|
1600
|
+
`Failed to start reloadable background task and cleanup also failed: ${BackgroundTaskRegistry.errorMessage(cleanupError)}`,
|
|
1601
|
+
);
|
|
1602
|
+
}
|
|
1603
|
+
throw new Error(`Failed to start background task: ${primary.message}`);
|
|
1604
|
+
} finally {
|
|
1605
|
+
if (abortListener !== undefined) {
|
|
1606
|
+
admission.controller.signal.removeEventListener('abort', abortListener);
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
private async startTaskAdmitted(
|
|
1612
|
+
ctx: BackgroundTaskContext,
|
|
1613
|
+
command: string,
|
|
1614
|
+
options: StartTaskOptions,
|
|
1615
|
+
admission: TaskAdmission,
|
|
1616
|
+
): Promise<BgTask> {
|
|
1617
|
+
const normalizedCommand = command.trim();
|
|
1618
|
+
if (!normalizedCommand) throw new Error('Background command is empty');
|
|
1619
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
871
1620
|
|
|
1621
|
+
const isAgent = options.isAgent ?? false;
|
|
1622
|
+
const shellPolicy = this.resolvedShellPolicy();
|
|
1623
|
+
const baseInvocation = shellInvocationForPolicy(normalizedCommand, shellPolicy);
|
|
1624
|
+
const piTelemetryRequested = isAgent && commandMayLaunchPiAgent(normalizedCommand, this.env);
|
|
1625
|
+
const piTelemetryLaunch =
|
|
1626
|
+
piTelemetryRequested && shellPolicy.supportsPosixFunctionWrapper
|
|
1627
|
+
? resolvePiLaunch({ platform: this.platform })
|
|
1628
|
+
: undefined;
|
|
1629
|
+
|
|
1630
|
+
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
1631
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1632
|
+
const id = this.makeTaskIdFn();
|
|
1633
|
+
const outputAbsPath = join(dir.abs, `${id}.output`);
|
|
1634
|
+
const metadataAbsPath = join(dir.abs, `${id}.json`);
|
|
1635
|
+
const outputPath = join(dir.display, `${id}.output`);
|
|
1636
|
+
let commandToSpawn = normalizedCommand;
|
|
1637
|
+
let wrapperAbsPath: string | undefined;
|
|
872
1638
|
try {
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
const wrapperAbsPath = join(dir.abs, `${id}.pi-telemetry-wrapper.cjs`);
|
|
1639
|
+
if (piTelemetryRequested && shellPolicy.supportsPosixFunctionWrapper) {
|
|
1640
|
+
if (piTelemetryLaunch === undefined)
|
|
1641
|
+
throw new Error('Pi telemetry launch spec was not resolved');
|
|
1642
|
+
wrapperAbsPath = join(dir.abs, `${id}.pi-telemetry-wrapper.cjs`);
|
|
1643
|
+
try {
|
|
879
1644
|
await writeFile(
|
|
880
1645
|
wrapperAbsPath,
|
|
881
1646
|
createPiTelemetryWrapperSource(buildModelWindowIndex(ctx), piTelemetryLaunch),
|
|
882
|
-
'utf8',
|
|
1647
|
+
{ encoding: 'utf8', signal: admission.controller.signal },
|
|
883
1648
|
);
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
task.telemetryUnavailableReason = WIN32_CMD_PI_TELEMETRY_UNAVAILABLE_REASON;
|
|
1649
|
+
} catch (error) {
|
|
1650
|
+
if (admission.controller.signal.aborted) throw this.taskAdmissionError(admission);
|
|
1651
|
+
throw error;
|
|
888
1652
|
}
|
|
1653
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1654
|
+
commandToSpawn = `pi() { ${shellQuote(process.execPath)} ${shellQuote(wrapperAbsPath)} "$@"; }\n${normalizedCommand}`;
|
|
889
1655
|
}
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
1656
|
+
} catch (error) {
|
|
1657
|
+
if (wrapperAbsPath !== undefined) await rm(wrapperAbsPath, { force: true });
|
|
1658
|
+
throw error;
|
|
1659
|
+
}
|
|
1660
|
+
const invocation =
|
|
1661
|
+
commandToSpawn === normalizedCommand
|
|
1662
|
+
? baseInvocation
|
|
1663
|
+
: shellInvocationForPolicy(commandToSpawn, shellPolicy);
|
|
1664
|
+
const timeoutSeconds =
|
|
1665
|
+
typeof options.timeoutSeconds === 'number' &&
|
|
1666
|
+
Number.isFinite(options.timeoutSeconds) &&
|
|
1667
|
+
options.timeoutSeconds > 0
|
|
1668
|
+
? Math.floor(options.timeoutSeconds)
|
|
1669
|
+
: undefined;
|
|
1670
|
+
const taskName =
|
|
1671
|
+
normalizeTaskName(options.name) ??
|
|
1672
|
+
normalizeTaskName(options.description) ??
|
|
1673
|
+
deriveTaskNameFromCommand(normalizedCommand);
|
|
1674
|
+
const trimmedDescription = options.description?.trim();
|
|
1675
|
+
const description =
|
|
1676
|
+
trimmedDescription && trimmedDescription.length > 0 ? trimmedDescription : undefined;
|
|
1677
|
+
|
|
1678
|
+
const task: BgTask = {
|
|
1679
|
+
id,
|
|
1680
|
+
name: taskName,
|
|
1681
|
+
command: normalizedCommand,
|
|
1682
|
+
description,
|
|
1683
|
+
status: 'running',
|
|
1684
|
+
outputPath,
|
|
1685
|
+
outputAbsPath,
|
|
1686
|
+
metadataAbsPath,
|
|
1687
|
+
cwd: ctx.cwd,
|
|
1688
|
+
startTime: this.now(),
|
|
1689
|
+
exitCode: undefined,
|
|
1690
|
+
pid: undefined,
|
|
1691
|
+
bytesWritten: 0,
|
|
1692
|
+
isAgent,
|
|
1693
|
+
surviveReload: false,
|
|
1694
|
+
notified: false,
|
|
1695
|
+
notifyOnCompletion: options.notifyOnCompletion ?? true,
|
|
1696
|
+
triggerOnCompletion: options.triggerOnCompletion ?? false,
|
|
1697
|
+
timeoutSeconds,
|
|
1698
|
+
terminalPublished: false,
|
|
1699
|
+
terminalPublicationState: 'pending',
|
|
1700
|
+
terminalPublishAttempts: 0,
|
|
1701
|
+
terminalPublicationGate: options.terminalPublicationGate,
|
|
1702
|
+
shellPolicy: shellPolicySnapshot(shellPolicy),
|
|
1703
|
+
waiters: [],
|
|
1704
|
+
};
|
|
1705
|
+
if (commandToSpawn !== normalizedCommand) task.telemetryWrapped = true;
|
|
1706
|
+
if (piTelemetryRequested && !shellPolicy.supportsPosixFunctionWrapper) {
|
|
1707
|
+
task.telemetryUnavailableReason =
|
|
1708
|
+
shellPolicy.dialect === 'cmd'
|
|
1709
|
+
? WIN32_CMD_PI_TELEMETRY_UNAVAILABLE_REASON
|
|
1710
|
+
: NON_POSIX_SHELL_PI_TELEMETRY_UNAVAILABLE_REASON;
|
|
1711
|
+
}
|
|
1712
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1713
|
+
this.tasks.set(id, task);
|
|
1714
|
+
|
|
1715
|
+
const stream = createWriteStream(outputAbsPath, { flags: 'a', encoding: 'utf8' });
|
|
1716
|
+
task.stream = stream;
|
|
1717
|
+
stream.on('error', (error) => {
|
|
1718
|
+
task.error = `Output file write failed: ${error.message}`;
|
|
1719
|
+
if (task.status === 'running') {
|
|
1720
|
+
task.killKind = 'output_cap';
|
|
1721
|
+
try {
|
|
1722
|
+
this.requestKill(task, 'SIGTERM');
|
|
1723
|
+
} catch (killError) {
|
|
1724
|
+
void this.finalizeTask(
|
|
1725
|
+
task,
|
|
1726
|
+
'failed',
|
|
1727
|
+
null,
|
|
1728
|
+
undefined,
|
|
1729
|
+
`${task.error}; kill failed: ${killError instanceof Error ? killError.message : String(killError)}`,
|
|
1730
|
+
);
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
});
|
|
1734
|
+
|
|
1735
|
+
let unbindAdmissionCancellation = (): void => undefined;
|
|
1736
|
+
try {
|
|
1737
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
1738
|
+
const child = this.spawn(invocation.shell, invocation.args, {
|
|
1739
|
+
cwd: ctx.cwd,
|
|
1740
|
+
detached: this.platform !== 'win32',
|
|
897
1741
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
898
1742
|
env: this.env,
|
|
899
1743
|
windowsHide: true,
|
|
900
1744
|
windowsVerbatimArguments: invocation.windowsVerbatimArguments,
|
|
901
1745
|
});
|
|
902
1746
|
|
|
903
|
-
task
|
|
904
|
-
task.pid = child.pid;
|
|
1747
|
+
this.captureSpawnedChild(task, child);
|
|
905
1748
|
|
|
906
1749
|
child.stdout?.on('data', (data) => {
|
|
907
1750
|
this.appendChildOutput(task, data, 'stdout');
|
|
@@ -956,14 +1799,39 @@ export class BackgroundTaskRegistry {
|
|
|
956
1799
|
}, timeoutSeconds * 1000);
|
|
957
1800
|
}
|
|
958
1801
|
|
|
959
|
-
|
|
1802
|
+
unbindAdmissionCancellation = this.bindOwnedTaskToAdmission(task, admission);
|
|
1803
|
+
await this.awaitTaskAdmissionBoundary(
|
|
1804
|
+
this.writeMetadata(task, admission.controller.signal),
|
|
1805
|
+
admission,
|
|
1806
|
+
);
|
|
1807
|
+
this.assertTaskAdmissionOpen('a background task', admission);
|
|
960
1808
|
this.onChange();
|
|
961
1809
|
return task;
|
|
962
1810
|
} catch (error) {
|
|
1811
|
+
if (admission.controller.signal.aborted) {
|
|
1812
|
+
const admissionError = this.taskAdmissionError(admission);
|
|
1813
|
+
let cleanupError: unknown;
|
|
1814
|
+
if (task.child === undefined) {
|
|
1815
|
+
try {
|
|
1816
|
+
await this.discardUnspawnedTask(task, [
|
|
1817
|
+
outputAbsPath,
|
|
1818
|
+
metadataAbsPath,
|
|
1819
|
+
...(wrapperAbsPath === undefined ? [] : [wrapperAbsPath]),
|
|
1820
|
+
]);
|
|
1821
|
+
} catch (cleanupFailure) {
|
|
1822
|
+
cleanupError = cleanupFailure;
|
|
1823
|
+
}
|
|
1824
|
+
} else {
|
|
1825
|
+
this.stopOwnedTaskAfterAdmissionCancellation(task, admissionError);
|
|
1826
|
+
}
|
|
1827
|
+
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
1828
|
+
}
|
|
963
1829
|
const message = error instanceof Error ? error.message : String(error);
|
|
964
1830
|
this.writeNotice(task, `\n[background task spawn exception: ${message}]\n`);
|
|
965
1831
|
await this.finalizeTask(task, 'failed', null, undefined, message);
|
|
966
1832
|
throw new Error(`Failed to start background task: ${message}`);
|
|
1833
|
+
} finally {
|
|
1834
|
+
unbindAdmissionCancellation();
|
|
967
1835
|
}
|
|
968
1836
|
}
|
|
969
1837
|
|
|
@@ -977,14 +1845,66 @@ export class BackgroundTaskRegistry {
|
|
|
977
1845
|
ctx: BackgroundTaskContext,
|
|
978
1846
|
request: StartManagedTaskOptions,
|
|
979
1847
|
): Promise<BgTask> {
|
|
980
|
-
|
|
981
|
-
|
|
1848
|
+
rejectSurvivalForTaskKind(request, 'managed background tasks');
|
|
1849
|
+
let admission: TaskAdmission | undefined;
|
|
1850
|
+
try {
|
|
1851
|
+
admission = this.beginTaskAdmission('a managed background task');
|
|
1852
|
+
return await this.startManagedTaskAdmitted(ctx, request, admission);
|
|
1853
|
+
} catch (error) {
|
|
1854
|
+
const admissionError =
|
|
1855
|
+
admission?.controller.signal.aborted === true
|
|
1856
|
+
? this.taskAdmissionError(admission)
|
|
1857
|
+
: error instanceof BackgroundTaskAdmissionClosedError
|
|
1858
|
+
? error
|
|
1859
|
+
: undefined;
|
|
1860
|
+
if (admissionError !== undefined) {
|
|
1861
|
+
const task = admission === undefined ? undefined : this.tasks.get(request.id);
|
|
1862
|
+
if (task === undefined) {
|
|
1863
|
+
try {
|
|
1864
|
+
request.cancel();
|
|
1865
|
+
} catch (cancelError) {
|
|
1866
|
+
this.logger.error(
|
|
1867
|
+
`[background-tasks] managed preflight cancellation failed for ${request.id}:`,
|
|
1868
|
+
cancelError,
|
|
1869
|
+
);
|
|
1870
|
+
}
|
|
1871
|
+
// The workflow promise owns child/artifact cleanup. Do not release a
|
|
1872
|
+
// pre-insertion admission while that cleanup can still run late.
|
|
1873
|
+
await request.completion.then(
|
|
1874
|
+
() => undefined,
|
|
1875
|
+
() => undefined,
|
|
1876
|
+
);
|
|
1877
|
+
} else {
|
|
1878
|
+
this.stopOwnedTaskAfterAdmissionCancellation(task, admissionError);
|
|
1879
|
+
await request.completion.then(
|
|
1880
|
+
() => undefined,
|
|
1881
|
+
() => undefined,
|
|
1882
|
+
);
|
|
1883
|
+
}
|
|
1884
|
+
if (admission !== undefined) {
|
|
1885
|
+
throw this.surfacedTaskAdmissionError(admission, error);
|
|
1886
|
+
}
|
|
1887
|
+
throw admissionError;
|
|
1888
|
+
}
|
|
1889
|
+
throw error;
|
|
1890
|
+
} finally {
|
|
1891
|
+
if (admission !== undefined) this.releaseTaskAdmission(admission);
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
|
|
1895
|
+
private async startManagedTaskAdmitted(
|
|
1896
|
+
ctx: BackgroundTaskContext,
|
|
1897
|
+
request: StartManagedTaskOptions,
|
|
1898
|
+
admission: TaskAdmission,
|
|
1899
|
+
): Promise<BgTask> {
|
|
1900
|
+
this.assertTaskAdmissionOpen('a managed background task', admission);
|
|
982
1901
|
if (!/^[a-zA-Z0-9_.-]+$/u.test(request.id))
|
|
983
1902
|
throw new Error(`Managed background task id is invalid: ${request.id}`);
|
|
984
1903
|
if (this.tasks.has(request.id))
|
|
985
1904
|
throw new Error(`Background task id already exists: ${request.id}`);
|
|
986
1905
|
|
|
987
|
-
const dir = await this.ensureRuntimeDir(ctx);
|
|
1906
|
+
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
1907
|
+
this.assertTaskAdmissionOpen('a managed background task', admission);
|
|
988
1908
|
const outputAbsPath = join(dir.abs, `${request.id}.output`);
|
|
989
1909
|
const metadataAbsPath = join(dir.abs, `${request.id}.json`);
|
|
990
1910
|
const outputPath = join(dir.display, `${request.id}.output`);
|
|
@@ -1003,15 +1923,20 @@ export class BackgroundTaskRegistry {
|
|
|
1003
1923
|
pid: undefined,
|
|
1004
1924
|
bytesWritten: 0,
|
|
1005
1925
|
isAgent: request.isAgent,
|
|
1926
|
+
surviveReload: false,
|
|
1006
1927
|
notified: false,
|
|
1007
1928
|
notifyOnCompletion: request.notifyOnCompletion,
|
|
1008
1929
|
triggerOnCompletion: request.triggerOnCompletion,
|
|
1009
1930
|
fusion: request.fusion,
|
|
1010
1931
|
managedCancel: request.cancel,
|
|
1011
1932
|
managedStopWaitMs: request.stopWaitMs,
|
|
1933
|
+
terminalPublished: false,
|
|
1934
|
+
terminalPublicationState: 'pending',
|
|
1935
|
+
terminalPublishAttempts: 0,
|
|
1012
1936
|
terminalPublicationGate: request.terminalPublicationGate,
|
|
1013
1937
|
waiters: [],
|
|
1014
1938
|
};
|
|
1939
|
+
this.assertTaskAdmissionOpen('a managed background task', admission);
|
|
1015
1940
|
this.tasks.set(task.id, task);
|
|
1016
1941
|
const stream = createWriteStream(outputAbsPath, { flags: 'a', encoding: 'utf8' });
|
|
1017
1942
|
task.stream = stream;
|
|
@@ -1026,13 +1951,54 @@ export class BackgroundTaskRegistry {
|
|
|
1026
1951
|
}
|
|
1027
1952
|
}
|
|
1028
1953
|
});
|
|
1954
|
+
const unbindAdmissionCancellation = this.bindOwnedTaskToAdmission(task, admission);
|
|
1955
|
+
|
|
1956
|
+
let completionAttached = false;
|
|
1957
|
+
const attachCompletion = (): void => {
|
|
1958
|
+
if (completionAttached) return;
|
|
1959
|
+
completionAttached = true;
|
|
1960
|
+
void request.completion
|
|
1961
|
+
.then(
|
|
1962
|
+
() => {
|
|
1963
|
+
const killed = task.killKind === 'user' || task.killKind === 'shutdown';
|
|
1964
|
+
const timedOut = task.killKind === 'timeout';
|
|
1965
|
+
return this.finalizeTask(
|
|
1966
|
+
task,
|
|
1967
|
+
killed ? 'killed' : timedOut ? 'failed' : 'completed',
|
|
1968
|
+
killed || timedOut ? null : 0,
|
|
1969
|
+
undefined,
|
|
1970
|
+
timedOut ? task.error : undefined,
|
|
1971
|
+
);
|
|
1972
|
+
},
|
|
1973
|
+
(error: unknown) => {
|
|
1974
|
+
const message = BackgroundTaskRegistry.errorMessage(error);
|
|
1975
|
+
const killed = task.killKind === 'user' || task.killKind === 'shutdown';
|
|
1976
|
+
return this.finalizeTask(task, killed ? 'killed' : 'failed', null, undefined, message);
|
|
1977
|
+
},
|
|
1978
|
+
)
|
|
1979
|
+
.catch((error: unknown) => {
|
|
1980
|
+
this.logger.error(
|
|
1981
|
+
`[background-tasks] managed task finalization failed for ${task.id}:`,
|
|
1982
|
+
error,
|
|
1983
|
+
);
|
|
1984
|
+
});
|
|
1985
|
+
};
|
|
1029
1986
|
|
|
1030
1987
|
try {
|
|
1031
|
-
await this.
|
|
1988
|
+
await this.awaitTaskAdmissionBoundary(
|
|
1989
|
+
this.writeMetadata(task, admission.controller.signal),
|
|
1990
|
+
admission,
|
|
1991
|
+
);
|
|
1992
|
+
attachCompletion();
|
|
1993
|
+
this.assertTaskAdmissionOpen('a managed background task', admission);
|
|
1032
1994
|
this.onChange();
|
|
1033
1995
|
} catch (error) {
|
|
1996
|
+
if (admission.controller.signal.aborted) {
|
|
1997
|
+
attachCompletion();
|
|
1998
|
+
throw this.surfacedTaskAdmissionError(admission, error);
|
|
1999
|
+
}
|
|
1034
2000
|
this.tasks.delete(task.id);
|
|
1035
|
-
|
|
2001
|
+
await this.destroyTaskStream(task);
|
|
1036
2002
|
try {
|
|
1037
2003
|
request.cancel();
|
|
1038
2004
|
} catch (cancelError) {
|
|
@@ -1041,26 +2007,17 @@ export class BackgroundTaskRegistry {
|
|
|
1041
2007
|
cancelError,
|
|
1042
2008
|
);
|
|
1043
2009
|
}
|
|
2010
|
+
await request.completion.then(
|
|
2011
|
+
() => undefined,
|
|
2012
|
+
() => undefined,
|
|
2013
|
+
);
|
|
1044
2014
|
throw new Error(
|
|
1045
2015
|
`Failed to register managed background task: ${BackgroundTaskRegistry.errorMessage(error)}`,
|
|
1046
2016
|
);
|
|
2017
|
+
} finally {
|
|
2018
|
+
unbindAdmissionCancellation();
|
|
1047
2019
|
}
|
|
1048
2020
|
|
|
1049
|
-
void request.completion
|
|
1050
|
-
.then(
|
|
1051
|
-
() => this.finalizeTask(task, 'completed', 0),
|
|
1052
|
-
(error: unknown) => {
|
|
1053
|
-
const message = BackgroundTaskRegistry.errorMessage(error);
|
|
1054
|
-
const killed = task.killKind === 'user' || task.killKind === 'shutdown';
|
|
1055
|
-
return this.finalizeTask(task, killed ? 'killed' : 'failed', null, undefined, message);
|
|
1056
|
-
},
|
|
1057
|
-
)
|
|
1058
|
-
.catch((error: unknown) => {
|
|
1059
|
-
this.logger.error(
|
|
1060
|
-
`[background-tasks] managed task finalization failed for ${task.id}:`,
|
|
1061
|
-
error,
|
|
1062
|
-
);
|
|
1063
|
-
});
|
|
1064
2021
|
return task;
|
|
1065
2022
|
}
|
|
1066
2023
|
|
|
@@ -1101,13 +2058,26 @@ export class BackgroundTaskRegistry {
|
|
|
1101
2058
|
ctx: BackgroundTaskContext,
|
|
1102
2059
|
request: StartDelegateTaskOptions,
|
|
1103
2060
|
): Promise<BgTask> {
|
|
1104
|
-
|
|
1105
|
-
|
|
2061
|
+
rejectSurvivalForTaskKind(request, 'delegate tasks');
|
|
2062
|
+
const admission = this.beginTaskAdmission('a delegate task');
|
|
2063
|
+
try {
|
|
2064
|
+
return await this.startDelegateTaskAdmitted(ctx, request, admission);
|
|
2065
|
+
} finally {
|
|
2066
|
+
this.releaseTaskAdmission(admission);
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
1106
2069
|
|
|
2070
|
+
private async startDelegateTaskAdmitted(
|
|
2071
|
+
ctx: BackgroundTaskContext,
|
|
2072
|
+
request: StartDelegateTaskOptions,
|
|
2073
|
+
admission: TaskAdmission,
|
|
2074
|
+
): Promise<BgTask> {
|
|
2075
|
+
this.assertTaskAdmissionOpen('a delegate task', admission);
|
|
1107
2076
|
const launch = resolvePiLaunch({ platform: this.platform });
|
|
1108
2077
|
assertWindowsCommandLineWithinLimit(launch, request.argv, this.platform, 'bg-delegate');
|
|
1109
2078
|
|
|
1110
|
-
const dir = await this.ensureRuntimeDir(ctx);
|
|
2079
|
+
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
2080
|
+
this.assertTaskAdmissionOpen('a delegate task', admission);
|
|
1111
2081
|
const id = request.facts.taskId;
|
|
1112
2082
|
const outputAbsPath = join(dir.abs, `${id}.output`);
|
|
1113
2083
|
const metadataAbsPath = join(dir.abs, `${id}.json`);
|
|
@@ -1127,14 +2097,19 @@ export class BackgroundTaskRegistry {
|
|
|
1127
2097
|
pid: undefined,
|
|
1128
2098
|
bytesWritten: 0,
|
|
1129
2099
|
isAgent: true,
|
|
2100
|
+
surviveReload: false,
|
|
1130
2101
|
notified: false,
|
|
1131
2102
|
notifyOnCompletion: request.notifyOnCompletion,
|
|
1132
2103
|
triggerOnCompletion: request.triggerOnCompletion,
|
|
1133
2104
|
timeoutSeconds: request.timeoutSeconds,
|
|
1134
2105
|
model: request.facts.route.qualifiedId,
|
|
1135
2106
|
delegate: request.facts,
|
|
2107
|
+
terminalPublished: false,
|
|
2108
|
+
terminalPublicationState: 'pending',
|
|
2109
|
+
terminalPublishAttempts: 0,
|
|
1136
2110
|
waiters: [],
|
|
1137
2111
|
};
|
|
2112
|
+
this.assertTaskAdmissionOpen('a delegate task', admission);
|
|
1138
2113
|
this.tasks.set(id, task);
|
|
1139
2114
|
|
|
1140
2115
|
const stream = createWriteStream(outputAbsPath, { flags: 'a', encoding: 'utf8' });
|
|
@@ -1143,7 +2118,9 @@ export class BackgroundTaskRegistry {
|
|
|
1143
2118
|
task.error = `Output file write failed: ${error.message}`;
|
|
1144
2119
|
});
|
|
1145
2120
|
|
|
2121
|
+
let unbindAdmissionCancellation = (): void => undefined;
|
|
1146
2122
|
try {
|
|
2123
|
+
this.assertTaskAdmissionOpen('a delegate task', admission);
|
|
1147
2124
|
const child = this.spawn(launch.executable, piLaunchArgv(launch, [...request.argv]), {
|
|
1148
2125
|
cwd: ctx.cwd,
|
|
1149
2126
|
detached: this.platform !== 'win32',
|
|
@@ -1155,20 +2132,7 @@ export class BackgroundTaskRegistry {
|
|
|
1155
2132
|
env: request.env,
|
|
1156
2133
|
windowsHide: true,
|
|
1157
2134
|
});
|
|
1158
|
-
task
|
|
1159
|
-
task.pid = child.pid;
|
|
1160
|
-
writeDelegateStdin(child, request.stdinBytes, (error) => {
|
|
1161
|
-
this.writeNotice(task, `\n[delegate stdin write failed: ${error.message}]\n`);
|
|
1162
|
-
if (task.status === 'running') {
|
|
1163
|
-
task.killKind = 'user';
|
|
1164
|
-
task.error = `Delegate seed could not be delivered: ${error.message}`;
|
|
1165
|
-
try {
|
|
1166
|
-
this.requestKill(task, 'SIGTERM');
|
|
1167
|
-
} catch {
|
|
1168
|
-
void this.finalizeTask(task, 'failed', null, undefined, task.error);
|
|
1169
|
-
}
|
|
1170
|
-
}
|
|
1171
|
-
});
|
|
2135
|
+
this.captureSpawnedChild(task, child);
|
|
1172
2136
|
|
|
1173
2137
|
child.stdout?.on('data', (data) => {
|
|
1174
2138
|
this.appendChildOutput(task, data, 'stdout');
|
|
@@ -1217,14 +2181,49 @@ export class BackgroundTaskRegistry {
|
|
|
1217
2181
|
}, request.timeoutSeconds * 1000);
|
|
1218
2182
|
}
|
|
1219
2183
|
|
|
1220
|
-
|
|
2184
|
+
unbindAdmissionCancellation = this.bindOwnedTaskToAdmission(task, admission);
|
|
2185
|
+
this.assertTaskAdmissionOpen('a delegate task', admission);
|
|
2186
|
+
writeDelegateStdin(child, request.stdinBytes, (error) => {
|
|
2187
|
+
this.writeNotice(task, `\n[delegate stdin write failed: ${error.message}]\n`);
|
|
2188
|
+
if (task.status === 'running') {
|
|
2189
|
+
task.killKind = 'user';
|
|
2190
|
+
task.error = `Delegate seed could not be delivered: ${error.message}`;
|
|
2191
|
+
try {
|
|
2192
|
+
this.requestKill(task, 'SIGTERM');
|
|
2193
|
+
} catch {
|
|
2194
|
+
void this.finalizeTask(task, 'failed', null, undefined, task.error);
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
});
|
|
2198
|
+
|
|
2199
|
+
await this.awaitTaskAdmissionBoundary(
|
|
2200
|
+
this.writeMetadata(task, admission.controller.signal),
|
|
2201
|
+
admission,
|
|
2202
|
+
);
|
|
2203
|
+
this.assertTaskAdmissionOpen('a delegate task', admission);
|
|
1221
2204
|
this.onChange();
|
|
1222
2205
|
return task;
|
|
1223
2206
|
} catch (error) {
|
|
2207
|
+
if (admission.controller.signal.aborted) {
|
|
2208
|
+
const admissionError = this.taskAdmissionError(admission);
|
|
2209
|
+
let cleanupError: unknown;
|
|
2210
|
+
if (task.child === undefined) {
|
|
2211
|
+
try {
|
|
2212
|
+
await this.discardUnspawnedTask(task, [outputAbsPath, metadataAbsPath]);
|
|
2213
|
+
} catch (cleanupFailure) {
|
|
2214
|
+
cleanupError = cleanupFailure;
|
|
2215
|
+
}
|
|
2216
|
+
} else {
|
|
2217
|
+
this.stopOwnedTaskAfterAdmissionCancellation(task, admissionError);
|
|
2218
|
+
}
|
|
2219
|
+
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
2220
|
+
}
|
|
1224
2221
|
const message = error instanceof Error ? error.message : String(error);
|
|
1225
2222
|
this.writeNotice(task, `\n[delegate spawn exception: ${message}]\n`);
|
|
1226
2223
|
await this.finalizeTask(task, 'failed', null, undefined, message);
|
|
1227
2224
|
throw new Error(`Failed to start delegate task: ${message}`);
|
|
2225
|
+
} finally {
|
|
2226
|
+
unbindAdmissionCancellation();
|
|
1228
2227
|
}
|
|
1229
2228
|
}
|
|
1230
2229
|
|
|
@@ -1232,12 +2231,26 @@ export class BackgroundTaskRegistry {
|
|
|
1232
2231
|
ctx: BackgroundTaskContext,
|
|
1233
2232
|
request: StartAttestedPiTaskOptions,
|
|
1234
2233
|
): Promise<BgTask> {
|
|
1235
|
-
|
|
1236
|
-
|
|
2234
|
+
rejectSurvivalForTaskKind(request, 'attested Pi tasks');
|
|
2235
|
+
const admission = this.beginTaskAdmission('an attested Pi task');
|
|
2236
|
+
try {
|
|
2237
|
+
return await this.startAttestedPiTaskAdmitted(ctx, request, admission);
|
|
2238
|
+
} finally {
|
|
2239
|
+
this.releaseTaskAdmission(admission);
|
|
2240
|
+
}
|
|
2241
|
+
}
|
|
1237
2242
|
|
|
2243
|
+
private async startAttestedPiTaskAdmitted(
|
|
2244
|
+
ctx: BackgroundTaskContext,
|
|
2245
|
+
request: StartAttestedPiTaskOptions,
|
|
2246
|
+
admission: TaskAdmission,
|
|
2247
|
+
): Promise<BgTask> {
|
|
2248
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2249
|
+
const attested = await this.awaitTaskAdmissionBoundary(this.loadAttestedRuntime(), admission);
|
|
2250
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
1238
2251
|
const attributionExtensionPath =
|
|
1239
2252
|
request.provider === 'anthropic' ? resolveAnthropicAttributionExtensionPath() : undefined;
|
|
1240
|
-
const argv = buildAttestedPiArgv(request, attributionExtensionPath);
|
|
2253
|
+
const argv = attested.buildAttestedPiArgv(request, attributionExtensionPath);
|
|
1241
2254
|
const attestedPiLaunch = resolvePiLaunch({ platform: this.platform });
|
|
1242
2255
|
assertWindowsCommandLineWithinLimit(
|
|
1243
2256
|
attestedPiLaunch,
|
|
@@ -1246,17 +2259,26 @@ export class BackgroundTaskRegistry {
|
|
|
1246
2259
|
'attested-pi-run',
|
|
1247
2260
|
);
|
|
1248
2261
|
|
|
1249
|
-
const dir = await this.ensureRuntimeDir(ctx);
|
|
1250
|
-
|
|
2262
|
+
const dir = await this.awaitTaskAdmissionBoundary(this.ensureRuntimeDir(ctx), admission);
|
|
2263
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2264
|
+
const id = attested.makeAttestedTaskId();
|
|
1251
2265
|
if (!ATTESTED_TASK_ID_PATTERN.test(id))
|
|
1252
2266
|
throw new Error('Generated attested task id is invalid');
|
|
1253
|
-
const paths = makeAttestedTaskPaths(dir.abs, dir.display, id);
|
|
2267
|
+
const paths = attested.makeAttestedTaskPaths(dir.abs, dir.display, id);
|
|
1254
2268
|
const promptBytes = Buffer.from(request.prompt, 'utf8');
|
|
1255
|
-
const reportAbsPath = await
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
2269
|
+
const reportAbsPath = await this.awaitTaskAdmissionBoundary(
|
|
2270
|
+
attested.resolveReportPath(ctx.cwd, request.reportPath),
|
|
2271
|
+
admission,
|
|
2272
|
+
);
|
|
2273
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2274
|
+
const auth = attested.observePiOAuth(ctx, request.provider, request.model);
|
|
2275
|
+
const gitOptions = this.attestedGitOptions(admission);
|
|
2276
|
+
const repoRootRealpath = await attested.gitRepoRoot(ctx.cwd, gitOptions);
|
|
2277
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2278
|
+
const cwdRealpath = await this.awaitTaskAdmissionBoundary(realpath(ctx.cwd), admission);
|
|
2279
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2280
|
+
const startAuthority = await attested.gitAuthoritySnapshot(ctx.cwd, gitOptions);
|
|
2281
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
1260
2282
|
if (!startAuthority.clean)
|
|
1261
2283
|
throw new Error('Attested Pi task requires a clean worktree at start');
|
|
1262
2284
|
const timeoutSeconds =
|
|
@@ -1284,6 +2306,7 @@ export class BackgroundTaskRegistry {
|
|
|
1284
2306
|
pid: undefined,
|
|
1285
2307
|
bytesWritten: 0,
|
|
1286
2308
|
isAgent: true,
|
|
2309
|
+
surviveReload: false,
|
|
1287
2310
|
notified: false,
|
|
1288
2311
|
notifyOnCompletion: false,
|
|
1289
2312
|
triggerOnCompletion: false,
|
|
@@ -1295,130 +2318,205 @@ export class BackgroundTaskRegistry {
|
|
|
1295
2318
|
wrapperPath: paths.wrapperPath,
|
|
1296
2319
|
attestationPath: paths.attestationPath,
|
|
1297
2320
|
},
|
|
2321
|
+
terminalPublished: false,
|
|
2322
|
+
terminalPublicationState: 'pending',
|
|
2323
|
+
terminalPublishAttempts: 0,
|
|
1298
2324
|
waiters: [],
|
|
1299
2325
|
};
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
await writeFileFsynced(paths.stderrAbsPath, '');
|
|
1305
|
-
await writeFileFsynced(
|
|
2326
|
+
const admissionArtifacts = [
|
|
2327
|
+
paths.outputAbsPath,
|
|
2328
|
+
paths.eventsAbsPath,
|
|
2329
|
+
paths.stderrAbsPath,
|
|
1306
2330
|
paths.wrapperAbsPath,
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
2331
|
+
paths.metadataAbsPath,
|
|
2332
|
+
paths.attestationAbsPath,
|
|
2333
|
+
];
|
|
2334
|
+
const admissionSignal = admission.controller.signal;
|
|
2335
|
+
try {
|
|
2336
|
+
await writeFileFsynced(paths.outputAbsPath, '', admissionSignal);
|
|
2337
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2338
|
+
await writeFileFsynced(paths.eventsAbsPath, '', admissionSignal);
|
|
2339
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2340
|
+
await writeFileFsynced(paths.stderrAbsPath, '', admissionSignal);
|
|
2341
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2342
|
+
await writeFileFsynced(
|
|
2343
|
+
paths.wrapperAbsPath,
|
|
2344
|
+
'direct-spawn attested Pi task; no shell telemetry wrapper is used\n',
|
|
2345
|
+
admissionSignal,
|
|
2346
|
+
);
|
|
2347
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2348
|
+
await this.writeMetadata(task, admissionSignal);
|
|
2349
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2350
|
+
} catch (error) {
|
|
2351
|
+
let cleanupError: unknown;
|
|
2352
|
+
try {
|
|
2353
|
+
await this.discardUnspawnedTask(task, admissionArtifacts);
|
|
2354
|
+
} catch (cleanupFailure) {
|
|
2355
|
+
cleanupError = cleanupFailure;
|
|
2356
|
+
}
|
|
2357
|
+
if (admissionSignal.aborted) {
|
|
2358
|
+
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
2359
|
+
}
|
|
2360
|
+
if (cleanupError !== undefined) {
|
|
2361
|
+
throw new AggregateError(
|
|
2362
|
+
[error, cleanupError],
|
|
2363
|
+
`Attested Pi preflight failed and artifact cleanup also failed: ${BackgroundTaskRegistry.errorMessage(cleanupError)}`,
|
|
2364
|
+
);
|
|
2365
|
+
}
|
|
2366
|
+
throw error;
|
|
2367
|
+
}
|
|
1329
2368
|
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
2369
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2370
|
+
this.tasks.set(id, task);
|
|
2371
|
+
let unbindAdmissionCancellation = (): void => undefined;
|
|
2372
|
+
try {
|
|
2373
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2374
|
+
const captured = attested.spawnAndCapturePi(
|
|
2375
|
+
this.spawn,
|
|
1334
2376
|
argv,
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
null,
|
|
1346
|
-
error.message,
|
|
2377
|
+
{
|
|
2378
|
+
cwd: ctx.cwd,
|
|
2379
|
+
detached: this.platform !== 'win32',
|
|
2380
|
+
shell: false,
|
|
2381
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
2382
|
+
env: attested.attestedPiChildEnv(this.env),
|
|
2383
|
+
windowsHide: true,
|
|
2384
|
+
},
|
|
2385
|
+
this.platform,
|
|
2386
|
+
attestedPiLaunch,
|
|
1347
2387
|
);
|
|
1348
|
-
|
|
2388
|
+
this.captureSpawnedChild(task, captured.child);
|
|
2389
|
+
|
|
2390
|
+
captured.child.on('error', (error) => {
|
|
2391
|
+
void this.finalizeAttestedPiTask(
|
|
2392
|
+
task,
|
|
2393
|
+
attested,
|
|
2394
|
+
paths,
|
|
2395
|
+
argv,
|
|
2396
|
+
cwdRealpath,
|
|
2397
|
+
repoRootRealpath,
|
|
2398
|
+
startAuthority,
|
|
2399
|
+
auth,
|
|
2400
|
+
promptBytes,
|
|
2401
|
+
reportAbsPath,
|
|
2402
|
+
captured.stdoutChunks,
|
|
2403
|
+
captured.stderrChunks,
|
|
2404
|
+
'failed',
|
|
2405
|
+
null,
|
|
2406
|
+
null,
|
|
2407
|
+
error.message,
|
|
2408
|
+
);
|
|
2409
|
+
});
|
|
1349
2410
|
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
2411
|
+
captured.child.on('close', (code, signalName) => {
|
|
2412
|
+
let status: TaskStatus = (code ?? 0) === 0 && signalName === null ? 'completed' : 'failed';
|
|
2413
|
+
let error: string | undefined;
|
|
2414
|
+
if (task.killKind === 'timeout') {
|
|
2415
|
+
status = 'failed';
|
|
2416
|
+
error = task.error ?? `Timed out after ${String(timeoutSeconds)}s`;
|
|
2417
|
+
} else if (task.killKind === 'user' || task.killKind === 'shutdown') {
|
|
2418
|
+
status = 'killed';
|
|
2419
|
+
error = task.error;
|
|
2420
|
+
} else if (status === 'failed') {
|
|
2421
|
+
const exitCode = code === null ? 'null' : String(code);
|
|
2422
|
+
error = `Exited with code ${exitCode}${signalName ? ` (${signalName})` : ''}`;
|
|
2423
|
+
}
|
|
2424
|
+
void this.finalizeAttestedPiTask(
|
|
2425
|
+
task,
|
|
2426
|
+
attested,
|
|
2427
|
+
paths,
|
|
2428
|
+
argv,
|
|
2429
|
+
cwdRealpath,
|
|
2430
|
+
repoRootRealpath,
|
|
2431
|
+
startAuthority,
|
|
2432
|
+
auth,
|
|
2433
|
+
promptBytes,
|
|
2434
|
+
reportAbsPath,
|
|
2435
|
+
captured.stdoutChunks,
|
|
2436
|
+
captured.stderrChunks,
|
|
2437
|
+
status,
|
|
2438
|
+
code,
|
|
2439
|
+
signalName,
|
|
2440
|
+
error,
|
|
2441
|
+
);
|
|
2442
|
+
});
|
|
2443
|
+
|
|
2444
|
+
unbindAdmissionCancellation = this.bindOwnedTaskToAdmission(task, admission);
|
|
2445
|
+
await this.awaitTaskAdmissionBoundary(this.writeMetadata(task, admissionSignal), admission);
|
|
2446
|
+
this.assertTaskAdmissionOpen('an attested Pi task', admission);
|
|
2447
|
+
this.onChange();
|
|
2448
|
+
|
|
2449
|
+
if (timeoutSeconds !== undefined) {
|
|
2450
|
+
task.timeoutHandle = setTimeout(() => {
|
|
2451
|
+
if (task.status !== 'running') return;
|
|
2452
|
+
task.killKind = 'timeout';
|
|
2453
|
+
task.error = `Timed out after ${String(timeoutSeconds)}s`;
|
|
2454
|
+
try {
|
|
2455
|
+
this.requestKill(task, 'SIGTERM');
|
|
2456
|
+
} catch (error) {
|
|
2457
|
+
void this.finalizeAttestedPiTask(
|
|
2458
|
+
task,
|
|
2459
|
+
attested,
|
|
2460
|
+
paths,
|
|
2461
|
+
argv,
|
|
2462
|
+
cwdRealpath,
|
|
2463
|
+
repoRootRealpath,
|
|
2464
|
+
startAuthority,
|
|
2465
|
+
auth,
|
|
2466
|
+
promptBytes,
|
|
2467
|
+
reportAbsPath,
|
|
2468
|
+
captured.stdoutChunks,
|
|
2469
|
+
captured.stderrChunks,
|
|
2470
|
+
'failed',
|
|
2471
|
+
null,
|
|
2472
|
+
null,
|
|
2473
|
+
error instanceof Error ? error.message : String(error),
|
|
2474
|
+
);
|
|
2475
|
+
}
|
|
2476
|
+
}, timeoutSeconds * 1000);
|
|
1362
2477
|
}
|
|
1363
|
-
void this.finalizeAttestedPiTask(
|
|
1364
|
-
task,
|
|
1365
|
-
paths,
|
|
1366
|
-
argv,
|
|
1367
|
-
cwdRealpath,
|
|
1368
|
-
repoRootRealpath,
|
|
1369
|
-
startAuthority,
|
|
1370
|
-
auth,
|
|
1371
|
-
promptBytes,
|
|
1372
|
-
reportAbsPath,
|
|
1373
|
-
captured.stdoutChunks,
|
|
1374
|
-
captured.stderrChunks,
|
|
1375
|
-
status,
|
|
1376
|
-
code,
|
|
1377
|
-
signalName,
|
|
1378
|
-
error,
|
|
1379
|
-
);
|
|
1380
|
-
});
|
|
1381
2478
|
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
task.error = `Timed out after ${String(timeoutSeconds)}s`;
|
|
2479
|
+
return task;
|
|
2480
|
+
} catch (error) {
|
|
2481
|
+
let cleanupError: unknown;
|
|
2482
|
+
if (task.child === undefined) {
|
|
1387
2483
|
try {
|
|
1388
|
-
this.
|
|
1389
|
-
} catch (
|
|
1390
|
-
|
|
1391
|
-
task,
|
|
1392
|
-
paths,
|
|
1393
|
-
argv,
|
|
1394
|
-
cwdRealpath,
|
|
1395
|
-
repoRootRealpath,
|
|
1396
|
-
startAuthority,
|
|
1397
|
-
auth,
|
|
1398
|
-
promptBytes,
|
|
1399
|
-
reportAbsPath,
|
|
1400
|
-
captured.stdoutChunks,
|
|
1401
|
-
captured.stderrChunks,
|
|
1402
|
-
'failed',
|
|
1403
|
-
null,
|
|
1404
|
-
null,
|
|
1405
|
-
error instanceof Error ? error.message : String(error),
|
|
1406
|
-
);
|
|
2484
|
+
await this.discardUnspawnedTask(task, admissionArtifacts);
|
|
2485
|
+
} catch (cleanupFailure) {
|
|
2486
|
+
cleanupError = cleanupFailure;
|
|
1407
2487
|
}
|
|
1408
|
-
}
|
|
2488
|
+
} else {
|
|
2489
|
+
const taskError = admissionSignal.aborted
|
|
2490
|
+
? this.taskAdmissionError(admission)
|
|
2491
|
+
: error instanceof Error
|
|
2492
|
+
? error
|
|
2493
|
+
: new Error(String(error));
|
|
2494
|
+
this.stopOwnedTaskAfterAdmissionCancellation(task, taskError);
|
|
2495
|
+
}
|
|
2496
|
+
if (admissionSignal.aborted) {
|
|
2497
|
+
throw this.surfacedTaskAdmissionError(admission, error, cleanupError);
|
|
2498
|
+
}
|
|
2499
|
+
if (cleanupError !== undefined) {
|
|
2500
|
+
throw new AggregateError(
|
|
2501
|
+
[error, cleanupError],
|
|
2502
|
+
`Attested Pi launch failed and artifact cleanup also failed: ${BackgroundTaskRegistry.errorMessage(cleanupError)}`,
|
|
2503
|
+
);
|
|
2504
|
+
}
|
|
2505
|
+
throw error;
|
|
2506
|
+
} finally {
|
|
2507
|
+
unbindAdmissionCancellation();
|
|
1409
2508
|
}
|
|
1410
|
-
|
|
1411
|
-
return task;
|
|
1412
2509
|
}
|
|
1413
2510
|
|
|
1414
2511
|
private async finalizeAttestedPiTask(
|
|
1415
2512
|
task: BgTask,
|
|
1416
|
-
|
|
2513
|
+
attested: AttestedPiRuntime,
|
|
2514
|
+
paths: ReturnType<AttestedPiRuntime['makeAttestedTaskPaths']>,
|
|
1417
2515
|
argv: string[],
|
|
1418
2516
|
cwdRealpath: string,
|
|
1419
2517
|
repoRootRealpath: string,
|
|
1420
|
-
startAuthority: Awaited<ReturnType<
|
|
1421
|
-
auth: ReturnType<
|
|
2518
|
+
startAuthority: Awaited<ReturnType<AttestedPiRuntime['gitAuthoritySnapshot']>>,
|
|
2519
|
+
auth: ReturnType<AttestedPiRuntime['observePiOAuth']>,
|
|
1422
2520
|
promptBytes: Buffer,
|
|
1423
2521
|
reportAbsPath: string,
|
|
1424
2522
|
stdoutChunks: Buffer[],
|
|
@@ -1431,16 +2529,18 @@ export class BackgroundTaskRegistry {
|
|
|
1431
2529
|
if (task.finalized) return;
|
|
1432
2530
|
task.finalized = true;
|
|
1433
2531
|
if (task.timeoutHandle) clearTimeout(task.timeoutHandle);
|
|
1434
|
-
if (
|
|
1435
|
-
clearTimeout(task.killEscalationTimer);
|
|
1436
|
-
task.killEscalationTimer = undefined;
|
|
1437
|
-
}
|
|
2532
|
+
if (this.platform === 'win32') this.clearKillEscalationTimer(task);
|
|
1438
2533
|
let finalStatus = status;
|
|
1439
2534
|
let finalError = error;
|
|
1440
|
-
const
|
|
1441
|
-
if (
|
|
2535
|
+
const posixForceFailure = await this.awaitPosixProcessGroupBeforeTerminal(task);
|
|
2536
|
+
if (posixForceFailure !== undefined) {
|
|
2537
|
+
finalStatus = 'failed';
|
|
2538
|
+
finalError = BackgroundTaskRegistry.appendTaskError(finalError, posixForceFailure.message);
|
|
2539
|
+
}
|
|
2540
|
+
const windowsForceFailure = await this.awaitWindowsForceBeforeTerminal(task);
|
|
2541
|
+
if (windowsForceFailure !== undefined) {
|
|
1442
2542
|
finalStatus = 'failed';
|
|
1443
|
-
finalError = BackgroundTaskRegistry.appendTaskError(finalError,
|
|
2543
|
+
finalError = BackgroundTaskRegistry.appendTaskError(finalError, windowsForceFailure.message);
|
|
1444
2544
|
}
|
|
1445
2545
|
task.exitCode = exitCode;
|
|
1446
2546
|
task.signal = signal;
|
|
@@ -1452,10 +2552,10 @@ export class BackgroundTaskRegistry {
|
|
|
1452
2552
|
await writeFileFsynced(paths.eventsAbsPath, rawEvents);
|
|
1453
2553
|
await writeFileFsynced(paths.stderrAbsPath, rawStderr);
|
|
1454
2554
|
|
|
1455
|
-
let parsed: ReturnType<
|
|
2555
|
+
let parsed: ReturnType<AttestedPiRuntime['parsePiJsonEvents']> | undefined;
|
|
1456
2556
|
if (finalStatus === 'completed') {
|
|
1457
2557
|
try {
|
|
1458
|
-
parsed = parsePiJsonEvents(rawEvents);
|
|
2558
|
+
parsed = attested.parsePiJsonEvents(rawEvents);
|
|
1459
2559
|
task.model = parsed.providerScopedModelId;
|
|
1460
2560
|
task.tokenUsage = {
|
|
1461
2561
|
input: parsed.tokenUsage.input,
|
|
@@ -1485,10 +2585,13 @@ export class BackgroundTaskRegistry {
|
|
|
1485
2585
|
|
|
1486
2586
|
try {
|
|
1487
2587
|
if (finalStatus === 'completed' && parsed) {
|
|
1488
|
-
const finishAuthority = await gitAuthoritySnapshot(
|
|
2588
|
+
const finishAuthority = await attested.gitAuthoritySnapshot(
|
|
2589
|
+
task.cwd,
|
|
2590
|
+
this.attestedGitOptions(),
|
|
2591
|
+
);
|
|
1489
2592
|
const completedSnapshot: BgTaskSnapshot = { ...snapshot(task), status: 'completed' };
|
|
1490
2593
|
await this.writeMetadataSnapshot(task, completedSnapshot);
|
|
1491
|
-
const attestation = await buildPiTaskAttestation({
|
|
2594
|
+
const attestation = await attested.buildPiTaskAttestation({
|
|
1492
2595
|
task: completedSnapshot,
|
|
1493
2596
|
paths,
|
|
1494
2597
|
sessionDir: dirNameFromDisplay(paths.outputPath),
|
|
@@ -1546,16 +2649,37 @@ export class BackgroundTaskRegistry {
|
|
|
1546
2649
|
if (task.status !== 'running') {
|
|
1547
2650
|
throw new Error(`Task ${task.id} is ${task.status}, not running`);
|
|
1548
2651
|
}
|
|
2652
|
+
if (task.reloadExecution !== undefined) {
|
|
2653
|
+
return task.reloadExecution.requestStop(kind, reason);
|
|
2654
|
+
}
|
|
2655
|
+
const stopWaitMs = task.managedStopWaitMs ?? this.stopWaitMs;
|
|
2656
|
+
if (
|
|
2657
|
+
this.platform !== 'win32' &&
|
|
2658
|
+
task.managedCancel === undefined &&
|
|
2659
|
+
task.posixProcessGroupSignalAuthorityReleased === true &&
|
|
2660
|
+
this.posixProcessGroupKillStates.get(task) === undefined
|
|
2661
|
+
) {
|
|
2662
|
+
const finalized = await this.waitForEnd(task, stopWaitMs);
|
|
2663
|
+
if (!finalized) {
|
|
2664
|
+
throw new Error(
|
|
2665
|
+
`Task ${task.id} did not finish terminalization within ${formatDuration(stopWaitMs)} after its process group signal authority was released`,
|
|
2666
|
+
);
|
|
2667
|
+
}
|
|
2668
|
+
return task;
|
|
2669
|
+
}
|
|
1549
2670
|
task.killKind = kind;
|
|
1550
2671
|
if (reason) task.error = reason;
|
|
1551
2672
|
this.requestKill(task, 'SIGTERM');
|
|
1552
|
-
const stopWaitMs = task.managedStopWaitMs ?? this.stopWaitMs;
|
|
1553
2673
|
const stopped =
|
|
1554
|
-
|
|
2674
|
+
task.managedCancel === undefined && this.platform === 'win32'
|
|
1555
2675
|
? await this.waitForEndOrWindowsForceFailure(task, stopWaitMs)
|
|
1556
|
-
:
|
|
1557
|
-
|
|
1558
|
-
|
|
2676
|
+
: task.managedCancel === undefined
|
|
2677
|
+
? await this.waitForEndOrPosixForceFailure(task, stopWaitMs)
|
|
2678
|
+
: await this.waitForEnd(task, stopWaitMs);
|
|
2679
|
+
const posixForceFailure = this.posixProcessGroupKillStates.get(task)?.failure;
|
|
2680
|
+
if (posixForceFailure !== undefined) throw posixForceFailure;
|
|
2681
|
+
const windowsForceFailure = this.windowsKillStates.get(task)?.forceFailure;
|
|
2682
|
+
if (windowsForceFailure !== undefined) throw windowsForceFailure;
|
|
1559
2683
|
if (!stopped) {
|
|
1560
2684
|
throw new Error(
|
|
1561
2685
|
`Task ${task.id} did not exit within ${formatDuration(stopWaitMs)} after cancellation`,
|
|
@@ -1616,13 +2740,17 @@ export class BackgroundTaskRegistry {
|
|
|
1616
2740
|
};
|
|
1617
2741
|
}
|
|
1618
2742
|
|
|
1619
|
-
private async writeMetadata(task: BgTask): Promise<void> {
|
|
1620
|
-
await this.writeMetadataSnapshot(task, snapshot(task));
|
|
2743
|
+
private async writeMetadata(task: BgTask, signal?: AbortSignal): Promise<void> {
|
|
2744
|
+
await this.writeMetadataSnapshot(task, snapshot(task), signal);
|
|
1621
2745
|
}
|
|
1622
2746
|
|
|
1623
|
-
private async writeMetadataSnapshot(
|
|
2747
|
+
private async writeMetadataSnapshot(
|
|
2748
|
+
task: BgTask,
|
|
2749
|
+
value: BgTaskSnapshot,
|
|
2750
|
+
signal?: AbortSignal,
|
|
2751
|
+
): Promise<void> {
|
|
1624
2752
|
const write = async () => {
|
|
1625
|
-
await writeJsonAtomic(task.metadataAbsPath, value);
|
|
2753
|
+
await writeJsonAtomic(task.metadataAbsPath, value, signal);
|
|
1626
2754
|
};
|
|
1627
2755
|
const previous = task.metadataWriteChain ?? Promise.resolve();
|
|
1628
2756
|
const next = previous.then(write, write);
|
|
@@ -1780,61 +2908,337 @@ export class BackgroundTaskRegistry {
|
|
|
1780
2908
|
this.commitTelemetry(task, latest);
|
|
1781
2909
|
}
|
|
1782
2910
|
|
|
1783
|
-
/** Flush a trailing partial wrapped-agent line on finalize so the last transcript fragment is never lost. */
|
|
1784
|
-
private flushAgentStdout(task: BgTask): void {
|
|
1785
|
-
const remainder = task.agentStdoutBuffer;
|
|
1786
|
-
if (!remainder) return;
|
|
1787
|
-
task.agentStdoutBuffer = '';
|
|
1788
|
-
const latest: TelemetryDelta = {};
|
|
1789
|
-
this.consumeAgentLine(task, remainder, latest);
|
|
1790
|
-
this.commitTelemetry(task, latest);
|
|
2911
|
+
/** Flush a trailing partial wrapped-agent line on finalize so the last transcript fragment is never lost. */
|
|
2912
|
+
private flushAgentStdout(task: BgTask): void {
|
|
2913
|
+
const remainder = task.agentStdoutBuffer;
|
|
2914
|
+
if (!remainder) return;
|
|
2915
|
+
task.agentStdoutBuffer = '';
|
|
2916
|
+
const latest: TelemetryDelta = {};
|
|
2917
|
+
this.consumeAgentLine(task, remainder, latest);
|
|
2918
|
+
this.commitTelemetry(task, latest);
|
|
2919
|
+
}
|
|
2920
|
+
|
|
2921
|
+
private consumeAgentLine(task: BgTask, rawLine: string, latest: TelemetryDelta): void {
|
|
2922
|
+
const line = rawLine.replace(/\r$/, '');
|
|
2923
|
+
const trimmed = line.trim();
|
|
2924
|
+
if (!trimmed) return;
|
|
2925
|
+
if (!trimmed.startsWith('{') || !trimmed.endsWith('}')) {
|
|
2926
|
+
this.writeNotice(task, `${line}\n`);
|
|
2927
|
+
return;
|
|
2928
|
+
}
|
|
2929
|
+
let parsed: unknown;
|
|
2930
|
+
try {
|
|
2931
|
+
parsed = parseJsonText(trimmed);
|
|
2932
|
+
} catch {
|
|
2933
|
+
this.writeNotice(task, `${line}\n`);
|
|
2934
|
+
return;
|
|
2935
|
+
}
|
|
2936
|
+
if (!isJsonObject(parsed)) {
|
|
2937
|
+
this.writeNotice(task, `${line}\n`);
|
|
2938
|
+
return;
|
|
2939
|
+
}
|
|
2940
|
+
const record: TelemetryControlPayload = parsed;
|
|
2941
|
+
const type = record.type;
|
|
2942
|
+
if (type === 'background-task-context-usage') {
|
|
2943
|
+
const context = normalizeContextUsage(record);
|
|
2944
|
+
if (context) latest.context = context;
|
|
2945
|
+
return;
|
|
2946
|
+
}
|
|
2947
|
+
if (type === 'background-task-telemetry') {
|
|
2948
|
+
const context = normalizeContextUsage(record.contextUsage);
|
|
2949
|
+
if (context) latest.context = context;
|
|
2950
|
+
const tokens = normalizeTokenUsage(record.tokenUsage);
|
|
2951
|
+
if (tokens) latest.tokens = tokens;
|
|
2952
|
+
const tools = normalizeToolUsage(record.toolUsage);
|
|
2953
|
+
if (tools) latest.tools = tools;
|
|
2954
|
+
const model = normalizeModel(record.model);
|
|
2955
|
+
if (model) latest.model = model;
|
|
2956
|
+
return;
|
|
2957
|
+
}
|
|
2958
|
+
const activity = parseAgentActivity(parsed);
|
|
2959
|
+
if (activity) {
|
|
2960
|
+
const formatted = formatAgentActivityLine(activity);
|
|
2961
|
+
if (formatted) this.writeNotice(task, `${formatted}\n`);
|
|
2962
|
+
return;
|
|
2963
|
+
}
|
|
2964
|
+
// Unknown JSON object: pass through to the transcript rather than silently dropping it.
|
|
2965
|
+
this.writeNotice(task, `${line}\n`);
|
|
2966
|
+
}
|
|
2967
|
+
|
|
2968
|
+
private beginPosixProcessGroupKill(task: BgTask, armGrace: boolean): PosixProcessGroupKillState {
|
|
2969
|
+
const existing = this.posixProcessGroupKillStates.get(task);
|
|
2970
|
+
if (existing !== undefined) return existing;
|
|
2971
|
+
if (task.posixProcessGroupSignalAuthorityReleased === true) {
|
|
2972
|
+
throw new Error(`Task ${task.id} has released its POSIX process-group signal authority`);
|
|
2973
|
+
}
|
|
2974
|
+
const groupId = task.ownedPosixProcessGroupId;
|
|
2975
|
+
if (groupId === undefined) {
|
|
2976
|
+
throw new Error(`Task ${task.id} has no owned POSIX process group`);
|
|
2977
|
+
}
|
|
2978
|
+
|
|
2979
|
+
let resolveCompletion = (): void => undefined;
|
|
2980
|
+
const completion = new Promise<void>((resolve) => {
|
|
2981
|
+
resolveCompletion = resolve;
|
|
2982
|
+
});
|
|
2983
|
+
// Finish ownership slightly before stopTask's waiter so a force/proof
|
|
2984
|
+
// failure is observed as that specific loud error instead of a generic
|
|
2985
|
+
// cancellation timeout.
|
|
2986
|
+
const reserveMs = Math.min(25, Math.max(1, Math.floor(this.stopWaitMs / 4)));
|
|
2987
|
+
const ownershipMs = Math.max(1, this.stopWaitMs - reserveMs);
|
|
2988
|
+
const state: PosixProcessGroupKillState = {
|
|
2989
|
+
groupId,
|
|
2990
|
+
completion,
|
|
2991
|
+
resolveCompletion,
|
|
2992
|
+
deadlineAt: Date.now() + ownershipMs,
|
|
2993
|
+
forceAttempted: false,
|
|
2994
|
+
settled: false,
|
|
2995
|
+
};
|
|
2996
|
+
this.posixProcessGroupKillStates.set(task, state);
|
|
2997
|
+
|
|
2998
|
+
if (armGrace) {
|
|
2999
|
+
// Publish the sole force owner before TERM. An injected signal can emit
|
|
3000
|
+
// close reentrantly; that close must see and await this exact state.
|
|
3001
|
+
task.killEscalationTimer = setTimeout(
|
|
3002
|
+
() => {
|
|
3003
|
+
task.killEscalationTimer = undefined;
|
|
3004
|
+
this.forceOwnedPosixProcessGroup(task, state);
|
|
3005
|
+
},
|
|
3006
|
+
Math.min(this.killGraceMs, ownershipMs),
|
|
3007
|
+
);
|
|
3008
|
+
// Unlike ordinary housekeeping timers, this owner stays referenced: a
|
|
3009
|
+
// departed leader must not let the host exit and strand its owned group.
|
|
3010
|
+
}
|
|
3011
|
+
return state;
|
|
3012
|
+
}
|
|
3013
|
+
|
|
3014
|
+
private finishPosixProcessGroupKill(
|
|
3015
|
+
task: BgTask,
|
|
3016
|
+
state: PosixProcessGroupKillState,
|
|
3017
|
+
releaseOwnership: boolean,
|
|
3018
|
+
): void {
|
|
3019
|
+
if (state.settled) return;
|
|
3020
|
+
state.settled = true;
|
|
3021
|
+
this.clearKillEscalationTimer(task);
|
|
3022
|
+
if (state.verificationTimer !== undefined) {
|
|
3023
|
+
clearTimeout(state.verificationTimer);
|
|
3024
|
+
state.verificationTimer = undefined;
|
|
3025
|
+
}
|
|
3026
|
+
task.posixProcessGroupSignalAuthorityReleased = true;
|
|
3027
|
+
if (releaseOwnership && task.ownedPosixProcessGroupId === state.groupId) {
|
|
3028
|
+
delete task.ownedPosixProcessGroupId;
|
|
3029
|
+
}
|
|
3030
|
+
state.resolveCompletion();
|
|
3031
|
+
const failure = state.failure;
|
|
3032
|
+
const listeners = state.failureListeners;
|
|
3033
|
+
if (listeners !== undefined) {
|
|
3034
|
+
delete state.failureListeners;
|
|
3035
|
+
if (failure !== undefined) {
|
|
3036
|
+
for (const listener of listeners) listener(failure);
|
|
3037
|
+
}
|
|
3038
|
+
}
|
|
3039
|
+
}
|
|
3040
|
+
|
|
3041
|
+
private observeOwnedPosixProcessGroupGone(
|
|
3042
|
+
task: BgTask,
|
|
3043
|
+
state: PosixProcessGroupKillState,
|
|
3044
|
+
): boolean {
|
|
3045
|
+
if (state.settled) return state.failure === undefined;
|
|
3046
|
+
try {
|
|
3047
|
+
const exists = this.killProcess(-state.groupId, 0);
|
|
3048
|
+
state.lastProbeError = exists
|
|
3049
|
+
? undefined
|
|
3050
|
+
: new Error(`process-group probe for ${String(state.groupId)} returned false`);
|
|
3051
|
+
return false;
|
|
3052
|
+
} catch (error) {
|
|
3053
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
3054
|
+
this.finishPosixProcessGroupKill(task, state, true);
|
|
3055
|
+
return true;
|
|
3056
|
+
}
|
|
3057
|
+
state.lastProbeError =
|
|
3058
|
+
error instanceof Error ? error : new Error(BackgroundTaskRegistry.errorMessage(error));
|
|
3059
|
+
return false;
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
|
|
3063
|
+
private recordPosixProcessGroupForceFailure(
|
|
3064
|
+
task: BgTask,
|
|
3065
|
+
state: PosixProcessGroupKillState,
|
|
3066
|
+
error: Error,
|
|
3067
|
+
): void {
|
|
3068
|
+
if (state.settled) return;
|
|
3069
|
+
state.failure = error;
|
|
3070
|
+
this.finishPosixProcessGroupKill(task, state, false);
|
|
3071
|
+
task.error = BackgroundTaskRegistry.appendTaskError(task.error, error.message);
|
|
3072
|
+
this.writeNotice(task, `\n[background task POSIX termination: ${error.message}]\n`);
|
|
3073
|
+
this.onChange();
|
|
3074
|
+
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
3075
|
+
this.logger.error(
|
|
3076
|
+
`[background-tasks] failed to write POSIX process-group failure metadata for ${task.id}:`,
|
|
3077
|
+
metadataError,
|
|
3078
|
+
);
|
|
3079
|
+
});
|
|
3080
|
+
}
|
|
3081
|
+
|
|
3082
|
+
private schedulePosixProcessGroupVerification(
|
|
3083
|
+
task: BgTask,
|
|
3084
|
+
state: PosixProcessGroupKillState,
|
|
3085
|
+
): void {
|
|
3086
|
+
if (state.settled) return;
|
|
3087
|
+
const remainingMs = state.deadlineAt - Date.now();
|
|
3088
|
+
if (remainingMs <= 0) {
|
|
3089
|
+
const probeDetail =
|
|
3090
|
+
state.lastProbeError === undefined
|
|
3091
|
+
? ''
|
|
3092
|
+
: `; last group probe failed: ${state.lastProbeError.message}`;
|
|
3093
|
+
this.recordPosixProcessGroupForceFailure(
|
|
3094
|
+
task,
|
|
3095
|
+
state,
|
|
3096
|
+
new Error(
|
|
3097
|
+
`POSIX process group ${String(state.groupId)} remained present after SIGKILL${probeDetail}. Descendant processes may have leaked.`,
|
|
3098
|
+
),
|
|
3099
|
+
);
|
|
3100
|
+
return;
|
|
3101
|
+
}
|
|
3102
|
+
state.verificationTimer = setTimeout(
|
|
3103
|
+
() => {
|
|
3104
|
+
state.verificationTimer = undefined;
|
|
3105
|
+
if (this.observeOwnedPosixProcessGroupGone(task, state)) return;
|
|
3106
|
+
this.schedulePosixProcessGroupVerification(task, state);
|
|
3107
|
+
},
|
|
3108
|
+
Math.min(10, remainingMs),
|
|
3109
|
+
);
|
|
3110
|
+
}
|
|
3111
|
+
|
|
3112
|
+
private forceOwnedPosixProcessGroup(task: BgTask, state: PosixProcessGroupKillState): void {
|
|
3113
|
+
if (state.settled || state.forceAttempted) return;
|
|
3114
|
+
// Latch before either probe or signal: both are injected boundaries that can
|
|
3115
|
+
// reentrantly emit root close, and no continuation may launch a second KILL.
|
|
3116
|
+
state.forceAttempted = true;
|
|
3117
|
+
this.clearKillEscalationTimer(task);
|
|
3118
|
+
if (this.observeOwnedPosixProcessGroupGone(task, state)) return;
|
|
3119
|
+
|
|
3120
|
+
try {
|
|
3121
|
+
const forced = this.killProcess(-state.groupId, 'SIGKILL');
|
|
3122
|
+
if (!forced) {
|
|
3123
|
+
this.recordPosixProcessGroupForceFailure(
|
|
3124
|
+
task,
|
|
3125
|
+
state,
|
|
3126
|
+
new Error(
|
|
3127
|
+
`POSIX process-group SIGKILL returned false for task ${task.id} group ${String(state.groupId)}. Descendant processes may have leaked.`,
|
|
3128
|
+
),
|
|
3129
|
+
);
|
|
3130
|
+
return;
|
|
3131
|
+
}
|
|
3132
|
+
} catch (error) {
|
|
3133
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
3134
|
+
this.finishPosixProcessGroupKill(task, state, true);
|
|
3135
|
+
return;
|
|
3136
|
+
}
|
|
3137
|
+
this.recordPosixProcessGroupForceFailure(
|
|
3138
|
+
task,
|
|
3139
|
+
state,
|
|
3140
|
+
new Error(
|
|
3141
|
+
`POSIX process-group SIGKILL failed for task ${task.id} group ${String(state.groupId)}: ${BackgroundTaskRegistry.errorMessage(error)}. Descendant processes may have leaked.`,
|
|
3142
|
+
),
|
|
3143
|
+
);
|
|
3144
|
+
return;
|
|
3145
|
+
}
|
|
3146
|
+
|
|
3147
|
+
if (this.observeOwnedPosixProcessGroupGone(task, state)) return;
|
|
3148
|
+
this.schedulePosixProcessGroupVerification(task, state);
|
|
1791
3149
|
}
|
|
1792
3150
|
|
|
1793
|
-
private
|
|
1794
|
-
const
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
this.writeNotice(task, `${line}\n`);
|
|
3151
|
+
private requestPosixKill(task: BgTask, signal: NodeJS.Signals): void {
|
|
3152
|
+
const state = this.beginPosixProcessGroupKill(task, signal !== 'SIGKILL');
|
|
3153
|
+
if (signal === 'SIGKILL') {
|
|
3154
|
+
task.killSignalSent = true;
|
|
3155
|
+
this.forceOwnedPosixProcessGroup(task, state);
|
|
1799
3156
|
return;
|
|
1800
3157
|
}
|
|
1801
|
-
|
|
3158
|
+
if (task.killSignalSent) return;
|
|
3159
|
+
// Publish de-duplication before the signal boundary for the same reason the
|
|
3160
|
+
// state/timer is published above: close may be emitted synchronously.
|
|
3161
|
+
task.killSignalSent = true;
|
|
3162
|
+
|
|
3163
|
+
const errors: string[] = [];
|
|
3164
|
+
let killed = false;
|
|
1802
3165
|
try {
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
3166
|
+
killed = this.killProcess(-state.groupId, signal);
|
|
3167
|
+
if (!killed) errors.push(`process group ${signal} returned false`);
|
|
3168
|
+
} catch (error) {
|
|
3169
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
3170
|
+
this.finishPosixProcessGroupKill(task, state, true);
|
|
3171
|
+
} else {
|
|
3172
|
+
errors.push(`process group kill failed: ${BackgroundTaskRegistry.errorMessage(error)}`);
|
|
3173
|
+
}
|
|
1811
3174
|
}
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
3175
|
+
|
|
3176
|
+
if (!killed) {
|
|
3177
|
+
try {
|
|
3178
|
+
killed = task.child?.kill(signal) === true;
|
|
3179
|
+
if (!killed) errors.push(`child ${signal} returned false`);
|
|
3180
|
+
} catch (error) {
|
|
3181
|
+
errors.push(`child kill failed: ${BackgroundTaskRegistry.errorMessage(error)}`);
|
|
3182
|
+
}
|
|
1818
3183
|
}
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
if (
|
|
1824
|
-
|
|
1825
|
-
if (tools) latest.tools = tools;
|
|
1826
|
-
const model = normalizeModel(record.model);
|
|
1827
|
-
if (model) latest.model = model;
|
|
1828
|
-
return;
|
|
3184
|
+
|
|
3185
|
+
if (!killed) {
|
|
3186
|
+
// If TERM reached neither target, waiting out grace has no benefit. Keep
|
|
3187
|
+
// the same owner but attempt its one force phase immediately.
|
|
3188
|
+
if (!state.settled) this.forceOwnedPosixProcessGroup(task, state);
|
|
3189
|
+
throw new Error(`Could not kill task ${task.id}: ${errors.join('; ')}`);
|
|
1829
3190
|
}
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
3191
|
+
}
|
|
3192
|
+
|
|
3193
|
+
private async awaitPosixProcessGroupBeforeTerminal(task: BgTask): Promise<Error | undefined> {
|
|
3194
|
+
if (this.platform === 'win32') return undefined;
|
|
3195
|
+
const state = this.posixProcessGroupKillStates.get(task);
|
|
3196
|
+
if (state === undefined) {
|
|
3197
|
+
// No tree stop won the race before direct-child finalization. Release
|
|
3198
|
+
// signal authority synchronously so a concurrent late stop waits for this
|
|
3199
|
+
// terminalization instead of targeting a potentially reused group id.
|
|
3200
|
+
task.posixProcessGroupSignalAuthorityReleased = true;
|
|
3201
|
+
delete task.ownedPosixProcessGroupId;
|
|
3202
|
+
return undefined;
|
|
1835
3203
|
}
|
|
1836
|
-
|
|
1837
|
-
|
|
3204
|
+
this.observeOwnedPosixProcessGroupGone(task, state);
|
|
3205
|
+
await state.completion;
|
|
3206
|
+
return state.failure;
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3209
|
+
private waitForEndOrPosixForceFailure(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
3210
|
+
const state = this.posixProcessGroupKillStates.get(task);
|
|
3211
|
+
if (state === undefined) return this.waitForEnd(task, timeoutMs);
|
|
3212
|
+
if (state.failure !== undefined) return Promise.reject(state.failure);
|
|
3213
|
+
if (task.status !== 'running') return Promise.resolve(true);
|
|
3214
|
+
return new Promise((resolve, reject) => {
|
|
3215
|
+
const cleanup = () => {
|
|
3216
|
+
clearTimeout(timeout);
|
|
3217
|
+
const waiterIndex = task.waiters.indexOf(done);
|
|
3218
|
+
if (waiterIndex >= 0) task.waiters.splice(waiterIndex, 1);
|
|
3219
|
+
const listeners = state.failureListeners;
|
|
3220
|
+
if (listeners !== undefined) {
|
|
3221
|
+
const listenerIndex = listeners.indexOf(failed);
|
|
3222
|
+
if (listenerIndex >= 0) listeners.splice(listenerIndex, 1);
|
|
3223
|
+
if (listeners.length === 0) delete state.failureListeners;
|
|
3224
|
+
}
|
|
3225
|
+
};
|
|
3226
|
+
const timeout = setTimeout(() => {
|
|
3227
|
+
cleanup();
|
|
3228
|
+
resolve(false);
|
|
3229
|
+
}, timeoutMs);
|
|
3230
|
+
const done = () => {
|
|
3231
|
+
cleanup();
|
|
3232
|
+
resolve(true);
|
|
3233
|
+
};
|
|
3234
|
+
const failed = (error: Error) => {
|
|
3235
|
+
cleanup();
|
|
3236
|
+
reject(error);
|
|
3237
|
+
};
|
|
3238
|
+
task.waiters.push(done);
|
|
3239
|
+
if (state.failureListeners === undefined) state.failureListeners = [];
|
|
3240
|
+
state.failureListeners.push(failed);
|
|
3241
|
+
});
|
|
1838
3242
|
}
|
|
1839
3243
|
|
|
1840
3244
|
private getWindowsKillState(task: BgTask): WindowsKillState {
|
|
@@ -2118,62 +3522,15 @@ export class BackgroundTaskRegistry {
|
|
|
2118
3522
|
if (!task.child) {
|
|
2119
3523
|
throw new Error(`Task ${task.id} has no child process handle`);
|
|
2120
3524
|
}
|
|
2121
|
-
if (!task.pid) {
|
|
2122
|
-
throw new Error(`Task ${task.id} has no process id`);
|
|
2123
|
-
}
|
|
2124
3525
|
if (task.killSignalSent && signal === 'SIGTERM') return;
|
|
2125
3526
|
|
|
2126
3527
|
if (this.platform === 'win32') {
|
|
3528
|
+
if (!task.pid) throw new Error(`Task ${task.id} has no process id`);
|
|
2127
3529
|
this.requestWindowsKill(task, task.pid, signal);
|
|
2128
3530
|
return;
|
|
2129
3531
|
}
|
|
2130
3532
|
|
|
2131
|
-
|
|
2132
|
-
let killed = false;
|
|
2133
|
-
|
|
2134
|
-
try {
|
|
2135
|
-
this.killProcess(-task.pid, signal);
|
|
2136
|
-
killed = true;
|
|
2137
|
-
} catch (error) {
|
|
2138
|
-
errors.push(
|
|
2139
|
-
`process group kill failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2140
|
-
);
|
|
2141
|
-
}
|
|
2142
|
-
|
|
2143
|
-
if (!killed) {
|
|
2144
|
-
try {
|
|
2145
|
-
task.child.kill(signal);
|
|
2146
|
-
killed = true;
|
|
2147
|
-
} catch (error) {
|
|
2148
|
-
errors.push(`child kill failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
2149
|
-
}
|
|
2150
|
-
}
|
|
2151
|
-
|
|
2152
|
-
if (!killed) {
|
|
2153
|
-
throw new Error(`Could not kill task ${task.id}: ${errors.join('; ')}`);
|
|
2154
|
-
}
|
|
2155
|
-
|
|
2156
|
-
task.killSignalSent = true;
|
|
2157
|
-
// SIGKILL is the terminal escalation; it must never schedule a further one.
|
|
2158
|
-
if (signal === 'SIGKILL') return;
|
|
2159
|
-
// Only one escalation timer may be outstanding. Concurrent stop requests
|
|
2160
|
-
// previously each scheduled their own, producing duplicate SIGKILLs.
|
|
2161
|
-
if (task.killEscalationTimer !== undefined) return;
|
|
2162
|
-
task.killEscalationTimer = setTimeout(() => {
|
|
2163
|
-
task.killEscalationTimer = undefined;
|
|
2164
|
-
if (task.status !== 'running') return;
|
|
2165
|
-
try {
|
|
2166
|
-
this.requestKill(task, 'SIGKILL');
|
|
2167
|
-
} catch (error) {
|
|
2168
|
-
task.error = `SIGKILL failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
2169
|
-
void this.writeMetadata(task).catch((metadataError: unknown) => {
|
|
2170
|
-
this.logger.error(
|
|
2171
|
-
`[background-tasks] failed to write metadata for ${task.id}:`,
|
|
2172
|
-
metadataError,
|
|
2173
|
-
);
|
|
2174
|
-
});
|
|
2175
|
-
}
|
|
2176
|
-
}, this.killGraceMs).unref();
|
|
3533
|
+
this.requestPosixKill(task, signal);
|
|
2177
3534
|
}
|
|
2178
3535
|
|
|
2179
3536
|
private waitForEnd(task: BgTask, timeoutMs: number): Promise<boolean> {
|
|
@@ -2239,8 +3596,112 @@ export class BackgroundTaskRegistry {
|
|
|
2239
3596
|
return state.forceFailure;
|
|
2240
3597
|
}
|
|
2241
3598
|
|
|
3599
|
+
private async deliverReloadTerminal(
|
|
3600
|
+
execution: ReloadableShellExecutionV1,
|
|
3601
|
+
lease: ReloadShellActivationLeaseV1,
|
|
3602
|
+
): Promise<void> {
|
|
3603
|
+
const task = execution.task;
|
|
3604
|
+
if (task.reloadHostDeliveryInFlight || task.reloadHostDeliverySettled) {
|
|
3605
|
+
this.maybeReleaseReloadExecution(task);
|
|
3606
|
+
return;
|
|
3607
|
+
}
|
|
3608
|
+
task.reloadHostDeliveryInFlight = true;
|
|
3609
|
+
try {
|
|
3610
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
3611
|
+
this.onChange();
|
|
3612
|
+
this.publishTerminal(task);
|
|
3613
|
+
const deliveryGate = await this.waitForTerminalPublicationGate(task);
|
|
3614
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
3615
|
+
if (deliveryGate.kind === 'rejected') {
|
|
3616
|
+
this.logger.error(
|
|
3617
|
+
`[background-tasks] completion delivery gate failed for ${task.id}: ${this.terminalPublicationError(deliveryGate.error)}`,
|
|
3618
|
+
);
|
|
3619
|
+
} else if (
|
|
3620
|
+
task.notifyOnCompletion &&
|
|
3621
|
+
!task.notified &&
|
|
3622
|
+
!this.shuttingDown &&
|
|
3623
|
+
execution.notificationState === 'pending'
|
|
3624
|
+
) {
|
|
3625
|
+
const token = execution.beginNotification(lease);
|
|
3626
|
+
if (token !== undefined) {
|
|
3627
|
+
try {
|
|
3628
|
+
this.notifyCompletion(task);
|
|
3629
|
+
execution.finishNotification(token, task.notified);
|
|
3630
|
+
} catch (error) {
|
|
3631
|
+
execution.finishNotification(token, false);
|
|
3632
|
+
this.logger.error(`[background-tasks] notification failed for ${task.id}:`, error);
|
|
3633
|
+
}
|
|
3634
|
+
}
|
|
3635
|
+
}
|
|
3636
|
+
if (!this.ownsReloadExecution(execution, lease)) return;
|
|
3637
|
+
task.reloadHostNotificationSettled = true;
|
|
3638
|
+
task.reloadHostDeliverySettled = true;
|
|
3639
|
+
try {
|
|
3640
|
+
await this.writeMetadata(task);
|
|
3641
|
+
} catch (error) {
|
|
3642
|
+
this.logger.error(
|
|
3643
|
+
`[background-tasks] failed to update survivor notification metadata for ${task.id}:`,
|
|
3644
|
+
error,
|
|
3645
|
+
);
|
|
3646
|
+
}
|
|
3647
|
+
} finally {
|
|
3648
|
+
task.reloadHostDeliveryInFlight = false;
|
|
3649
|
+
this.maybeReleaseReloadExecution(task);
|
|
3650
|
+
}
|
|
3651
|
+
}
|
|
3652
|
+
|
|
3653
|
+
private maybeReleaseReloadExecution(task: BgTask): void {
|
|
3654
|
+
const execution = task.reloadExecution;
|
|
3655
|
+
const lease = this.reloadShellLease;
|
|
3656
|
+
if (
|
|
3657
|
+
execution === undefined ||
|
|
3658
|
+
execution.phase !== 'terminal' ||
|
|
3659
|
+
task.reloadHostNotificationSettled !== true ||
|
|
3660
|
+
task.terminalPublicationState === 'pending' ||
|
|
3661
|
+
!this.ownsReloadExecution(execution, lease) ||
|
|
3662
|
+
this.reloadShellOwner === undefined ||
|
|
3663
|
+
lease === undefined
|
|
3664
|
+
) {
|
|
3665
|
+
return;
|
|
3666
|
+
}
|
|
3667
|
+
try {
|
|
3668
|
+
this.reloadShellOwner.releaseExecution(lease, execution);
|
|
3669
|
+
} catch (error) {
|
|
3670
|
+
if (
|
|
3671
|
+
typeof error !== 'object' ||
|
|
3672
|
+
error === null ||
|
|
3673
|
+
Reflect.get(error, 'code') !== 'pi_bg_reload_owner_stale_claim'
|
|
3674
|
+
) {
|
|
3675
|
+
this.logger.error(
|
|
3676
|
+
`[background-tasks] failed to release reload shell execution ${task.id}:`,
|
|
3677
|
+
error,
|
|
3678
|
+
);
|
|
3679
|
+
}
|
|
3680
|
+
}
|
|
3681
|
+
}
|
|
3682
|
+
|
|
3683
|
+
private terminalPublicationAbandonSignal(task: BgTask): TerminalPublicationAbandonSignal {
|
|
3684
|
+
const existing = this.terminalPublicationAbandonSignals.get(task);
|
|
3685
|
+
if (existing !== undefined) return existing;
|
|
3686
|
+
let resolveSignal: (reason: TerminalPublicationAbandonReason) => void = () => {};
|
|
3687
|
+
const promise = new Promise<TerminalPublicationAbandonReason>((resolve) => {
|
|
3688
|
+
resolveSignal = resolve;
|
|
3689
|
+
});
|
|
3690
|
+
const signal = { promise, resolve: resolveSignal };
|
|
3691
|
+
this.terminalPublicationAbandonSignals.set(task, signal);
|
|
3692
|
+
return signal;
|
|
3693
|
+
}
|
|
3694
|
+
|
|
2242
3695
|
private publishTerminal(task: BgTask): void {
|
|
2243
|
-
if (task.
|
|
3696
|
+
if (task.reloadExecution !== undefined && this.tasks.get(task.id) !== task) return;
|
|
3697
|
+
if (task.terminalPublicationState !== 'pending' || task.terminalPublishInFlight) return;
|
|
3698
|
+
if (this.terminalPublicationClosed) {
|
|
3699
|
+
this.abandonTerminalPublication(
|
|
3700
|
+
task,
|
|
3701
|
+
this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
3702
|
+
);
|
|
3703
|
+
return;
|
|
3704
|
+
}
|
|
2244
3705
|
task.terminalPublishInFlight = true;
|
|
2245
3706
|
if (task.terminalPublicationGate === undefined) {
|
|
2246
3707
|
this.tryPublishTerminalNow(task);
|
|
@@ -2250,42 +3711,209 @@ export class BackgroundTaskRegistry {
|
|
|
2250
3711
|
}
|
|
2251
3712
|
|
|
2252
3713
|
private async publishTerminalWhenReady(task: BgTask): Promise<void> {
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
3714
|
+
const outcome = await this.waitForTerminalPublicationGate(task);
|
|
3715
|
+
if (task.reloadExecution !== undefined && this.tasks.get(task.id) !== task) {
|
|
3716
|
+
task.terminalPublishInFlight = false;
|
|
3717
|
+
return;
|
|
3718
|
+
}
|
|
3719
|
+
if (outcome.kind === 'closed') {
|
|
3720
|
+
this.abandonTerminalPublication(task, outcome.reason);
|
|
3721
|
+
return;
|
|
3722
|
+
}
|
|
3723
|
+
if (outcome.kind === 'rejected') {
|
|
3724
|
+
this.abandonTerminalPublication(task, 'gate_rejected', outcome.error);
|
|
3725
|
+
return;
|
|
3726
|
+
}
|
|
3727
|
+
// The gate and registry closure can settle in the same microtask turn.
|
|
3728
|
+
// Re-check after the await so a late gate cannot publish into a disposed
|
|
3729
|
+
// activation or revive a task already abandoned by shutdown.
|
|
3730
|
+
if (this.terminalPublicationClosed || task.terminalPublicationState !== 'pending') {
|
|
3731
|
+
this.abandonTerminalPublication(
|
|
3732
|
+
task,
|
|
3733
|
+
this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
3734
|
+
);
|
|
2257
3735
|
return;
|
|
2258
3736
|
}
|
|
2259
3737
|
this.tryPublishTerminalNow(task);
|
|
2260
3738
|
}
|
|
2261
3739
|
|
|
3740
|
+
private async waitForTerminalPublicationGate(
|
|
3741
|
+
task: BgTask,
|
|
3742
|
+
): Promise<TerminalPublicationGateOutcome> {
|
|
3743
|
+
if (this.terminalPublicationClosed) {
|
|
3744
|
+
return {
|
|
3745
|
+
kind: 'closed',
|
|
3746
|
+
reason: this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
3747
|
+
};
|
|
3748
|
+
}
|
|
3749
|
+
if (task.terminalPublicationState === 'abandoned') {
|
|
3750
|
+
if (task.terminalPublicationAbandonReason === 'gate_rejected') {
|
|
3751
|
+
return { kind: 'rejected', error: new Error('terminal publication gate rejected') };
|
|
3752
|
+
}
|
|
3753
|
+
return {
|
|
3754
|
+
kind: 'closed',
|
|
3755
|
+
reason: task.terminalPublicationAbandonReason ?? 'registry_shutdown',
|
|
3756
|
+
};
|
|
3757
|
+
}
|
|
3758
|
+
const gate = task.terminalPublicationGate;
|
|
3759
|
+
if (gate === undefined) return { kind: 'released' };
|
|
3760
|
+
|
|
3761
|
+
const gateOutcome: Promise<TerminalPublicationGateOutcome> = gate.then(
|
|
3762
|
+
() => ({ kind: 'released' }),
|
|
3763
|
+
(error: unknown) => ({ kind: 'rejected', error }),
|
|
3764
|
+
);
|
|
3765
|
+
const closureOutcome: Promise<TerminalPublicationGateOutcome> =
|
|
3766
|
+
this.terminalPublicationClosedSignal.then((reason) => ({ kind: 'closed', reason }));
|
|
3767
|
+
const abandonmentOutcome: Promise<TerminalPublicationGateOutcome> =
|
|
3768
|
+
this.terminalPublicationAbandonSignal(task).promise.then((reason) => ({
|
|
3769
|
+
kind: 'closed',
|
|
3770
|
+
reason,
|
|
3771
|
+
}));
|
|
3772
|
+
const outcome = await Promise.race([gateOutcome, closureOutcome, abandonmentOutcome]);
|
|
3773
|
+
|
|
3774
|
+
// Closure wins if it happened before this continuation resumed, regardless
|
|
3775
|
+
// of which promise queued its reaction first.
|
|
3776
|
+
if (this.terminalPublicationClosed) {
|
|
3777
|
+
return {
|
|
3778
|
+
kind: 'closed',
|
|
3779
|
+
reason: this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
3780
|
+
};
|
|
3781
|
+
}
|
|
3782
|
+
return outcome;
|
|
3783
|
+
}
|
|
3784
|
+
|
|
2262
3785
|
private tryPublishTerminalNow(task: BgTask): void {
|
|
3786
|
+
if (task.reloadExecution !== undefined && this.tasks.get(task.id) !== task) {
|
|
3787
|
+
task.terminalPublishInFlight = false;
|
|
3788
|
+
return;
|
|
3789
|
+
}
|
|
3790
|
+
if (task.terminalPublicationState !== 'pending') {
|
|
3791
|
+
task.terminalPublishInFlight = false;
|
|
3792
|
+
return;
|
|
3793
|
+
}
|
|
3794
|
+
if (this.terminalPublicationClosed) {
|
|
3795
|
+
task.terminalPublishInFlight = false;
|
|
3796
|
+
this.abandonTerminalPublication(
|
|
3797
|
+
task,
|
|
3798
|
+
this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
3799
|
+
);
|
|
3800
|
+
this.pruneOldTasks();
|
|
3801
|
+
return;
|
|
3802
|
+
}
|
|
3803
|
+
|
|
3804
|
+
task.terminalPublishAttempts += 1;
|
|
3805
|
+
task.terminalEmitInFlight = true;
|
|
3806
|
+
let emitFailed = false;
|
|
3807
|
+
let emitError: unknown;
|
|
2263
3808
|
try {
|
|
2264
|
-
if (task.terminalPublished) return;
|
|
2265
3809
|
this.publishTerminalSnapshot(snapshot(task));
|
|
2266
|
-
task.terminalPublished = true;
|
|
2267
|
-
if (task.terminalPublishRetryHandle) {
|
|
2268
|
-
clearTimeout(task.terminalPublishRetryHandle);
|
|
2269
|
-
task.terminalPublishRetryHandle = undefined;
|
|
2270
|
-
}
|
|
2271
3810
|
} catch (error) {
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
}
|
|
2275
|
-
|
|
3811
|
+
emitFailed = true;
|
|
3812
|
+
emitError = error;
|
|
3813
|
+
}
|
|
3814
|
+
task.terminalEmitInFlight = false;
|
|
3815
|
+
task.terminalPublishInFlight = false;
|
|
3816
|
+
|
|
3817
|
+
if (emitFailed) {
|
|
3818
|
+
const execution = task.reloadExecution;
|
|
3819
|
+
if (execution !== undefined && !this.ownsReloadExecution(execution, this.reloadShellLease)) {
|
|
3820
|
+
// The emitter synchronously detached this task into a reload handoff
|
|
3821
|
+
// before throwing. Attempt 1 is consumed, but only the fresh owner may
|
|
3822
|
+
// retry or decide abandonment on the shared publication ledger.
|
|
3823
|
+
return;
|
|
3824
|
+
}
|
|
3825
|
+
this.handleTerminalPublishFailure(task, emitError);
|
|
3826
|
+
} else {
|
|
3827
|
+
this.markTerminalPublicationDelivered(task);
|
|
3828
|
+
}
|
|
3829
|
+
this.pruneOldTasks();
|
|
3830
|
+
}
|
|
3831
|
+
|
|
3832
|
+
private markTerminalPublicationDelivered(task: BgTask): void {
|
|
3833
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
3834
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
3835
|
+
task.terminalPublishRetryHandle = undefined;
|
|
3836
|
+
}
|
|
3837
|
+
task.terminalPublicationGate = undefined;
|
|
3838
|
+
task.terminalEmitInFlight = false;
|
|
3839
|
+
task.terminalPublishInFlight = false;
|
|
3840
|
+
task.terminalPublicationState = 'delivered';
|
|
3841
|
+
delete task.terminalPublicationAbandonReason;
|
|
3842
|
+
task.terminalPublished = true;
|
|
3843
|
+
this.maybeReleaseReloadExecution(task);
|
|
3844
|
+
}
|
|
3845
|
+
|
|
3846
|
+
private abandonTerminalPublication(
|
|
3847
|
+
task: BgTask,
|
|
3848
|
+
reason: TerminalPublicationAbandonReason,
|
|
3849
|
+
error?: unknown,
|
|
3850
|
+
log = true,
|
|
3851
|
+
): void {
|
|
3852
|
+
if (task.terminalPublishRetryHandle !== undefined) {
|
|
3853
|
+
clearTimeout(task.terminalPublishRetryHandle);
|
|
3854
|
+
task.terminalPublishRetryHandle = undefined;
|
|
3855
|
+
}
|
|
3856
|
+
task.terminalPublicationGate = undefined;
|
|
3857
|
+
if (task.terminalEmitInFlight !== true) task.terminalPublishInFlight = false;
|
|
3858
|
+
if (task.terminalPublicationState === 'delivered') return;
|
|
3859
|
+
if (task.terminalPublicationState === 'abandoned') return;
|
|
3860
|
+
|
|
3861
|
+
task.terminalPublicationState = 'abandoned';
|
|
3862
|
+
task.terminalPublicationAbandonReason = reason;
|
|
3863
|
+
task.terminalPublished = false;
|
|
3864
|
+
this.terminalPublicationAbandonSignal(task).resolve(reason);
|
|
3865
|
+
if (log) {
|
|
3866
|
+
const detail = error === undefined ? '' : `: ${this.terminalPublicationError(error)}`;
|
|
3867
|
+
this.logger.error(
|
|
3868
|
+
`[background-tasks] terminal publication abandoned for ${task.id} (${reason}) after ${String(task.terminalPublishAttempts)}/${String(TERMINAL_PUBLICATION_MAX_ATTEMPTS)} emit attempts${detail}`,
|
|
3869
|
+
);
|
|
2276
3870
|
}
|
|
3871
|
+
this.maybeReleaseReloadExecution(task);
|
|
3872
|
+
}
|
|
3873
|
+
|
|
3874
|
+
private terminalPublicationError(error: unknown): string {
|
|
3875
|
+
const compact = BackgroundTaskRegistry.errorMessage(error).replace(/\s+/gu, ' ').trim();
|
|
3876
|
+
if (compact.length <= TERMINAL_PUBLICATION_DIAGNOSTIC_CHARS) return compact;
|
|
3877
|
+
return `${compact.slice(0, TERMINAL_PUBLICATION_DIAGNOSTIC_CHARS - 1)}…`;
|
|
2277
3878
|
}
|
|
2278
3879
|
|
|
2279
3880
|
private handleTerminalPublishFailure(task: BgTask, error: unknown): void {
|
|
2280
|
-
this.logger.error(`[background-tasks] terminal publication failed for ${task.id}:`, error);
|
|
2281
3881
|
task.terminalPublishInFlight = false;
|
|
2282
|
-
if (
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
3882
|
+
if (task.terminalPublicationState !== 'pending') return;
|
|
3883
|
+
if (error instanceof BackgroundTaskExtensionServiceClosedError) {
|
|
3884
|
+
this.closeTerminalPublication('publisher_closed');
|
|
3885
|
+
return;
|
|
3886
|
+
}
|
|
3887
|
+
if (this.terminalPublicationClosed) {
|
|
3888
|
+
this.abandonTerminalPublication(
|
|
3889
|
+
task,
|
|
3890
|
+
this.terminalPublicationCloseReason ?? 'registry_shutdown',
|
|
3891
|
+
error,
|
|
3892
|
+
);
|
|
3893
|
+
this.pruneOldTasks();
|
|
3894
|
+
return;
|
|
3895
|
+
}
|
|
3896
|
+
if (task.terminalPublishAttempts >= TERMINAL_PUBLICATION_MAX_ATTEMPTS) {
|
|
3897
|
+
this.abandonTerminalPublication(task, 'retry_exhausted', error);
|
|
3898
|
+
this.pruneOldTasks();
|
|
3899
|
+
return;
|
|
2288
3900
|
}
|
|
3901
|
+
|
|
3902
|
+
this.logger.error(
|
|
3903
|
+
`[background-tasks] terminal publication failed for ${task.id} (attempt ${String(task.terminalPublishAttempts)}/${String(TERMINAL_PUBLICATION_MAX_ATTEMPTS)}; retrying): ${this.terminalPublicationError(error)}`,
|
|
3904
|
+
);
|
|
3905
|
+
if (task.terminalPublishRetryHandle !== undefined) return;
|
|
3906
|
+
task.terminalPublishRetryHandle = setTimeout(() => {
|
|
3907
|
+
task.terminalPublishRetryHandle = undefined;
|
|
3908
|
+
if (
|
|
3909
|
+
this.terminalPublicationClosed ||
|
|
3910
|
+
task.terminalPublicationState !== 'pending' ||
|
|
3911
|
+
(task.reloadExecution !== undefined && this.tasks.get(task.id) !== task)
|
|
3912
|
+
)
|
|
3913
|
+
return;
|
|
3914
|
+
this.publishTerminal(task);
|
|
3915
|
+
}, TERMINAL_PUBLICATION_RETRY_MS);
|
|
3916
|
+
task.terminalPublishRetryHandle.unref();
|
|
2289
3917
|
}
|
|
2290
3918
|
|
|
2291
3919
|
private notifyCompletion(task: BgTask): void {
|
|
@@ -2344,16 +3972,18 @@ export class BackgroundTaskRegistry {
|
|
|
2344
3972
|
if (task.finalized) return;
|
|
2345
3973
|
task.finalized = true;
|
|
2346
3974
|
if (task.timeoutHandle) clearTimeout(task.timeoutHandle);
|
|
2347
|
-
if (
|
|
2348
|
-
clearTimeout(task.killEscalationTimer);
|
|
2349
|
-
task.killEscalationTimer = undefined;
|
|
2350
|
-
}
|
|
3975
|
+
if (this.platform === 'win32') this.clearKillEscalationTimer(task);
|
|
2351
3976
|
let finalStatus = status;
|
|
2352
3977
|
let finalError = error;
|
|
2353
|
-
const
|
|
2354
|
-
if (
|
|
3978
|
+
const posixForceFailure = await this.awaitPosixProcessGroupBeforeTerminal(task);
|
|
3979
|
+
if (posixForceFailure !== undefined) {
|
|
3980
|
+
finalStatus = 'failed';
|
|
3981
|
+
finalError = BackgroundTaskRegistry.appendTaskError(finalError, posixForceFailure.message);
|
|
3982
|
+
}
|
|
3983
|
+
const windowsForceFailure = await this.awaitWindowsForceBeforeTerminal(task);
|
|
3984
|
+
if (windowsForceFailure !== undefined) {
|
|
2355
3985
|
finalStatus = 'failed';
|
|
2356
|
-
finalError = BackgroundTaskRegistry.appendTaskError(finalError,
|
|
3986
|
+
finalError = BackgroundTaskRegistry.appendTaskError(finalError, windowsForceFailure.message);
|
|
2357
3987
|
}
|
|
2358
3988
|
task.exitCode = exitCode;
|
|
2359
3989
|
task.signal = signal ?? null;
|
|
@@ -2406,19 +4036,14 @@ export class BackgroundTaskRegistry {
|
|
|
2406
4036
|
for (const waiter of task.waiters.splice(0)) waiter();
|
|
2407
4037
|
this.onChange();
|
|
2408
4038
|
this.publishTerminal(task);
|
|
2409
|
-
|
|
2410
|
-
if (
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
error,
|
|
2418
|
-
);
|
|
2419
|
-
}
|
|
2420
|
-
}
|
|
2421
|
-
if (deliveryGateReady) {
|
|
4039
|
+
const deliveryGate = await this.waitForTerminalPublicationGate(task);
|
|
4040
|
+
if (deliveryGate.kind === 'rejected') {
|
|
4041
|
+
this.logger.error(
|
|
4042
|
+
`[background-tasks] completion delivery gate failed for ${task.id}: ${this.terminalPublicationError(deliveryGate.error)}`,
|
|
4043
|
+
);
|
|
4044
|
+
} else {
|
|
4045
|
+
// EventBus disposal abandons only EventBus publication. Notification truth
|
|
4046
|
+
// remains independent; notifyCompletion itself suppresses session shutdown.
|
|
2422
4047
|
try {
|
|
2423
4048
|
this.notifyCompletion(task);
|
|
2424
4049
|
} catch (notificationError) {
|
|
@@ -2442,11 +4067,15 @@ export class BackgroundTaskRegistry {
|
|
|
2442
4067
|
private pruneOldTasks(): void {
|
|
2443
4068
|
if (this.tasks.size <= this.maxRecentTasks) return;
|
|
2444
4069
|
const removable = [...this.tasks.values()]
|
|
2445
|
-
.filter((task) => task.status !== 'running')
|
|
4070
|
+
.filter((task) => task.status !== 'running' && task.terminalEmitInFlight !== true)
|
|
2446
4071
|
.sort((a, b) => (a.endTime ?? a.startTime) - (b.endTime ?? b.startTime));
|
|
2447
4072
|
while (this.tasks.size > this.maxRecentTasks && removable.length > 0) {
|
|
2448
4073
|
const task = removable.shift();
|
|
2449
|
-
if (task)
|
|
4074
|
+
if (task === undefined) continue;
|
|
4075
|
+
if (task.terminalPublicationState === 'pending') {
|
|
4076
|
+
this.abandonTerminalPublication(task, 'retention_limit');
|
|
4077
|
+
}
|
|
4078
|
+
this.tasks.delete(task.id);
|
|
2450
4079
|
}
|
|
2451
4080
|
}
|
|
2452
4081
|
}
|