pi-background-tasks 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BACKGROUND-TASKS-INSTRUCTIONS.md +1 -1
- package/README.md +70 -25
- package/TESTING.md +42 -2
- package/TEST_PLAN.md +7 -4
- package/dist/extensions/anthropic-attribution-child.js +4 -0
- package/dist/extensions/anthropic-attribution-child.js.map +1 -0
- package/dist/extensions/anthropic-attribution.js +107 -0
- package/dist/extensions/anthropic-attribution.js.map +1 -0
- package/dist/extensions/background-tasks.js +2 -0
- package/dist/extensions/background-tasks.js.map +1 -0
- package/dist/extensions/delegate-child.js +2 -0
- package/dist/extensions/delegate-child.js.map +1 -0
- package/dist/extensions/fusion-child.js +2 -0
- package/dist/extensions/fusion-child.js.map +1 -0
- package/dist/package.json +5 -0
- package/dist/src/core/anthropic-attribution-path.js +24 -0
- package/dist/src/core/anthropic-attribution-path.js.map +1 -0
- package/dist/src/core/anthropic-attribution.js +2247 -0
- package/dist/src/core/anthropic-attribution.js.map +1 -0
- package/dist/src/core/attested-pi-contract.js +5 -0
- package/dist/src/core/attested-pi-contract.js.map +1 -0
- package/dist/src/core/attested-pi-run.js +749 -0
- package/dist/src/core/attested-pi-run.js.map +1 -0
- package/dist/src/core/canonical-json.js +19 -0
- package/dist/src/core/canonical-json.js.map +1 -0
- package/dist/src/core/common.js +787 -0
- package/dist/src/core/common.js.map +1 -0
- package/dist/src/core/config.js +78 -0
- package/dist/src/core/config.js.map +1 -0
- package/dist/src/core/context/parent-snapshot.js +75 -0
- package/dist/src/core/context/parent-snapshot.js.map +1 -0
- package/dist/src/core/context/token-budget.js +631 -0
- package/dist/src/core/context/token-budget.js.map +1 -0
- package/dist/src/core/context/visible-conversation-v2.js +390 -0
- package/dist/src/core/context/visible-conversation-v2.js.map +1 -0
- package/dist/src/core/delegate/artifacts.js +357 -0
- package/dist/src/core/delegate/artifacts.js.map +1 -0
- package/dist/src/core/delegate/budget.js +266 -0
- package/dist/src/core/delegate/budget.js.map +1 -0
- package/dist/src/core/delegate/facade-contract.js +7 -0
- package/dist/src/core/delegate/facade-contract.js.map +1 -0
- package/dist/src/core/delegate/hook-contract-evidence.json +18 -0
- package/dist/src/core/delegate/hook-contract.js +124 -0
- package/dist/src/core/delegate/hook-contract.js.map +1 -0
- package/dist/src/core/delegate/launch.js +366 -0
- package/dist/src/core/delegate/launch.js.map +1 -0
- package/dist/src/core/delegate/result-package.js +293 -0
- package/dist/src/core/delegate/result-package.js.map +1 -0
- package/dist/src/core/delegate/runner.js +398 -0
- package/dist/src/core/delegate/runner.js.map +1 -0
- package/dist/src/core/delegate/seed.js +334 -0
- package/dist/src/core/delegate/seed.js.map +1 -0
- package/dist/src/core/delegate/types.js +103 -0
- package/dist/src/core/delegate/types.js.map +1 -0
- package/dist/src/core/durable-fs.js +372 -0
- package/dist/src/core/durable-fs.js.map +1 -0
- package/dist/src/core/extension-api.js +382 -0
- package/dist/src/core/extension-api.js.map +1 -0
- package/dist/src/core/fusion/artifacts.js +654 -0
- package/dist/src/core/fusion/artifacts.js.map +1 -0
- package/dist/src/core/fusion/budget.js +949 -0
- package/dist/src/core/fusion/budget.js.map +1 -0
- package/dist/src/core/fusion/child-protocol.js +176 -0
- package/dist/src/core/fusion/child-protocol.js.map +1 -0
- package/dist/src/core/fusion/claude-cache.js +156 -0
- package/dist/src/core/fusion/claude-cache.js.map +1 -0
- package/dist/src/core/fusion/clean-context.js +64 -0
- package/dist/src/core/fusion/clean-context.js.map +1 -0
- package/dist/src/core/fusion/config.js +349 -0
- package/dist/src/core/fusion/config.js.map +1 -0
- package/dist/src/core/fusion/context.js +184 -0
- package/dist/src/core/fusion/context.js.map +1 -0
- package/dist/src/core/fusion/evaluation.js +696 -0
- package/dist/src/core/fusion/evaluation.js.map +1 -0
- package/dist/src/core/fusion/facade-contract.js +3 -0
- package/dist/src/core/fusion/facade-contract.js.map +1 -0
- package/dist/src/core/fusion/orchestrator.js +867 -0
- package/dist/src/core/fusion/orchestrator.js.map +1 -0
- package/dist/src/core/fusion/output-contract.js +27 -0
- package/dist/src/core/fusion/output-contract.js.map +1 -0
- package/dist/src/core/fusion/pi-child.js +1741 -0
- package/dist/src/core/fusion/pi-child.js.map +1 -0
- package/dist/src/core/fusion/prompts.js +260 -0
- package/dist/src/core/fusion/prompts.js.map +1 -0
- package/dist/src/core/fusion/result-package.js +857 -0
- package/dist/src/core/fusion/result-package.js.map +1 -0
- package/dist/src/core/fusion/source-policy.js +259 -0
- package/dist/src/core/fusion/source-policy.js.map +1 -0
- package/dist/src/core/fusion/types.js +207 -0
- package/dist/src/core/fusion/types.js.map +1 -0
- package/dist/src/core/fusion/web-fetch.js +777 -0
- package/dist/src/core/fusion/web-fetch.js.map +1 -0
- package/dist/src/core/fusion/workflows.js +126 -0
- package/dist/src/core/fusion/workflows.js.map +1 -0
- package/dist/src/core/lazy-module.js +181 -0
- package/dist/src/core/lazy-module.js.map +1 -0
- package/dist/src/core/pi-launch.js +431 -0
- package/dist/src/core/pi-launch.js.map +1 -0
- package/dist/src/core/registry.js +3331 -0
- package/dist/src/core/registry.js.map +1 -0
- package/dist/src/core/reload-shell-owner.js +1396 -0
- package/dist/src/core/reload-shell-owner.js.map +1 -0
- package/dist/src/core/shell-policy.js +80 -0
- package/dist/src/core/shell-policy.js.map +1 -0
- package/dist/src/core/task-durable.js +51 -0
- package/dist/src/core/task-durable.js.map +1 -0
- package/dist/src/core/update-check.js +92 -0
- package/dist/src/core/update-check.js.map +1 -0
- package/dist/src/core/windows-taskkill.js +185 -0
- package/dist/src/core/windows-taskkill.js.map +1 -0
- package/dist/src/delegate-child-extension.js +776 -0
- package/dist/src/delegate-child-extension.js.map +1 -0
- package/dist/src/delegate-extension.js +746 -0
- package/dist/src/delegate-extension.js.map +1 -0
- package/dist/src/extension.js +953 -0
- package/dist/src/extension.js.map +1 -0
- package/dist/src/fusion-child-extension.js +760 -0
- package/dist/src/fusion-child-extension.js.map +1 -0
- package/dist/src/fusion-extension.js +1030 -0
- package/dist/src/fusion-extension.js.map +1 -0
- package/dist/src/ui/background-tasks-manager.js +689 -0
- package/dist/src/ui/background-tasks-manager.js.map +1 -0
- package/dist/src/ui/fusion-model-selector.js +277 -0
- package/dist/src/ui/fusion-model-selector.js.map +1 -0
- package/docs/INDEX.md +35 -33
- package/docs/api/eventbus-v1.md +21 -5
- package/docs/choose-a-workflow.md +2 -0
- package/docs/commands/bg-clear.md +3 -3
- package/docs/commands/bg-update.md +3 -3
- package/docs/commands/bg.md +18 -6
- package/docs/commands/claude-cache.md +4 -4
- package/docs/commands/fusion-models.md +49 -4
- package/docs/commands/fusion.md +3 -3
- package/docs/commands/jobs.md +3 -3
- package/docs/commands/kill.md +3 -3
- package/docs/commands/logs.md +3 -3
- package/docs/commands/task-manager.md +5 -5
- package/docs/concepts/completion-delivery.md +6 -2
- package/docs/getting-started.md +7 -1
- package/docs/manifest.json +228 -42
- package/docs/operations/configuration.md +58 -5
- package/docs/operations/releasing.md +4 -2
- package/docs/operations/testing.md +22 -0
- package/docs/operations/troubleshooting.md +6 -2
- package/docs/read-before-edit.md +44 -31
- package/docs/reference/runtime-contracts.md +87 -51
- package/docs/reference/shortcuts-and-dock.md +28 -14
- package/docs/subsystems/anthropic-attribution.md +23 -11
- package/docs/subsystems/attested-pi-runs.md +12 -7
- package/docs/subsystems/background-task-runtime.md +53 -11
- package/docs/subsystems/child-launch-durability-and-safety.md +28 -9
- package/docs/subsystems/delegation.md +16 -3
- package/docs/subsystems/docs-freshness-gate.md +14 -7
- package/docs/subsystems/fusion.md +36 -11
- package/docs/subsystems/host-ui-and-telemetry.md +22 -9
- package/docs/tools/bg_delegate.md +3 -1
- package/docs/tools/bg_kill.md +3 -1
- package/docs/tools/bg_logs.md +3 -1
- package/docs/tools/bg_result.md +5 -3
- package/docs/tools/bg_run.md +31 -5
- package/docs/tools/bg_run_pi_attested.md +3 -1
- package/docs/tools/bg_status.md +3 -1
- package/docs/tools/fusion_investigate.md +3 -1
- package/docs/tools/fusion_reason.md +3 -1
- package/docs/tools/fusion_research.md +3 -1
- package/docs/tools/fusion_validate.md +3 -1
- package/extensions/anthropic-attribution-child.ts +3 -0
- package/extensions/anthropic-attribution.ts +166 -1
- package/package.json +12 -10
- package/src/core/anthropic-attribution-path.ts +5 -4
- package/src/core/anthropic-attribution.ts +293 -113
- package/src/core/attested-pi-contract.ts +4 -0
- package/src/core/attested-pi-run.ts +468 -81
- package/src/core/canonical-json.ts +21 -0
- package/src/core/common.ts +450 -40
- package/src/core/config.ts +121 -0
- package/src/core/context/visible-conversation-v2.ts +3 -6
- package/src/core/delegate/artifacts.ts +13 -9
- package/src/core/delegate/budget.ts +13 -6
- package/src/core/delegate/facade-contract.ts +6 -0
- package/src/core/delegate/result-package.ts +32 -14
- package/src/core/delegate/runner.ts +129 -58
- package/src/core/delegate/seed.ts +5 -7
- package/src/core/durable-fs.ts +178 -30
- package/src/core/extension-api.ts +36 -8
- package/src/core/fusion/artifacts.ts +17 -16
- package/src/core/fusion/clean-context.ts +8 -5
- package/src/core/fusion/config.ts +2 -1
- package/src/core/fusion/context.ts +2 -6
- package/src/core/fusion/facade-contract.ts +2 -0
- package/src/core/fusion/orchestrator.ts +3 -3
- package/src/core/fusion/prompts.ts +1 -1
- package/src/core/fusion/result-package.ts +259 -72
- package/src/core/fusion/source-policy.ts +43 -17
- package/src/core/lazy-module.ts +215 -0
- package/src/core/pi-launch.ts +390 -65
- package/src/core/registry.ts +2010 -381
- package/src/core/reload-shell-owner.ts +1662 -0
- package/src/core/shell-policy.ts +134 -0
- package/src/core/task-durable.ts +67 -0
- package/src/delegate-extension.ts +404 -144
- package/src/extension.ts +387 -114
- package/src/fusion-extension.ts +202 -96
|
@@ -0,0 +1,1662 @@
|
|
|
1
|
+
import { spawn as nodeSpawn, type SpawnOptions } from 'node:child_process';
|
|
2
|
+
import { randomBytes } from 'node:crypto';
|
|
3
|
+
import { createWriteStream } from 'node:fs';
|
|
4
|
+
import { isAbsolute } from 'node:path';
|
|
5
|
+
import { formatSize } from '@earendil-works/pi-coding-agent';
|
|
6
|
+
import {
|
|
7
|
+
isJsonObject,
|
|
8
|
+
parseJsonText,
|
|
9
|
+
snapshot,
|
|
10
|
+
type BgTask,
|
|
11
|
+
type ReloadShellActivationClaimV1,
|
|
12
|
+
type ReloadShellActivationLeaseV1,
|
|
13
|
+
type ReloadShellHostAdapterV1,
|
|
14
|
+
type ReloadShellIdentityV1,
|
|
15
|
+
type ReloadShellOwnerEventSinkV1,
|
|
16
|
+
type ReloadShellOwnerHubV1,
|
|
17
|
+
type ReloadShellProcessV1,
|
|
18
|
+
type ReloadShellStopKind,
|
|
19
|
+
type ReloadSurvivalSnapshotV1,
|
|
20
|
+
type ReloadableShellExecutionV1,
|
|
21
|
+
type ReloadSurvivalErrorCode,
|
|
22
|
+
ReloadSurvivalError,
|
|
23
|
+
type TaskContextUsage,
|
|
24
|
+
type TaskStatus,
|
|
25
|
+
type TaskTokenUsage,
|
|
26
|
+
type TaskToolUsage,
|
|
27
|
+
} from './common.js';
|
|
28
|
+
import { replaceFileDurable } from './durable-fs.js';
|
|
29
|
+
import {
|
|
30
|
+
runWindowsTaskkill,
|
|
31
|
+
type TaskkillOutcome,
|
|
32
|
+
type WindowsKillPhase,
|
|
33
|
+
type WindowsTaskkillOptions,
|
|
34
|
+
} from './windows-taskkill.js';
|
|
35
|
+
|
|
36
|
+
export const RELOAD_SHELL_OWNER_PROTOCOL = 'pi-background-tasks.reload-shell-owner.v1' as const;
|
|
37
|
+
export const RELOAD_SHELL_OWNER_SYMBOL = Symbol.for(RELOAD_SHELL_OWNER_PROTOCOL);
|
|
38
|
+
export const RELOAD_SHELL_HANDOFF_TIMEOUT_MS = 30_000;
|
|
39
|
+
|
|
40
|
+
const TELEMETRY_BUFFER_CHARS = 512 * 1024;
|
|
41
|
+
|
|
42
|
+
export { ReloadSurvivalError } from './common.js';
|
|
43
|
+
|
|
44
|
+
interface OwnerSlot {
|
|
45
|
+
readonly identity: ReloadShellIdentityV1;
|
|
46
|
+
readonly identityKey: string;
|
|
47
|
+
generation: number;
|
|
48
|
+
handoffCount: number;
|
|
49
|
+
phase: 'claiming' | 'bound' | 'handoff' | 'releasing' | 'orphaned';
|
|
50
|
+
activationNonce: string;
|
|
51
|
+
claimNonce?: string | undefined;
|
|
52
|
+
claim?: ReloadShellActivationClaimV1 | undefined;
|
|
53
|
+
lease?: ReloadShellActivationLeaseV1 | undefined;
|
|
54
|
+
adapter?: ReloadShellHostAdapterV1 | undefined;
|
|
55
|
+
expiresAt?: number | undefined;
|
|
56
|
+
handoffTimer?: NodeJS.Timeout | undefined;
|
|
57
|
+
executions: Map<string, ReloadableShellExecutionV1>;
|
|
58
|
+
queuedChanged: Set<string>;
|
|
59
|
+
queuedTerminal: Set<string>;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface ReloadShellOwnerDependencies {
|
|
63
|
+
readonly handoffTimeoutMs?: number | undefined;
|
|
64
|
+
readonly now?: (() => number) | undefined;
|
|
65
|
+
readonly randomNonce?: (() => string) | undefined;
|
|
66
|
+
readonly logger?: Pick<Console, 'error'> | undefined;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
interface InternalHubState {
|
|
70
|
+
readonly slots: Map<string, OwnerSlot>;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const hubStates = new WeakMap<ReloadShellOwnerHubV1, InternalHubState>();
|
|
74
|
+
const OWNER_STATE_SYMBOL = Symbol.for(`${RELOAD_SHELL_OWNER_PROTOCOL}.test-state`);
|
|
75
|
+
|
|
76
|
+
function ownerError(code: ReloadSurvivalErrorCode, message: string): ReloadSurvivalError {
|
|
77
|
+
return new ReloadSurvivalError(code, message);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function randomNonce(): string {
|
|
81
|
+
return randomBytes(16).toString('hex');
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function requireNonce(value: string, label: string): void {
|
|
85
|
+
if (!/^[0-9a-f]{32}$/u.test(value)) {
|
|
86
|
+
throw ownerError('pi_bg_reload_owner_stale_claim', `${label} is not a 128-bit lowercase hex nonce`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function positiveTimeout(value: number | undefined, fallback: number): number {
|
|
91
|
+
const candidate = value ?? fallback;
|
|
92
|
+
if (!Number.isFinite(candidate) || candidate <= 0) {
|
|
93
|
+
throw ownerError('pi_bg_reload_owner_protocol_incompatible', 'handoff timeout must be positive');
|
|
94
|
+
}
|
|
95
|
+
return Math.max(1, Math.floor(candidate));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function lengthPart(value: string): string {
|
|
99
|
+
return `${String(Buffer.byteLength(value, 'utf8'))}:${value}`;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function makeReloadShellIdentity(
|
|
103
|
+
sessionId: string,
|
|
104
|
+
cwdRealpath: string,
|
|
105
|
+
hostPid: number = process.pid,
|
|
106
|
+
): ReloadShellIdentityV1 {
|
|
107
|
+
if (!Number.isSafeInteger(hostPid) || hostPid <= 0 || hostPid !== process.pid) {
|
|
108
|
+
throw ownerError(
|
|
109
|
+
'pi_bg_reload_owner_stale_claim',
|
|
110
|
+
`reload owner host pid must equal the current process pid ${String(process.pid)}`,
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
if (typeof sessionId !== 'string' || sessionId.length === 0) {
|
|
114
|
+
throw ownerError('pi_bg_reload_owner_stale_claim', 'reload owner session id must be non-empty');
|
|
115
|
+
}
|
|
116
|
+
if (typeof cwdRealpath !== 'string' || cwdRealpath.length === 0 || !isAbsolute(cwdRealpath)) {
|
|
117
|
+
throw ownerError(
|
|
118
|
+
'pi_bg_reload_owner_stale_claim',
|
|
119
|
+
'reload owner cwd identity must be a non-empty absolute canonical path',
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
return Object.freeze({ hostPid, sessionId, cwdRealpath });
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function reloadShellIdentityKey(identity: ReloadShellIdentityV1): string {
|
|
126
|
+
if (identity.hostPid !== process.pid) {
|
|
127
|
+
throw ownerError('pi_bg_reload_owner_stale_claim', 'reload owner identity belongs to another process');
|
|
128
|
+
}
|
|
129
|
+
return `${lengthPart(String(identity.hostPid))}${lengthPart(identity.sessionId)}${lengthPart(identity.cwdRealpath)}`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function structuralHub(value: unknown): value is ReloadShellOwnerHubV1 {
|
|
133
|
+
if (typeof value !== 'object' || value === null) return false;
|
|
134
|
+
if (Reflect.get(value, 'protocol') !== RELOAD_SHELL_OWNER_PROTOCOL) return false;
|
|
135
|
+
if (typeof Reflect.get(value, 'hubNonce') !== 'string') return false;
|
|
136
|
+
for (const method of [
|
|
137
|
+
'beginActivation',
|
|
138
|
+
'commitActivation',
|
|
139
|
+
'abortActivation',
|
|
140
|
+
'beginReloadHandoff',
|
|
141
|
+
'releaseActivation',
|
|
142
|
+
'registerExecution',
|
|
143
|
+
'markAdmissionCommitted',
|
|
144
|
+
'releaseExecution',
|
|
145
|
+
'isCurrentLease',
|
|
146
|
+
]) {
|
|
147
|
+
if (typeof Reflect.get(value, method) !== 'function') return false;
|
|
148
|
+
}
|
|
149
|
+
return true;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function sameIdentity(left: ReloadShellIdentityV1, right: ReloadShellIdentityV1): boolean {
|
|
153
|
+
return (
|
|
154
|
+
left.hostPid === right.hostPid &&
|
|
155
|
+
left.sessionId === right.sessionId &&
|
|
156
|
+
left.cwdRealpath === right.cwdRealpath
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function clearHandoffTimer(slot: OwnerSlot): void {
|
|
161
|
+
if (slot.handoffTimer !== undefined) {
|
|
162
|
+
clearTimeout(slot.handoffTimer);
|
|
163
|
+
slot.handoffTimer = undefined;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function boundedError(error: unknown): string {
|
|
168
|
+
const text = (error instanceof Error ? error.message : String(error)).replace(/\s+/gu, ' ').trim();
|
|
169
|
+
return text.length <= 500 ? text : `${text.slice(0, 499)}…`;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function createReloadShellOwnerHubForTests(
|
|
173
|
+
dependencies: ReloadShellOwnerDependencies = {},
|
|
174
|
+
): ReloadShellOwnerHubV1 {
|
|
175
|
+
const now = dependencies.now ?? Date.now;
|
|
176
|
+
const nextNonce = dependencies.randomNonce ?? randomNonce;
|
|
177
|
+
const logger = dependencies.logger ?? console;
|
|
178
|
+
const handoffTimeoutMs = positiveTimeout(
|
|
179
|
+
dependencies.handoffTimeoutMs,
|
|
180
|
+
RELOAD_SHELL_HANDOFF_TIMEOUT_MS,
|
|
181
|
+
);
|
|
182
|
+
const slots = new Map<string, OwnerSlot>();
|
|
183
|
+
const terminalReleaseContinuations = new WeakMap<
|
|
184
|
+
ReloadableShellExecutionV1,
|
|
185
|
+
Promise<void>
|
|
186
|
+
>();
|
|
187
|
+
const hubNonce = nextNonce();
|
|
188
|
+
requireNonce(hubNonce, 'hub nonce');
|
|
189
|
+
|
|
190
|
+
const failStale = (message: string): never => {
|
|
191
|
+
throw ownerError('pi_bg_reload_owner_stale_claim', message);
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
const currentSlotForLease = (lease: ReloadShellActivationLeaseV1): OwnerSlot => {
|
|
195
|
+
if (lease.protocol !== RELOAD_SHELL_OWNER_PROTOCOL || lease.hubNonce !== hubNonce) {
|
|
196
|
+
return failStale('activation lease belongs to another owner protocol instance');
|
|
197
|
+
}
|
|
198
|
+
const slot = slots.get(lease.identityKey);
|
|
199
|
+
if (
|
|
200
|
+
slot === undefined ||
|
|
201
|
+
slot.phase !== 'bound' ||
|
|
202
|
+
slot.lease?.generation !== lease.generation ||
|
|
203
|
+
slot.lease.activationNonce !== lease.activationNonce
|
|
204
|
+
) {
|
|
205
|
+
return failStale('activation lease is stale or no longer bound');
|
|
206
|
+
}
|
|
207
|
+
return slot;
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
const slotForClaim = (claim: ReloadShellActivationClaimV1): OwnerSlot => {
|
|
211
|
+
if (claim.protocol !== RELOAD_SHELL_OWNER_PROTOCOL || claim.hubNonce !== hubNonce) {
|
|
212
|
+
return failStale('activation claim belongs to another owner protocol instance');
|
|
213
|
+
}
|
|
214
|
+
const slot = slots.get(claim.identityKey);
|
|
215
|
+
if (
|
|
216
|
+
slot === undefined ||
|
|
217
|
+
slot.phase !== 'claiming' ||
|
|
218
|
+
slot.claim !== claim ||
|
|
219
|
+
slot.claimNonce !== claim.claimNonce ||
|
|
220
|
+
slot.generation !== claim.generation ||
|
|
221
|
+
slot.activationNonce !== claim.activationNonce ||
|
|
222
|
+
!sameIdentity(slot.identity, claim.identity)
|
|
223
|
+
) {
|
|
224
|
+
return failStale('activation claim is stale or no longer current');
|
|
225
|
+
}
|
|
226
|
+
if (claim.expiresAt !== undefined && now() >= claim.expiresAt) {
|
|
227
|
+
return failStale('activation claim expired before mutation');
|
|
228
|
+
}
|
|
229
|
+
return slot;
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
const dispatch = (
|
|
233
|
+
slot: OwnerSlot,
|
|
234
|
+
kind: 'changed' | 'terminal',
|
|
235
|
+
execution: ReloadableShellExecutionV1,
|
|
236
|
+
): void => {
|
|
237
|
+
if (slot.executions.get(execution.launchNonce) !== execution) return;
|
|
238
|
+
const adapter = slot.adapter;
|
|
239
|
+
if (slot.phase === 'releasing' && kind === 'terminal') {
|
|
240
|
+
removeExecution(slot, execution);
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
if (slot.phase !== 'bound' || adapter === undefined) {
|
|
244
|
+
if (kind === 'changed') slot.queuedChanged.add(execution.launchNonce);
|
|
245
|
+
else slot.queuedTerminal.add(execution.launchNonce);
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
try {
|
|
249
|
+
if (kind === 'changed') adapter.onChanged(execution);
|
|
250
|
+
else adapter.onTerminal(execution);
|
|
251
|
+
} catch (error) {
|
|
252
|
+
logger.error(
|
|
253
|
+
`[background-tasks] reload owner ${kind} adapter failed for ${execution.task.id}: ${boundedError(error)}`,
|
|
254
|
+
);
|
|
255
|
+
if (kind === 'changed') slot.queuedChanged.add(execution.launchNonce);
|
|
256
|
+
else slot.queuedTerminal.add(execution.launchNonce);
|
|
257
|
+
}
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
function removeExecution(slot: OwnerSlot, execution: ReloadableShellExecutionV1): void {
|
|
261
|
+
if (slot.executions.get(execution.launchNonce) !== execution) return;
|
|
262
|
+
slot.executions.delete(execution.launchNonce);
|
|
263
|
+
slot.queuedChanged.delete(execution.launchNonce);
|
|
264
|
+
slot.queuedTerminal.delete(execution.launchNonce);
|
|
265
|
+
execution.setOwnerEventSink(undefined);
|
|
266
|
+
execution.releaseResources();
|
|
267
|
+
if (
|
|
268
|
+
slot.executions.size === 0 &&
|
|
269
|
+
(slot.phase === 'handoff' || slot.phase === 'releasing' || slot.phase === 'orphaned')
|
|
270
|
+
) {
|
|
271
|
+
clearHandoffTimer(slot);
|
|
272
|
+
slots.delete(slot.identityKey);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const releaseAfterTerminal = (
|
|
277
|
+
slot: OwnerSlot,
|
|
278
|
+
execution: ReloadableShellExecutionV1,
|
|
279
|
+
): Promise<void> => {
|
|
280
|
+
const existing = terminalReleaseContinuations.get(execution);
|
|
281
|
+
if (existing !== undefined) return existing;
|
|
282
|
+
const continuation = execution.terminal.then(
|
|
283
|
+
() => {
|
|
284
|
+
if (
|
|
285
|
+
slots.get(slot.identityKey) !== slot ||
|
|
286
|
+
slot.executions.get(execution.launchNonce) !== execution
|
|
287
|
+
) {
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
if (execution.phase !== 'terminal') {
|
|
291
|
+
logger.error(
|
|
292
|
+
`[background-tasks] retained reload shell execution ${execution.task.id} settled without terminal ownership proof; keeping its owner slot`,
|
|
293
|
+
);
|
|
294
|
+
return;
|
|
295
|
+
}
|
|
296
|
+
try {
|
|
297
|
+
removeExecution(slot, execution);
|
|
298
|
+
} catch (error) {
|
|
299
|
+
logger.error(
|
|
300
|
+
`[background-tasks] retained reload shell execution cleanup failed for ${execution.task.id}: ${boundedError(error)}`,
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
},
|
|
304
|
+
(error: unknown) => {
|
|
305
|
+
logger.error(
|
|
306
|
+
`[background-tasks] retained reload shell execution terminal promise rejected for ${execution.task.id}; keeping its owner slot: ${boundedError(error)}`,
|
|
307
|
+
);
|
|
308
|
+
},
|
|
309
|
+
);
|
|
310
|
+
terminalReleaseContinuations.set(execution, continuation);
|
|
311
|
+
return continuation;
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
const expireHandoff = (slot: OwnerSlot, deadline: number): void => {
|
|
315
|
+
if (slots.get(slot.identityKey) !== slot) return;
|
|
316
|
+
if (
|
|
317
|
+
(slot.phase !== 'handoff' && slot.phase !== 'claiming') ||
|
|
318
|
+
slot.expiresAt !== deadline ||
|
|
319
|
+
now() < deadline
|
|
320
|
+
) {
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
slot.phase = 'orphaned';
|
|
324
|
+
slot.adapter = undefined;
|
|
325
|
+
slot.claim = undefined;
|
|
326
|
+
delete slot.claimNonce;
|
|
327
|
+
clearHandoffTimer(slot);
|
|
328
|
+
const executions = [...slot.executions.values()];
|
|
329
|
+
void Promise.allSettled(
|
|
330
|
+
executions.map(async (execution) => {
|
|
331
|
+
execution.abandonReloadHandoff();
|
|
332
|
+
const terminalRelease = releaseAfterTerminal(slot, execution);
|
|
333
|
+
try {
|
|
334
|
+
if (execution.phase === 'running' || execution.phase === 'stop_requested') {
|
|
335
|
+
await execution.requestStop(
|
|
336
|
+
'handoff_expired',
|
|
337
|
+
`pi_bg_reload_handoff_expired: no compatible reload activation claimed the live shell execution before the ${String(handoffTimeoutMs)}ms handoff deadline`,
|
|
338
|
+
);
|
|
339
|
+
} else if (execution.phase === 'starting') {
|
|
340
|
+
execution.failAdmission(
|
|
341
|
+
ownerError(
|
|
342
|
+
'pi_bg_reload_handoff_expired',
|
|
343
|
+
'uncommitted reload shell execution reached handoff expiry',
|
|
344
|
+
),
|
|
345
|
+
);
|
|
346
|
+
await execution.requestStop('handoff_expired');
|
|
347
|
+
} else if (execution.phase === 'finalizing') {
|
|
348
|
+
await execution.terminal;
|
|
349
|
+
}
|
|
350
|
+
} catch (error) {
|
|
351
|
+
logger.error(
|
|
352
|
+
`[background-tasks] reload handoff expiry could not settle ${execution.task.id}: ${boundedError(error)}`,
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
await terminalRelease;
|
|
356
|
+
}),
|
|
357
|
+
).then(() => {
|
|
358
|
+
if (slots.get(slot.identityKey) === slot && slot.executions.size === 0) {
|
|
359
|
+
slots.delete(slot.identityKey);
|
|
360
|
+
}
|
|
361
|
+
});
|
|
362
|
+
logger.error(
|
|
363
|
+
`[background-tasks] pi_bg_reload_handoff_expired: ${String(executions.length)} live reload shell execution(s) were not claimed before the fixed handoff deadline`,
|
|
364
|
+
);
|
|
365
|
+
};
|
|
366
|
+
|
|
367
|
+
const armHandoffDeadline = (slot: OwnerSlot): void => {
|
|
368
|
+
const deadline = slot.expiresAt;
|
|
369
|
+
if (deadline === undefined || slot.handoffTimer !== undefined) return;
|
|
370
|
+
const remaining = Math.max(0, deadline - now());
|
|
371
|
+
slot.handoffTimer = setTimeout(() => {
|
|
372
|
+
slot.handoffTimer = undefined;
|
|
373
|
+
expireHandoff(slot, deadline);
|
|
374
|
+
}, remaining);
|
|
375
|
+
// Deliberately referenced. The retained child/tree authority must keep the
|
|
376
|
+
// process alive until a claimant or orphan cleanup owns it.
|
|
377
|
+
};
|
|
378
|
+
|
|
379
|
+
const hub = Object.assign(Object.create(null), {
|
|
380
|
+
protocol: RELOAD_SHELL_OWNER_PROTOCOL,
|
|
381
|
+
hubNonce,
|
|
382
|
+
beginActivation(
|
|
383
|
+
identity: ReloadShellIdentityV1,
|
|
384
|
+
startReason: string,
|
|
385
|
+
activationNonce: string,
|
|
386
|
+
): ReloadShellActivationClaimV1 {
|
|
387
|
+
requireNonce(activationNonce, 'activation nonce');
|
|
388
|
+
const identityKey = reloadShellIdentityKey(identity);
|
|
389
|
+
let slot = slots.get(identityKey);
|
|
390
|
+
if (slot === undefined) {
|
|
391
|
+
const claimNonce = nextNonce();
|
|
392
|
+
requireNonce(claimNonce, 'claim nonce');
|
|
393
|
+
slot = {
|
|
394
|
+
identity: Object.freeze({ ...identity }),
|
|
395
|
+
identityKey,
|
|
396
|
+
generation: 1,
|
|
397
|
+
handoffCount: 0,
|
|
398
|
+
phase: 'claiming',
|
|
399
|
+
activationNonce,
|
|
400
|
+
claimNonce,
|
|
401
|
+
executions: new Map(),
|
|
402
|
+
queuedChanged: new Set(),
|
|
403
|
+
queuedTerminal: new Set(),
|
|
404
|
+
};
|
|
405
|
+
const claim: ReloadShellActivationClaimV1 = Object.freeze({
|
|
406
|
+
protocol: RELOAD_SHELL_OWNER_PROTOCOL,
|
|
407
|
+
hubNonce,
|
|
408
|
+
claimNonce,
|
|
409
|
+
identity: slot.identity,
|
|
410
|
+
identityKey,
|
|
411
|
+
generation: 1,
|
|
412
|
+
activationNonce,
|
|
413
|
+
executions: Object.freeze([]),
|
|
414
|
+
});
|
|
415
|
+
slot.claim = claim;
|
|
416
|
+
slots.set(identityKey, slot);
|
|
417
|
+
return claim;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
if (!sameIdentity(slot.identity, identity)) {
|
|
421
|
+
throw ownerError(
|
|
422
|
+
'pi_bg_reload_owner_activation_conflict',
|
|
423
|
+
'another activation owns a non-identical reload shell identity under this key',
|
|
424
|
+
);
|
|
425
|
+
}
|
|
426
|
+
if (slot.phase !== 'handoff' || startReason !== 'reload') {
|
|
427
|
+
throw ownerError(
|
|
428
|
+
'pi_bg_reload_owner_activation_conflict',
|
|
429
|
+
`another package activation already owns session ${JSON.stringify(identity.sessionId)} in state ${slot.phase}`,
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
if (slot.expiresAt === undefined || now() >= slot.expiresAt) {
|
|
433
|
+
return failStale('reload handoff expired before activation claim');
|
|
434
|
+
}
|
|
435
|
+
slot.phase = 'claiming';
|
|
436
|
+
slot.generation += 1;
|
|
437
|
+
slot.activationNonce = activationNonce;
|
|
438
|
+
const claimNonce = nextNonce();
|
|
439
|
+
requireNonce(claimNonce, 'claim nonce');
|
|
440
|
+
slot.claimNonce = claimNonce;
|
|
441
|
+
const claim: ReloadShellActivationClaimV1 = Object.freeze({
|
|
442
|
+
protocol: RELOAD_SHELL_OWNER_PROTOCOL,
|
|
443
|
+
hubNonce,
|
|
444
|
+
claimNonce,
|
|
445
|
+
identity: slot.identity,
|
|
446
|
+
identityKey,
|
|
447
|
+
generation: slot.generation,
|
|
448
|
+
activationNonce,
|
|
449
|
+
expiresAt: slot.expiresAt,
|
|
450
|
+
executions: Object.freeze(
|
|
451
|
+
[...slot.executions.values()].filter((execution) => execution.admissionCommitted),
|
|
452
|
+
),
|
|
453
|
+
});
|
|
454
|
+
slot.claim = claim;
|
|
455
|
+
return claim;
|
|
456
|
+
},
|
|
457
|
+
commitActivation(
|
|
458
|
+
claim: ReloadShellActivationClaimV1,
|
|
459
|
+
adapter: ReloadShellHostAdapterV1,
|
|
460
|
+
): ReloadShellActivationLeaseV1 {
|
|
461
|
+
const slot = slotForClaim(claim);
|
|
462
|
+
if (adapter.activationNonce !== claim.activationNonce) {
|
|
463
|
+
return failStale('host adapter activation nonce does not match its claim');
|
|
464
|
+
}
|
|
465
|
+
const lease: ReloadShellActivationLeaseV1 = Object.freeze({
|
|
466
|
+
protocol: RELOAD_SHELL_OWNER_PROTOCOL,
|
|
467
|
+
hubNonce,
|
|
468
|
+
identityKey: slot.identityKey,
|
|
469
|
+
generation: slot.generation,
|
|
470
|
+
activationNonce: slot.activationNonce,
|
|
471
|
+
});
|
|
472
|
+
try {
|
|
473
|
+
adapter.onBound(lease);
|
|
474
|
+
} catch (error) {
|
|
475
|
+
slot.phase = slot.expiresAt === undefined ? 'releasing' : 'handoff';
|
|
476
|
+
slot.claim = undefined;
|
|
477
|
+
delete slot.claimNonce;
|
|
478
|
+
if (slot.expiresAt === undefined || slot.executions.size === 0) {
|
|
479
|
+
clearHandoffTimer(slot);
|
|
480
|
+
slots.delete(slot.identityKey);
|
|
481
|
+
} else {
|
|
482
|
+
armHandoffDeadline(slot);
|
|
483
|
+
}
|
|
484
|
+
throw error;
|
|
485
|
+
}
|
|
486
|
+
slot.lease = lease;
|
|
487
|
+
slot.claim = undefined;
|
|
488
|
+
delete slot.claimNonce;
|
|
489
|
+
slot.phase = 'bound';
|
|
490
|
+
slot.adapter = adapter;
|
|
491
|
+
clearHandoffTimer(slot);
|
|
492
|
+
delete slot.expiresAt;
|
|
493
|
+
|
|
494
|
+
for (const execution of claim.executions) {
|
|
495
|
+
execution.updateLeaseAudit(slot.generation, slot.handoffCount);
|
|
496
|
+
slot.queuedChanged.add(execution.launchNonce);
|
|
497
|
+
if (execution.phase === 'terminal') slot.queuedTerminal.add(execution.launchNonce);
|
|
498
|
+
}
|
|
499
|
+
for (const nonce of [...slot.queuedChanged]) {
|
|
500
|
+
const execution = slot.executions.get(nonce);
|
|
501
|
+
if (execution === undefined) continue;
|
|
502
|
+
slot.queuedChanged.delete(nonce);
|
|
503
|
+
dispatch(slot, 'changed', execution);
|
|
504
|
+
}
|
|
505
|
+
for (const nonce of [...slot.queuedTerminal]) {
|
|
506
|
+
const execution = slot.executions.get(nonce);
|
|
507
|
+
if (execution === undefined) continue;
|
|
508
|
+
slot.queuedTerminal.delete(nonce);
|
|
509
|
+
dispatch(slot, 'terminal', execution);
|
|
510
|
+
}
|
|
511
|
+
return lease;
|
|
512
|
+
},
|
|
513
|
+
abortActivation(claim: ReloadShellActivationClaimV1, error: unknown): void {
|
|
514
|
+
void error;
|
|
515
|
+
const slot = slotForClaim(claim);
|
|
516
|
+
slot.claim = undefined;
|
|
517
|
+
delete slot.claimNonce;
|
|
518
|
+
if (slot.expiresAt === undefined) {
|
|
519
|
+
slot.phase = 'releasing';
|
|
520
|
+
if (slot.executions.size === 0) slots.delete(slot.identityKey);
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
slot.phase = 'handoff';
|
|
524
|
+
if (slot.executions.size === 0) {
|
|
525
|
+
clearHandoffTimer(slot);
|
|
526
|
+
slots.delete(slot.identityKey);
|
|
527
|
+
return;
|
|
528
|
+
}
|
|
529
|
+
armHandoffDeadline(slot);
|
|
530
|
+
},
|
|
531
|
+
beginReloadHandoff(
|
|
532
|
+
lease: ReloadShellActivationLeaseV1,
|
|
533
|
+
): readonly ReloadableShellExecutionV1[] {
|
|
534
|
+
const slot = currentSlotForLease(lease);
|
|
535
|
+
// Detach first: no child/timer callback after this line can reach an old
|
|
536
|
+
// registry or Pi closure.
|
|
537
|
+
slot.adapter = undefined;
|
|
538
|
+
slot.phase = 'handoff';
|
|
539
|
+
slot.handoffCount += 1;
|
|
540
|
+
slot.expiresAt = now() + handoffTimeoutMs;
|
|
541
|
+
slot.lease = undefined;
|
|
542
|
+
const transferred: ReloadableShellExecutionV1[] = [];
|
|
543
|
+
for (const execution of [...slot.executions.values()]) {
|
|
544
|
+
if (execution.admissionCommitted) {
|
|
545
|
+
slot.queuedChanged.add(execution.launchNonce);
|
|
546
|
+
if (execution.phase === 'terminal') slot.queuedTerminal.add(execution.launchNonce);
|
|
547
|
+
transferred.push(execution);
|
|
548
|
+
continue;
|
|
549
|
+
}
|
|
550
|
+
// Admission closure remains old-registry owned and uncommitted work is
|
|
551
|
+
// never claimable. Keep its process authority in the hub until terminal
|
|
552
|
+
// settlement, without requiring either the old or fresh host adapter.
|
|
553
|
+
void releaseAfterTerminal(slot, execution);
|
|
554
|
+
}
|
|
555
|
+
if (slot.executions.size === 0) {
|
|
556
|
+
clearHandoffTimer(slot);
|
|
557
|
+
slots.delete(slot.identityKey);
|
|
558
|
+
return Object.freeze([]);
|
|
559
|
+
}
|
|
560
|
+
armHandoffDeadline(slot);
|
|
561
|
+
return Object.freeze(transferred);
|
|
562
|
+
},
|
|
563
|
+
releaseActivation(lease: ReloadShellActivationLeaseV1): void {
|
|
564
|
+
const slot = currentSlotForLease(lease);
|
|
565
|
+
slot.adapter = undefined;
|
|
566
|
+
slot.lease = undefined;
|
|
567
|
+
slot.phase = 'releasing';
|
|
568
|
+
clearHandoffTimer(slot);
|
|
569
|
+
for (const execution of [...slot.executions.values()]) {
|
|
570
|
+
if (execution.phase === 'terminal') removeExecution(slot, execution);
|
|
571
|
+
}
|
|
572
|
+
if (slot.executions.size === 0) slots.delete(slot.identityKey);
|
|
573
|
+
},
|
|
574
|
+
registerExecution(
|
|
575
|
+
lease: ReloadShellActivationLeaseV1,
|
|
576
|
+
execution: ReloadableShellExecutionV1,
|
|
577
|
+
): void {
|
|
578
|
+
const slot = currentSlotForLease(lease);
|
|
579
|
+
if (execution.protocol !== RELOAD_SHELL_OWNER_PROTOCOL) {
|
|
580
|
+
throw ownerError(
|
|
581
|
+
'pi_bg_reload_owner_protocol_incompatible',
|
|
582
|
+
'reload shell execution protocol is incompatible',
|
|
583
|
+
);
|
|
584
|
+
}
|
|
585
|
+
if (slot.executions.has(execution.launchNonce)) {
|
|
586
|
+
throw ownerError(
|
|
587
|
+
'pi_bg_reload_owner_activation_conflict',
|
|
588
|
+
`reload shell launch nonce ${execution.launchNonce} is already registered`,
|
|
589
|
+
);
|
|
590
|
+
}
|
|
591
|
+
slot.executions.set(execution.launchNonce, execution);
|
|
592
|
+
execution.setOwnerEventSink({
|
|
593
|
+
onChanged: (changed) => dispatch(slot, 'changed', changed),
|
|
594
|
+
onTerminal: (terminal) => dispatch(slot, 'terminal', terminal),
|
|
595
|
+
});
|
|
596
|
+
},
|
|
597
|
+
markAdmissionCommitted(
|
|
598
|
+
lease: ReloadShellActivationLeaseV1,
|
|
599
|
+
execution: ReloadableShellExecutionV1,
|
|
600
|
+
): void {
|
|
601
|
+
const slot = currentSlotForLease(lease);
|
|
602
|
+
if (slot.executions.get(execution.launchNonce) !== execution) {
|
|
603
|
+
return failStale('cannot commit an execution not registered to this activation');
|
|
604
|
+
}
|
|
605
|
+
execution.markAdmissionCommitted(slot.generation, slot.handoffCount);
|
|
606
|
+
},
|
|
607
|
+
releaseExecution(
|
|
608
|
+
leaseOrClaim: ReloadShellActivationLeaseV1 | ReloadShellActivationClaimV1,
|
|
609
|
+
execution: ReloadableShellExecutionV1,
|
|
610
|
+
): void {
|
|
611
|
+
let slot: OwnerSlot;
|
|
612
|
+
if ('claimNonce' in leaseOrClaim) slot = slotForClaim(leaseOrClaim);
|
|
613
|
+
else slot = currentSlotForLease(leaseOrClaim);
|
|
614
|
+
if (slot.executions.get(execution.launchNonce) !== execution) {
|
|
615
|
+
return failStale('cannot release an execution not owned by this activation');
|
|
616
|
+
}
|
|
617
|
+
if (execution.phase === 'terminal') {
|
|
618
|
+
removeExecution(slot, execution);
|
|
619
|
+
return;
|
|
620
|
+
}
|
|
621
|
+
// A bounded stop wait is not terminal proof. Retain the child, streams,
|
|
622
|
+
// listeners, tree state, and timers under this slot until the execution's
|
|
623
|
+
// one terminal continuation can release them safely.
|
|
624
|
+
void releaseAfterTerminal(slot, execution);
|
|
625
|
+
},
|
|
626
|
+
isCurrentLease(lease: ReloadShellActivationLeaseV1): boolean {
|
|
627
|
+
try {
|
|
628
|
+
currentSlotForLease(lease);
|
|
629
|
+
return true;
|
|
630
|
+
} catch {
|
|
631
|
+
return false;
|
|
632
|
+
}
|
|
633
|
+
},
|
|
634
|
+
}) as ReloadShellOwnerHubV1;
|
|
635
|
+
|
|
636
|
+
const internalState = { slots };
|
|
637
|
+
hubStates.set(hub, internalState);
|
|
638
|
+
Object.defineProperty(hub, OWNER_STATE_SYMBOL, {
|
|
639
|
+
value: internalState,
|
|
640
|
+
enumerable: false,
|
|
641
|
+
configurable: false,
|
|
642
|
+
writable: false,
|
|
643
|
+
});
|
|
644
|
+
return hub;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
export function getProcessReloadShellOwnerV1(): ReloadShellOwnerHubV1 {
|
|
648
|
+
const current = Reflect.get(globalThis, RELOAD_SHELL_OWNER_SYMBOL);
|
|
649
|
+
if (current !== undefined) {
|
|
650
|
+
if (!structuralHub(current)) {
|
|
651
|
+
throw ownerError(
|
|
652
|
+
'pi_bg_reload_owner_protocol_incompatible',
|
|
653
|
+
`global symbol ${RELOAD_SHELL_OWNER_PROTOCOL} contains an incompatible value`,
|
|
654
|
+
);
|
|
655
|
+
}
|
|
656
|
+
return current;
|
|
657
|
+
}
|
|
658
|
+
const created = createReloadShellOwnerHubForTests();
|
|
659
|
+
Reflect.set(globalThis, RELOAD_SHELL_OWNER_SYMBOL, created);
|
|
660
|
+
return created;
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
export interface ReloadShellOwnerInspection {
|
|
664
|
+
readonly phase: string | undefined;
|
|
665
|
+
readonly generation: number | undefined;
|
|
666
|
+
readonly expiresAt: number | undefined;
|
|
667
|
+
readonly hasAdapter: boolean;
|
|
668
|
+
readonly executions: readonly ReloadableShellExecutionV1[];
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/** Direct-source-only deterministic inspection seam; not exported by a package facade. */
|
|
672
|
+
export function inspectReloadShellOwnerForTests(
|
|
673
|
+
hub: ReloadShellOwnerHubV1,
|
|
674
|
+
identity: ReloadShellIdentityV1,
|
|
675
|
+
): ReloadShellOwnerInspection {
|
|
676
|
+
const localState = hubStates.get(hub);
|
|
677
|
+
const reflectedState = Reflect.get(hub, OWNER_STATE_SYMBOL);
|
|
678
|
+
const state =
|
|
679
|
+
localState ??
|
|
680
|
+
(typeof reflectedState === 'object' && reflectedState !== null
|
|
681
|
+
? (reflectedState as InternalHubState)
|
|
682
|
+
: undefined);
|
|
683
|
+
if (state === undefined) {
|
|
684
|
+
return {
|
|
685
|
+
phase: undefined,
|
|
686
|
+
generation: undefined,
|
|
687
|
+
expiresAt: undefined,
|
|
688
|
+
hasAdapter: false,
|
|
689
|
+
executions: Object.freeze([]),
|
|
690
|
+
};
|
|
691
|
+
}
|
|
692
|
+
const slot = state.slots.get(reloadShellIdentityKey(identity));
|
|
693
|
+
return {
|
|
694
|
+
phase: slot?.phase,
|
|
695
|
+
generation: slot?.generation,
|
|
696
|
+
expiresAt: slot?.expiresAt,
|
|
697
|
+
hasAdapter: slot?.adapter !== undefined,
|
|
698
|
+
executions: Object.freeze(slot === undefined ? [] : [...slot.executions.values()]),
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
interface ContextUsagePayload {
|
|
703
|
+
readonly contextWindow?: unknown;
|
|
704
|
+
readonly tokens?: unknown;
|
|
705
|
+
readonly percent?: unknown;
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
interface TokenUsagePayload {
|
|
709
|
+
readonly input?: unknown;
|
|
710
|
+
readonly output?: unknown;
|
|
711
|
+
readonly cacheRead?: unknown;
|
|
712
|
+
readonly cacheWrite?: unknown;
|
|
713
|
+
readonly totalTokens?: unknown;
|
|
714
|
+
readonly costTotal?: unknown;
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
interface ToolUsagePayload {
|
|
718
|
+
readonly byName?: unknown;
|
|
719
|
+
readonly failed?: unknown;
|
|
720
|
+
readonly total?: unknown;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
function nonNegativeInteger(value: unknown): number {
|
|
724
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0
|
|
725
|
+
? Math.floor(value)
|
|
726
|
+
: 0;
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
function normalizeContextUsage(value: unknown): TaskContextUsage | undefined {
|
|
730
|
+
if (!isJsonObject(value)) return undefined;
|
|
731
|
+
const input: ContextUsagePayload = value;
|
|
732
|
+
const contextWindow =
|
|
733
|
+
typeof input.contextWindow === 'number' &&
|
|
734
|
+
Number.isFinite(input.contextWindow) &&
|
|
735
|
+
input.contextWindow > 0
|
|
736
|
+
? Math.floor(input.contextWindow)
|
|
737
|
+
: undefined;
|
|
738
|
+
if (contextWindow === undefined) return undefined;
|
|
739
|
+
const tokens =
|
|
740
|
+
input.tokens === null
|
|
741
|
+
? null
|
|
742
|
+
: typeof input.tokens === 'number' && Number.isFinite(input.tokens) && input.tokens >= 0
|
|
743
|
+
? Math.floor(input.tokens)
|
|
744
|
+
: null;
|
|
745
|
+
const percent =
|
|
746
|
+
input.percent === null
|
|
747
|
+
? null
|
|
748
|
+
: typeof input.percent === 'number' && Number.isFinite(input.percent) && input.percent >= 0
|
|
749
|
+
? input.percent
|
|
750
|
+
: tokens === null
|
|
751
|
+
? null
|
|
752
|
+
: (tokens / contextWindow) * 100;
|
|
753
|
+
return { tokens, contextWindow, percent };
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function normalizeTokenUsage(value: unknown): TaskTokenUsage | undefined {
|
|
757
|
+
if (!isJsonObject(value)) return undefined;
|
|
758
|
+
const input: TokenUsagePayload = value;
|
|
759
|
+
const usage: TaskTokenUsage = {
|
|
760
|
+
input: nonNegativeInteger(input.input),
|
|
761
|
+
output: nonNegativeInteger(input.output),
|
|
762
|
+
cacheRead: nonNegativeInteger(input.cacheRead),
|
|
763
|
+
cacheWrite: nonNegativeInteger(input.cacheWrite),
|
|
764
|
+
totalTokens: nonNegativeInteger(input.totalTokens),
|
|
765
|
+
};
|
|
766
|
+
if (usage.totalTokens <= 0) {
|
|
767
|
+
usage.totalTokens = usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
|
|
768
|
+
}
|
|
769
|
+
if (
|
|
770
|
+
typeof input.costTotal === 'number' &&
|
|
771
|
+
Number.isFinite(input.costTotal) &&
|
|
772
|
+
input.costTotal >= 0
|
|
773
|
+
) {
|
|
774
|
+
usage.costTotal = input.costTotal;
|
|
775
|
+
}
|
|
776
|
+
return usage.totalTokens > 0 ? usage : undefined;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
function normalizeToolUsage(value: unknown): TaskToolUsage | undefined {
|
|
780
|
+
if (!isJsonObject(value)) return undefined;
|
|
781
|
+
const input: ToolUsagePayload = value;
|
|
782
|
+
const byName: Record<string, number> = {};
|
|
783
|
+
if (isJsonObject(input.byName)) {
|
|
784
|
+
for (const [name, count] of Object.entries(input.byName)) {
|
|
785
|
+
const normalized = nonNegativeInteger(count);
|
|
786
|
+
if (normalized > 0) byName[name] = normalized;
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
const failed = nonNegativeInteger(input.failed);
|
|
790
|
+
const total = Math.max(
|
|
791
|
+
nonNegativeInteger(input.total),
|
|
792
|
+
failed,
|
|
793
|
+
Object.values(byName).reduce((sum, count) => sum + count, 0),
|
|
794
|
+
);
|
|
795
|
+
return total > 0 || failed > 0 ? { total, failed, byName } : undefined;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
function normalizeModel(value: unknown): string | undefined {
|
|
799
|
+
if (typeof value !== 'string') return undefined;
|
|
800
|
+
const trimmed = value.trim();
|
|
801
|
+
if (trimmed.length === 0) return undefined;
|
|
802
|
+
return trimmed.length <= 120 ? trimmed : trimmed.slice(0, 120);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function parseContextUsageXml(xml: string): TaskContextUsage | undefined {
|
|
806
|
+
const number = (tag: string): number | null | undefined => {
|
|
807
|
+
const match = new RegExp(`<${tag}>(.*?)</${tag}>`, 'iu').exec(xml);
|
|
808
|
+
if (match === null) return undefined;
|
|
809
|
+
const raw = match[1]?.trim();
|
|
810
|
+
if (raw === 'null' || raw === '?') return null;
|
|
811
|
+
const parsed = Number(raw);
|
|
812
|
+
return Number.isFinite(parsed) ? parsed : undefined;
|
|
813
|
+
};
|
|
814
|
+
return normalizeContextUsage({
|
|
815
|
+
tokens: number('tokens'),
|
|
816
|
+
contextWindow: number('context-window') ?? number('contextWindow'),
|
|
817
|
+
percent: number('percent'),
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
interface PosixTreeState {
|
|
822
|
+
readonly groupId: number;
|
|
823
|
+
readonly deadlineAt: number;
|
|
824
|
+
readonly completion: Promise<void>;
|
|
825
|
+
readonly resolve: () => void;
|
|
826
|
+
forceAttempted: boolean;
|
|
827
|
+
settled: boolean;
|
|
828
|
+
failure?: Error | undefined;
|
|
829
|
+
lastProbeError?: Error | undefined;
|
|
830
|
+
verificationTimer?: NodeJS.Timeout | undefined;
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
interface WindowsTreeState {
|
|
834
|
+
readonly pid: number;
|
|
835
|
+
readonly completion: Promise<void>;
|
|
836
|
+
readonly resolve: () => void;
|
|
837
|
+
softController?: AbortController | undefined;
|
|
838
|
+
softPromise?: Promise<void> | undefined;
|
|
839
|
+
forcePromise?: Promise<void> | undefined;
|
|
840
|
+
forceAttempted: boolean;
|
|
841
|
+
settled: boolean;
|
|
842
|
+
failure?: Error | undefined;
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
export interface ReloadableShellExecutionOptions {
|
|
846
|
+
readonly task: BgTask;
|
|
847
|
+
readonly identity: ReloadShellIdentityV1;
|
|
848
|
+
readonly lease: ReloadShellActivationLeaseV1;
|
|
849
|
+
readonly launchNonce: string;
|
|
850
|
+
readonly invocation: {
|
|
851
|
+
readonly shell: string;
|
|
852
|
+
readonly args: readonly string[];
|
|
853
|
+
readonly windowsVerbatimArguments: boolean;
|
|
854
|
+
};
|
|
855
|
+
readonly spawn?: (
|
|
856
|
+
command: string,
|
|
857
|
+
args: string[],
|
|
858
|
+
options: SpawnOptions,
|
|
859
|
+
) => ReloadShellProcessV1;
|
|
860
|
+
readonly killProcess?: (pid: number, signal?: NodeJS.Signals | number) => boolean;
|
|
861
|
+
readonly killTree?: (
|
|
862
|
+
pid: number,
|
|
863
|
+
phase: WindowsKillPhase,
|
|
864
|
+
signal?: AbortSignal,
|
|
865
|
+
) => Promise<TaskkillOutcome>;
|
|
866
|
+
readonly platform?: NodeJS.Platform;
|
|
867
|
+
readonly env?: NodeJS.ProcessEnv;
|
|
868
|
+
readonly maxOutputBytes: number;
|
|
869
|
+
readonly killGraceMs: number;
|
|
870
|
+
readonly stopWaitMs: number;
|
|
871
|
+
readonly now?: (() => number) | undefined;
|
|
872
|
+
readonly logger?: Pick<Console, 'error'> | undefined;
|
|
873
|
+
}
|
|
874
|
+
|
|
875
|
+
async function writeOwnerMetadata(
|
|
876
|
+
path: string,
|
|
877
|
+
value: unknown,
|
|
878
|
+
signal?: AbortSignal,
|
|
879
|
+
): Promise<void> {
|
|
880
|
+
await replaceFileDurable(
|
|
881
|
+
path,
|
|
882
|
+
`${JSON.stringify(value, null, 2)}\n`,
|
|
883
|
+
signal === undefined ? {} : { signal },
|
|
884
|
+
);
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
async function closeOwnerOutputStream(
|
|
888
|
+
stream: ReturnType<typeof createWriteStream> | undefined,
|
|
889
|
+
): Promise<void> {
|
|
890
|
+
if (stream === undefined || stream.destroyed || stream.closed) return;
|
|
891
|
+
await new Promise<void>((resolve, reject) => {
|
|
892
|
+
let settled = false;
|
|
893
|
+
const finish = () => {
|
|
894
|
+
if (settled) return;
|
|
895
|
+
settled = true;
|
|
896
|
+
stream.off('error', fail);
|
|
897
|
+
stream.off('close', finish);
|
|
898
|
+
stream.off('finish', finish);
|
|
899
|
+
resolve();
|
|
900
|
+
};
|
|
901
|
+
const fail = (error: Error) => {
|
|
902
|
+
if (settled) return;
|
|
903
|
+
settled = true;
|
|
904
|
+
stream.off('close', finish);
|
|
905
|
+
stream.off('finish', finish);
|
|
906
|
+
reject(error);
|
|
907
|
+
};
|
|
908
|
+
stream.once('close', finish);
|
|
909
|
+
stream.once('finish', finish);
|
|
910
|
+
stream.once('error', fail);
|
|
911
|
+
stream.end();
|
|
912
|
+
});
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
function appendError(existing: string | undefined, next: string): string {
|
|
916
|
+
if (existing === undefined || existing.length === 0) return next;
|
|
917
|
+
return existing.includes(next) ? existing : `${existing}; ${next}`;
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
function taskkillDescription(outcome: TaskkillOutcome): string {
|
|
921
|
+
return [
|
|
922
|
+
`exit=${String(outcome.exitCode)}`,
|
|
923
|
+
`signal=${String(outcome.signal)}`,
|
|
924
|
+
outcome.stdout.length > 0 ? `stdout=${JSON.stringify(outcome.stdout)}` : '',
|
|
925
|
+
outcome.stderr.length > 0 ? `stderr=${JSON.stringify(outcome.stderr)}` : '',
|
|
926
|
+
outcome.stdoutTruncated ? 'stdout_truncated=true' : '',
|
|
927
|
+
outcome.stderrTruncated ? 'stderr_truncated=true' : '',
|
|
928
|
+
]
|
|
929
|
+
.filter(Boolean)
|
|
930
|
+
.join(' ');
|
|
931
|
+
}
|
|
932
|
+
|
|
933
|
+
/**
|
|
934
|
+
* Create and spawn the complete live execution retained by the process-global
|
|
935
|
+
* owner. The returned object is structural and survives hot-loaded module copies.
|
|
936
|
+
*/
|
|
937
|
+
export function createReloadableShellExecutionV1(
|
|
938
|
+
options: ReloadableShellExecutionOptions,
|
|
939
|
+
): ReloadableShellExecutionV1 {
|
|
940
|
+
const task = options.task;
|
|
941
|
+
const platform = options.platform ?? process.platform;
|
|
942
|
+
const env = options.env ?? process.env;
|
|
943
|
+
const now = options.now ?? Date.now;
|
|
944
|
+
const logger = options.logger ?? console;
|
|
945
|
+
const spawn =
|
|
946
|
+
options.spawn ??
|
|
947
|
+
((command: string, args: string[], spawnOptions: SpawnOptions) =>
|
|
948
|
+
nodeSpawn(command, args, spawnOptions));
|
|
949
|
+
const killProcess = options.killProcess ?? process.kill.bind(process);
|
|
950
|
+
const killTree =
|
|
951
|
+
options.killTree ??
|
|
952
|
+
((pid: number, phase: WindowsKillPhase, signal?: AbortSignal) => {
|
|
953
|
+
const taskkillOptions: WindowsTaskkillOptions =
|
|
954
|
+
signal === undefined ? { env } : { env, signal };
|
|
955
|
+
return runWindowsTaskkill(pid, phase, taskkillOptions);
|
|
956
|
+
});
|
|
957
|
+
|
|
958
|
+
const outputStream = createWriteStream(task.outputAbsPath, { flags: 'a', encoding: 'utf8' });
|
|
959
|
+
let child: ReloadShellProcessV1;
|
|
960
|
+
try {
|
|
961
|
+
child = spawn(options.invocation.shell, [...options.invocation.args], {
|
|
962
|
+
cwd: task.cwd,
|
|
963
|
+
detached: platform !== 'win32',
|
|
964
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
965
|
+
env,
|
|
966
|
+
windowsHide: true,
|
|
967
|
+
windowsVerbatimArguments: options.invocation.windowsVerbatimArguments,
|
|
968
|
+
});
|
|
969
|
+
} catch (error) {
|
|
970
|
+
outputStream.destroy();
|
|
971
|
+
throw error;
|
|
972
|
+
}
|
|
973
|
+
const childPid = child.pid;
|
|
974
|
+
if (childPid === undefined || !Number.isSafeInteger(childPid) || childPid <= 0) {
|
|
975
|
+
outputStream.destroy();
|
|
976
|
+
try {
|
|
977
|
+
child.kill('SIGKILL');
|
|
978
|
+
} catch {
|
|
979
|
+
// The loud launch error remains authoritative; no PID authority was acquired.
|
|
980
|
+
}
|
|
981
|
+
throw ownerError(
|
|
982
|
+
'pi_bg_reload_owner_unavailable',
|
|
983
|
+
'opted reload shell spawn did not provide a positive child pid',
|
|
984
|
+
);
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
task.child = child;
|
|
988
|
+
task.pid = childPid;
|
|
989
|
+
task.stream = outputStream;
|
|
990
|
+
if (platform !== 'win32') task.ownedPosixProcessGroupId = childPid;
|
|
991
|
+
|
|
992
|
+
const spawnedAt = task.startTime;
|
|
993
|
+
const timeoutDeadlineAt =
|
|
994
|
+
task.timeoutSeconds === undefined ? undefined : spawnedAt + task.timeoutSeconds * 1000;
|
|
995
|
+
const audit: ReloadSurvivalSnapshotV1 = {
|
|
996
|
+
schemaVersion: 'pi-background-tasks.reload-shell.v1',
|
|
997
|
+
authority: 'same-process-live-owner',
|
|
998
|
+
hostPid: options.identity.hostPid,
|
|
999
|
+
sessionId: options.identity.sessionId,
|
|
1000
|
+
cwdRealpath: options.identity.cwdRealpath,
|
|
1001
|
+
launchNonce: options.launchNonce,
|
|
1002
|
+
completionId: `${task.id}:1`,
|
|
1003
|
+
spawnedAt,
|
|
1004
|
+
childPid,
|
|
1005
|
+
outputCapBytes: options.maxOutputBytes,
|
|
1006
|
+
leaseGeneration: options.lease.generation,
|
|
1007
|
+
handoffCount: 0,
|
|
1008
|
+
};
|
|
1009
|
+
if (timeoutDeadlineAt !== undefined) audit.timeoutDeadlineAt = timeoutDeadlineAt;
|
|
1010
|
+
if (platform === 'win32') audit.windowsTreeRootPid = childPid;
|
|
1011
|
+
else audit.posixProcessGroupId = childPid;
|
|
1012
|
+
task.reloadSurvival = audit;
|
|
1013
|
+
|
|
1014
|
+
let sink: ReloadShellOwnerEventSinkV1 | undefined;
|
|
1015
|
+
let resolveTerminal: (task: BgTask) => void = () => {};
|
|
1016
|
+
const terminal = new Promise<BgTask>((resolve) => {
|
|
1017
|
+
resolveTerminal = resolve;
|
|
1018
|
+
});
|
|
1019
|
+
let resolveInitialMetadata: () => void = () => {};
|
|
1020
|
+
let rejectInitialMetadata: (error: unknown) => void = () => {};
|
|
1021
|
+
let initialMetadataSettled = false;
|
|
1022
|
+
const initialMetadata = new Promise<void>((resolve, reject) => {
|
|
1023
|
+
resolveInitialMetadata = resolve;
|
|
1024
|
+
rejectInitialMetadata = reject;
|
|
1025
|
+
});
|
|
1026
|
+
void initialMetadata.catch(() => undefined);
|
|
1027
|
+
let finalization: Promise<void> | undefined;
|
|
1028
|
+
let posixTree: PosixTreeState | undefined;
|
|
1029
|
+
let windowsTree: WindowsTreeState | undefined;
|
|
1030
|
+
let stopKind: ReloadShellStopKind | undefined;
|
|
1031
|
+
let notificationToken: string | undefined;
|
|
1032
|
+
|
|
1033
|
+
const writeMetadata = async (value = snapshot(task), signal?: AbortSignal): Promise<void> => {
|
|
1034
|
+
const write = async () => {
|
|
1035
|
+
await writeOwnerMetadata(task.metadataAbsPath, value, signal);
|
|
1036
|
+
};
|
|
1037
|
+
const previous = task.metadataWriteChain ?? Promise.resolve();
|
|
1038
|
+
const next = previous.then(write, write);
|
|
1039
|
+
task.metadataWriteChain = next.catch(() => undefined);
|
|
1040
|
+
await next;
|
|
1041
|
+
};
|
|
1042
|
+
|
|
1043
|
+
const changed = (): void => {
|
|
1044
|
+
sink?.onChanged(execution);
|
|
1045
|
+
};
|
|
1046
|
+
|
|
1047
|
+
const writeBuffer = (buffer: Buffer): void => {
|
|
1048
|
+
const stream = execution.outputStream;
|
|
1049
|
+
if (stream === undefined || stream.destroyed || buffer.length === 0) return;
|
|
1050
|
+
const nextBytes = task.bytesWritten + buffer.length;
|
|
1051
|
+
if (nextBytes <= execution.outputCapBytes) {
|
|
1052
|
+
stream.write(buffer);
|
|
1053
|
+
task.bytesWritten = nextBytes;
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
const remaining = Math.max(0, execution.outputCapBytes - task.bytesWritten);
|
|
1057
|
+
if (remaining > 0) {
|
|
1058
|
+
stream.write(buffer.subarray(0, remaining));
|
|
1059
|
+
task.bytesWritten += remaining;
|
|
1060
|
+
}
|
|
1061
|
+
if (task.capExceeded) return;
|
|
1062
|
+
task.capExceeded = true;
|
|
1063
|
+
task.error = `Output exceeded cap of ${formatSize(execution.outputCapBytes)}; terminating task`;
|
|
1064
|
+
const notice = Buffer.from(`\n\n[background task error: ${task.error}]\n`, 'utf8');
|
|
1065
|
+
stream.write(notice);
|
|
1066
|
+
task.bytesWritten += notice.length;
|
|
1067
|
+
changed();
|
|
1068
|
+
void execution.requestStop('output_cap', task.error).catch((error: unknown) => {
|
|
1069
|
+
task.error = appendError(task.error, `kill failed: ${boundedError(error)}`);
|
|
1070
|
+
});
|
|
1071
|
+
};
|
|
1072
|
+
|
|
1073
|
+
const ingestTelemetry = (text: string): void => {
|
|
1074
|
+
if (text.length === 0) return;
|
|
1075
|
+
const telemetryText = `${task.contextUsageBuffer ?? ''}${text}`;
|
|
1076
|
+
let context = task.contextUsage;
|
|
1077
|
+
let tokens = task.tokenUsage;
|
|
1078
|
+
let tools = task.toolUsage;
|
|
1079
|
+
let model = task.model;
|
|
1080
|
+
for (const line of telemetryText.split(/\r?\n/u)) {
|
|
1081
|
+
if (!line.includes('background-task-')) continue;
|
|
1082
|
+
const trimmed = line.trim();
|
|
1083
|
+
if (trimmed.startsWith('{') && trimmed.endsWith('}')) {
|
|
1084
|
+
try {
|
|
1085
|
+
const parsed = parseJsonText(trimmed);
|
|
1086
|
+
if (!isJsonObject(parsed)) continue;
|
|
1087
|
+
if (parsed['type'] === 'background-task-context-usage') {
|
|
1088
|
+
context = normalizeContextUsage(parsed) ?? context;
|
|
1089
|
+
} else if (parsed['type'] === 'background-task-telemetry') {
|
|
1090
|
+
context = normalizeContextUsage(parsed['contextUsage']) ?? context;
|
|
1091
|
+
tokens = normalizeTokenUsage(parsed['tokenUsage']) ?? tokens;
|
|
1092
|
+
tools = normalizeToolUsage(parsed['toolUsage']) ?? tools;
|
|
1093
|
+
model = normalizeModel(parsed['model']) ?? model;
|
|
1094
|
+
}
|
|
1095
|
+
} catch {
|
|
1096
|
+
// Optional telemetry never replaces raw output truth.
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
for (const match of telemetryText.matchAll(
|
|
1101
|
+
/<background-task-context-usage>[\s\S]*?<\/background-task-context-usage>/giu,
|
|
1102
|
+
)) {
|
|
1103
|
+
context = parseContextUsageXml(match[0]) ?? context;
|
|
1104
|
+
}
|
|
1105
|
+
const lastNewline = Math.max(telemetryText.lastIndexOf('\n'), telemetryText.lastIndexOf('\r'));
|
|
1106
|
+
let retained = lastNewline >= 0 ? telemetryText.slice(lastNewline + 1) : telemetryText;
|
|
1107
|
+
const lower = telemetryText.toLowerCase();
|
|
1108
|
+
const open = lower.lastIndexOf('<background-task-context-usage');
|
|
1109
|
+
const close = lower.lastIndexOf('</background-task-context-usage>');
|
|
1110
|
+
if (open > close) retained = telemetryText.slice(open);
|
|
1111
|
+
task.contextUsageBuffer = retained.slice(-TELEMETRY_BUFFER_CHARS);
|
|
1112
|
+
const before = JSON.stringify({
|
|
1113
|
+
contextUsage: task.contextUsage,
|
|
1114
|
+
tokenUsage: task.tokenUsage,
|
|
1115
|
+
toolUsage: task.toolUsage,
|
|
1116
|
+
model: task.model,
|
|
1117
|
+
});
|
|
1118
|
+
task.contextUsage = context;
|
|
1119
|
+
task.tokenUsage = tokens;
|
|
1120
|
+
task.toolUsage = tools;
|
|
1121
|
+
task.model = model;
|
|
1122
|
+
const after = JSON.stringify({ contextUsage: context, tokenUsage: tokens, toolUsage: tools, model });
|
|
1123
|
+
if (before !== after) {
|
|
1124
|
+
changed();
|
|
1125
|
+
void writeMetadata().catch((error: unknown) => {
|
|
1126
|
+
logger.error(`[background-tasks] failed to write survivor telemetry for ${task.id}:`, error);
|
|
1127
|
+
});
|
|
1128
|
+
}
|
|
1129
|
+
};
|
|
1130
|
+
|
|
1131
|
+
const finishPosix = (state: PosixTreeState, failure?: Error): void => {
|
|
1132
|
+
if (state.settled) return;
|
|
1133
|
+
state.settled = true;
|
|
1134
|
+
state.failure = failure;
|
|
1135
|
+
if (state.verificationTimer !== undefined) clearTimeout(state.verificationTimer);
|
|
1136
|
+
if (task.killEscalationTimer !== undefined) clearTimeout(task.killEscalationTimer);
|
|
1137
|
+
delete task.killEscalationTimer;
|
|
1138
|
+
task.posixProcessGroupSignalAuthorityReleased = true;
|
|
1139
|
+
if (failure === undefined && task.ownedPosixProcessGroupId === state.groupId) {
|
|
1140
|
+
delete task.ownedPosixProcessGroupId;
|
|
1141
|
+
}
|
|
1142
|
+
state.resolve();
|
|
1143
|
+
};
|
|
1144
|
+
|
|
1145
|
+
const probePosixGone = (state: PosixTreeState): boolean => {
|
|
1146
|
+
if (state.settled) return state.failure === undefined;
|
|
1147
|
+
try {
|
|
1148
|
+
const present = killProcess(-state.groupId, 0);
|
|
1149
|
+
state.lastProbeError = present
|
|
1150
|
+
? undefined
|
|
1151
|
+
: new Error(`process-group probe for ${String(state.groupId)} returned false`);
|
|
1152
|
+
return false;
|
|
1153
|
+
} catch (error) {
|
|
1154
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
1155
|
+
finishPosix(state);
|
|
1156
|
+
return true;
|
|
1157
|
+
}
|
|
1158
|
+
state.lastProbeError = error instanceof Error ? error : new Error(String(error));
|
|
1159
|
+
return false;
|
|
1160
|
+
}
|
|
1161
|
+
};
|
|
1162
|
+
|
|
1163
|
+
const recordPosixFailure = (state: PosixTreeState, error: Error): void => {
|
|
1164
|
+
task.error = appendError(task.error, error.message);
|
|
1165
|
+
writeBuffer(Buffer.from(`\n[background task POSIX termination: ${error.message}]\n`, 'utf8'));
|
|
1166
|
+
finishPosix(state, error);
|
|
1167
|
+
changed();
|
|
1168
|
+
};
|
|
1169
|
+
|
|
1170
|
+
const verifyPosix = (state: PosixTreeState): void => {
|
|
1171
|
+
if (state.settled) return;
|
|
1172
|
+
const remaining = state.deadlineAt - now();
|
|
1173
|
+
if (remaining <= 0) {
|
|
1174
|
+
const detail = state.lastProbeError ? `; last group probe failed: ${state.lastProbeError.message}` : '';
|
|
1175
|
+
recordPosixFailure(
|
|
1176
|
+
state,
|
|
1177
|
+
new Error(
|
|
1178
|
+
`POSIX process group ${String(state.groupId)} remained present after SIGKILL${detail}. Descendant processes may have leaked.`,
|
|
1179
|
+
),
|
|
1180
|
+
);
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
state.verificationTimer = setTimeout(() => {
|
|
1184
|
+
state.verificationTimer = undefined;
|
|
1185
|
+
if (!probePosixGone(state)) verifyPosix(state);
|
|
1186
|
+
}, Math.min(10, remaining));
|
|
1187
|
+
};
|
|
1188
|
+
|
|
1189
|
+
const forcePosix = (state: PosixTreeState): void => {
|
|
1190
|
+
if (state.settled || state.forceAttempted) return;
|
|
1191
|
+
state.forceAttempted = true;
|
|
1192
|
+
if (task.killEscalationTimer !== undefined) clearTimeout(task.killEscalationTimer);
|
|
1193
|
+
delete task.killEscalationTimer;
|
|
1194
|
+
if (probePosixGone(state)) return;
|
|
1195
|
+
try {
|
|
1196
|
+
if (!killProcess(-state.groupId, 'SIGKILL')) {
|
|
1197
|
+
recordPosixFailure(
|
|
1198
|
+
state,
|
|
1199
|
+
new Error(
|
|
1200
|
+
`POSIX process-group SIGKILL returned false for task ${task.id} group ${String(state.groupId)}. Descendant processes may have leaked.`,
|
|
1201
|
+
),
|
|
1202
|
+
);
|
|
1203
|
+
return;
|
|
1204
|
+
}
|
|
1205
|
+
} catch (error) {
|
|
1206
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
1207
|
+
finishPosix(state);
|
|
1208
|
+
return;
|
|
1209
|
+
}
|
|
1210
|
+
recordPosixFailure(
|
|
1211
|
+
state,
|
|
1212
|
+
new Error(
|
|
1213
|
+
`POSIX process-group SIGKILL failed for task ${task.id} group ${String(state.groupId)}: ${boundedError(error)}. Descendant processes may have leaked.`,
|
|
1214
|
+
),
|
|
1215
|
+
);
|
|
1216
|
+
return;
|
|
1217
|
+
}
|
|
1218
|
+
if (!probePosixGone(state)) verifyPosix(state);
|
|
1219
|
+
};
|
|
1220
|
+
|
|
1221
|
+
const beginPosixStop = (): PosixTreeState => {
|
|
1222
|
+
if (posixTree !== undefined) return posixTree;
|
|
1223
|
+
if (task.posixProcessGroupSignalAuthorityReleased === true) {
|
|
1224
|
+
throw new Error(`Task ${task.id} has released its POSIX process-group signal authority`);
|
|
1225
|
+
}
|
|
1226
|
+
const groupId = task.ownedPosixProcessGroupId;
|
|
1227
|
+
if (groupId === undefined) throw new Error(`Task ${task.id} has no owned POSIX process group`);
|
|
1228
|
+
let resolve: () => void = () => {};
|
|
1229
|
+
const completion = new Promise<void>((resolvePromise) => {
|
|
1230
|
+
resolve = resolvePromise;
|
|
1231
|
+
});
|
|
1232
|
+
const reserve = Math.min(25, Math.max(1, Math.floor(options.stopWaitMs / 4)));
|
|
1233
|
+
const state: PosixTreeState = {
|
|
1234
|
+
groupId,
|
|
1235
|
+
deadlineAt: now() + Math.max(1, options.stopWaitMs - reserve),
|
|
1236
|
+
completion,
|
|
1237
|
+
resolve,
|
|
1238
|
+
forceAttempted: false,
|
|
1239
|
+
settled: false,
|
|
1240
|
+
};
|
|
1241
|
+
posixTree = state;
|
|
1242
|
+
task.killEscalationTimer = setTimeout(() => {
|
|
1243
|
+
delete task.killEscalationTimer;
|
|
1244
|
+
forcePosix(state);
|
|
1245
|
+
}, Math.min(options.killGraceMs, Math.max(1, options.stopWaitMs - reserve)));
|
|
1246
|
+
return state;
|
|
1247
|
+
};
|
|
1248
|
+
|
|
1249
|
+
const requestPosixStop = (): void => {
|
|
1250
|
+
const state = beginPosixStop();
|
|
1251
|
+
if (task.killSignalSent) return;
|
|
1252
|
+
task.killSignalSent = true;
|
|
1253
|
+
const failures: string[] = [];
|
|
1254
|
+
let sent = false;
|
|
1255
|
+
try {
|
|
1256
|
+
sent = killProcess(-state.groupId, 'SIGTERM');
|
|
1257
|
+
if (!sent) failures.push('process-group SIGTERM returned false');
|
|
1258
|
+
} catch (error) {
|
|
1259
|
+
if (typeof error === 'object' && error !== null && Reflect.get(error, 'code') === 'ESRCH') {
|
|
1260
|
+
finishPosix(state);
|
|
1261
|
+
} else {
|
|
1262
|
+
failures.push(`process-group SIGTERM failed: ${boundedError(error)}`);
|
|
1263
|
+
}
|
|
1264
|
+
}
|
|
1265
|
+
if (!sent && !state.settled) {
|
|
1266
|
+
try {
|
|
1267
|
+
sent = execution.child?.kill('SIGTERM') === true;
|
|
1268
|
+
if (!sent) failures.push('child SIGTERM returned false');
|
|
1269
|
+
} catch (error) {
|
|
1270
|
+
failures.push(`child SIGTERM failed: ${boundedError(error)}`);
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
if (!sent && !state.settled) {
|
|
1274
|
+
forcePosix(state);
|
|
1275
|
+
throw new Error(`Could not kill task ${task.id}: ${failures.join('; ')}`);
|
|
1276
|
+
}
|
|
1277
|
+
};
|
|
1278
|
+
|
|
1279
|
+
const finishWindows = (state: WindowsTreeState, failure?: Error): void => {
|
|
1280
|
+
if (state.settled) return;
|
|
1281
|
+
state.settled = true;
|
|
1282
|
+
state.failure = failure;
|
|
1283
|
+
if (task.killEscalationTimer !== undefined) clearTimeout(task.killEscalationTimer);
|
|
1284
|
+
delete task.killEscalationTimer;
|
|
1285
|
+
state.resolve();
|
|
1286
|
+
};
|
|
1287
|
+
|
|
1288
|
+
const evaluateTaskkill = (
|
|
1289
|
+
state: WindowsTreeState,
|
|
1290
|
+
phase: WindowsKillPhase,
|
|
1291
|
+
outcome: TaskkillOutcome,
|
|
1292
|
+
): Error | undefined => {
|
|
1293
|
+
if (outcome.exitCode === 0 || outcome.exitCode === 128) return undefined;
|
|
1294
|
+
if (phase === 'terminate') return undefined;
|
|
1295
|
+
return new Error(
|
|
1296
|
+
`Windows taskkill /T /F force termination failed for task ${task.id} pid ${String(state.pid)}: ${taskkillDescription(outcome)}. Descendant processes may have leaked.`,
|
|
1297
|
+
);
|
|
1298
|
+
};
|
|
1299
|
+
|
|
1300
|
+
const forceWindows = (state: WindowsTreeState): Promise<void> => {
|
|
1301
|
+
if (state.forcePromise !== undefined) return state.forcePromise;
|
|
1302
|
+
state.forceAttempted = true;
|
|
1303
|
+
state.softController?.abort();
|
|
1304
|
+
if (task.killEscalationTimer !== undefined) clearTimeout(task.killEscalationTimer);
|
|
1305
|
+
delete task.killEscalationTimer;
|
|
1306
|
+
const launched = Promise.resolve().then(() => killTree(state.pid, 'force'));
|
|
1307
|
+
state.forcePromise = launched.then(
|
|
1308
|
+
(outcome) => {
|
|
1309
|
+
const failure = evaluateTaskkill(state, 'force', outcome);
|
|
1310
|
+
if (failure !== undefined) {
|
|
1311
|
+
task.error = appendError(task.error, failure.message);
|
|
1312
|
+
finishWindows(state, failure);
|
|
1313
|
+
throw failure;
|
|
1314
|
+
}
|
|
1315
|
+
finishWindows(state);
|
|
1316
|
+
},
|
|
1317
|
+
(error: unknown) => {
|
|
1318
|
+
const failure = new Error(
|
|
1319
|
+
`Windows taskkill /T /F force termination failed for task ${task.id} pid ${String(state.pid)}: ${boundedError(error)}. Descendant processes may have leaked.`,
|
|
1320
|
+
);
|
|
1321
|
+
task.error = appendError(task.error, failure.message);
|
|
1322
|
+
finishWindows(state, failure);
|
|
1323
|
+
throw failure;
|
|
1324
|
+
},
|
|
1325
|
+
);
|
|
1326
|
+
void state.forcePromise.catch((error: unknown) => {
|
|
1327
|
+
logger.error(`[background-tasks] Windows survivor force kill failed for ${task.id}:`, error);
|
|
1328
|
+
});
|
|
1329
|
+
return state.forcePromise;
|
|
1330
|
+
};
|
|
1331
|
+
|
|
1332
|
+
const beginWindowsStop = (): WindowsTreeState => {
|
|
1333
|
+
if (windowsTree !== undefined) return windowsTree;
|
|
1334
|
+
let resolve: () => void = () => {};
|
|
1335
|
+
const completion = new Promise<void>((resolvePromise) => {
|
|
1336
|
+
resolve = resolvePromise;
|
|
1337
|
+
});
|
|
1338
|
+
const state: WindowsTreeState = {
|
|
1339
|
+
pid: childPid,
|
|
1340
|
+
completion,
|
|
1341
|
+
resolve,
|
|
1342
|
+
forceAttempted: false,
|
|
1343
|
+
settled: false,
|
|
1344
|
+
};
|
|
1345
|
+
windowsTree = state;
|
|
1346
|
+
const controller = new AbortController();
|
|
1347
|
+
state.softController = controller;
|
|
1348
|
+
state.softPromise = Promise.resolve()
|
|
1349
|
+
.then(() => killTree(childPid, 'terminate', controller.signal))
|
|
1350
|
+
.then(
|
|
1351
|
+
(outcome) => {
|
|
1352
|
+
if (state.forceAttempted || state.settled) return;
|
|
1353
|
+
if (outcome.exitCode === 0 || outcome.exitCode === 128) finishWindows(state);
|
|
1354
|
+
// Other soft failures deliberately retain force escalation.
|
|
1355
|
+
},
|
|
1356
|
+
() => {
|
|
1357
|
+
// Soft failure deliberately retains force escalation.
|
|
1358
|
+
},
|
|
1359
|
+
);
|
|
1360
|
+
task.killEscalationTimer = setTimeout(() => {
|
|
1361
|
+
delete task.killEscalationTimer;
|
|
1362
|
+
void forceWindows(state);
|
|
1363
|
+
}, options.killGraceMs);
|
|
1364
|
+
return state;
|
|
1365
|
+
};
|
|
1366
|
+
|
|
1367
|
+
const requestWindowsStop = (): void => {
|
|
1368
|
+
beginWindowsStop();
|
|
1369
|
+
task.killSignalSent = true;
|
|
1370
|
+
};
|
|
1371
|
+
|
|
1372
|
+
const awaitTreeBeforeTerminal = async (): Promise<Error | undefined> => {
|
|
1373
|
+
if (platform === 'win32') {
|
|
1374
|
+
if (windowsTree === undefined) return undefined;
|
|
1375
|
+
await windowsTree.completion;
|
|
1376
|
+
return windowsTree.failure;
|
|
1377
|
+
}
|
|
1378
|
+
if (posixTree === undefined) {
|
|
1379
|
+
task.posixProcessGroupSignalAuthorityReleased = true;
|
|
1380
|
+
delete task.ownedPosixProcessGroupId;
|
|
1381
|
+
return undefined;
|
|
1382
|
+
}
|
|
1383
|
+
probePosixGone(posixTree);
|
|
1384
|
+
await posixTree.completion;
|
|
1385
|
+
return posixTree.failure;
|
|
1386
|
+
};
|
|
1387
|
+
|
|
1388
|
+
const terminalStatus = (
|
|
1389
|
+
code: number | null,
|
|
1390
|
+
signal: NodeJS.Signals | null,
|
|
1391
|
+
): { status: TaskStatus; error?: string | undefined } => {
|
|
1392
|
+
if (stopKind === 'user' || stopKind === 'shutdown') return { status: 'killed' };
|
|
1393
|
+
if (stopKind === 'timeout') {
|
|
1394
|
+
return { status: 'failed', error: task.error ?? `Timed out after ${String(task.timeoutSeconds)}s` };
|
|
1395
|
+
}
|
|
1396
|
+
if (stopKind === 'output_cap') {
|
|
1397
|
+
return { status: 'failed', error: task.error ?? `Output exceeded cap of ${formatSize(execution.outputCapBytes)}` };
|
|
1398
|
+
}
|
|
1399
|
+
if (stopKind === 'handoff_expired') {
|
|
1400
|
+
return {
|
|
1401
|
+
status: 'failed',
|
|
1402
|
+
error:
|
|
1403
|
+
task.error ??
|
|
1404
|
+
'pi_bg_reload_handoff_expired: reload shell execution was not claimed before its handoff deadline',
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
if ((code ?? 0) === 0) return { status: 'completed' };
|
|
1408
|
+
return {
|
|
1409
|
+
status: 'failed',
|
|
1410
|
+
error: `Exited with code ${code === null ? 'null' : String(code)}${signal ? ` (${signal})` : ''}`,
|
|
1411
|
+
};
|
|
1412
|
+
};
|
|
1413
|
+
|
|
1414
|
+
const finalize = (code: number | null, signal: NodeJS.Signals | null): void => {
|
|
1415
|
+
if (finalization !== undefined) return;
|
|
1416
|
+
execution.closeObservation = { code, signal, observedAt: now() };
|
|
1417
|
+
execution.phase = 'finalizing';
|
|
1418
|
+
finalization = (async () => {
|
|
1419
|
+
let result = terminalStatus(code, signal);
|
|
1420
|
+
try {
|
|
1421
|
+
await initialMetadata;
|
|
1422
|
+
} catch (error) {
|
|
1423
|
+
result = {
|
|
1424
|
+
status: 'failed',
|
|
1425
|
+
error: appendError(result.error, `Initial metadata write failed: ${boundedError(error)}`),
|
|
1426
|
+
};
|
|
1427
|
+
}
|
|
1428
|
+
const treeFailure = await awaitTreeBeforeTerminal();
|
|
1429
|
+
if (treeFailure !== undefined) {
|
|
1430
|
+
result = { status: 'failed', error: appendError(result.error, treeFailure.message) };
|
|
1431
|
+
}
|
|
1432
|
+
try {
|
|
1433
|
+
await closeOwnerOutputStream(execution.outputStream);
|
|
1434
|
+
} catch (error) {
|
|
1435
|
+
result = {
|
|
1436
|
+
status: 'failed',
|
|
1437
|
+
error: appendError(result.error, `Final output durability failed: ${boundedError(error)}`),
|
|
1438
|
+
};
|
|
1439
|
+
}
|
|
1440
|
+
task.exitCode = code;
|
|
1441
|
+
task.signal = signal;
|
|
1442
|
+
task.endTime = now();
|
|
1443
|
+
if (result.error !== undefined) task.error = result.error;
|
|
1444
|
+
try {
|
|
1445
|
+
await writeMetadata({ ...snapshot(task), status: result.status });
|
|
1446
|
+
task.status = result.status;
|
|
1447
|
+
} catch (error) {
|
|
1448
|
+
task.status = 'failed';
|
|
1449
|
+
task.error = `Terminal metadata write failed: ${boundedError(error)}`;
|
|
1450
|
+
logger.error(`[background-tasks] failed to write survivor metadata for ${task.id}:`, error);
|
|
1451
|
+
await writeMetadata().catch(() => undefined);
|
|
1452
|
+
}
|
|
1453
|
+
task.finalized = true;
|
|
1454
|
+
if (task.timeoutHandle !== undefined) clearTimeout(task.timeoutHandle);
|
|
1455
|
+
delete task.timeoutHandle;
|
|
1456
|
+
for (const waiter of task.waiters.splice(0)) waiter();
|
|
1457
|
+
execution.phase = 'terminal';
|
|
1458
|
+
resolveTerminal(task);
|
|
1459
|
+
changed();
|
|
1460
|
+
sink?.onTerminal(execution);
|
|
1461
|
+
})().catch((error: unknown) => {
|
|
1462
|
+
logger.error(`[background-tasks] reload shell finalization failed for ${task.id}:`, error);
|
|
1463
|
+
});
|
|
1464
|
+
};
|
|
1465
|
+
|
|
1466
|
+
let stdoutListener: (data: Buffer | string) => void = () => {};
|
|
1467
|
+
let stderrListener: (data: Buffer | string) => void = () => {};
|
|
1468
|
+
let childErrorListener: (error: Error) => void = () => {};
|
|
1469
|
+
let childCloseListener: (
|
|
1470
|
+
code: number | null,
|
|
1471
|
+
signal: NodeJS.Signals | null,
|
|
1472
|
+
) => void = () => {};
|
|
1473
|
+
let streamErrorListener: (error: Error) => void = () => {};
|
|
1474
|
+
|
|
1475
|
+
const execution: ReloadableShellExecutionV1 = {
|
|
1476
|
+
protocol: RELOAD_SHELL_OWNER_PROTOCOL,
|
|
1477
|
+
launchNonce: options.launchNonce,
|
|
1478
|
+
completionId: `${task.id}:1`,
|
|
1479
|
+
task,
|
|
1480
|
+
child,
|
|
1481
|
+
outputStream,
|
|
1482
|
+
spawnedAt,
|
|
1483
|
+
timeoutDeadlineAt,
|
|
1484
|
+
outputCapBytes: options.maxOutputBytes,
|
|
1485
|
+
terminal,
|
|
1486
|
+
phase: 'starting',
|
|
1487
|
+
admissionCommitted: false,
|
|
1488
|
+
notificationState: task.notifyOnCompletion ? 'pending' : 'disabled',
|
|
1489
|
+
async requestStop(kind, reason) {
|
|
1490
|
+
if (execution.phase === 'released') return task;
|
|
1491
|
+
if (execution.phase === 'terminal') return task;
|
|
1492
|
+
const firstStop = stopKind === undefined;
|
|
1493
|
+
if (firstStop) {
|
|
1494
|
+
stopKind = kind;
|
|
1495
|
+
task.killKind = kind === 'handoff_expired' ? 'shutdown' : kind;
|
|
1496
|
+
if (reason !== undefined) task.error = reason;
|
|
1497
|
+
}
|
|
1498
|
+
if (firstStop && kind === 'handoff_expired' && !task.error?.startsWith('pi_bg_reload_handoff_expired')) {
|
|
1499
|
+
task.error = `pi_bg_reload_handoff_expired: ${task.error ?? 'reload handoff expired'}`;
|
|
1500
|
+
}
|
|
1501
|
+
if (execution.phase === 'finalizing') return terminal;
|
|
1502
|
+
if (execution.phase === 'running' || execution.phase === 'starting') {
|
|
1503
|
+
execution.phase = 'stop_requested';
|
|
1504
|
+
}
|
|
1505
|
+
if (task.status === 'running') {
|
|
1506
|
+
if (platform === 'win32') requestWindowsStop();
|
|
1507
|
+
else requestPosixStop();
|
|
1508
|
+
}
|
|
1509
|
+
let timeout: NodeJS.Timeout | undefined;
|
|
1510
|
+
try {
|
|
1511
|
+
return await Promise.race([
|
|
1512
|
+
terminal,
|
|
1513
|
+
new Promise<never>((_resolve, reject) => {
|
|
1514
|
+
timeout = setTimeout(
|
|
1515
|
+
() => reject(new Error(`Task ${task.id} did not exit within ${String(options.stopWaitMs)}ms after cancellation`)),
|
|
1516
|
+
options.stopWaitMs,
|
|
1517
|
+
);
|
|
1518
|
+
}),
|
|
1519
|
+
]);
|
|
1520
|
+
} finally {
|
|
1521
|
+
if (timeout !== undefined) clearTimeout(timeout);
|
|
1522
|
+
}
|
|
1523
|
+
},
|
|
1524
|
+
async commitInitialMetadata(signal) {
|
|
1525
|
+
try {
|
|
1526
|
+
await writeMetadata(snapshot(task), signal);
|
|
1527
|
+
if (!initialMetadataSettled) {
|
|
1528
|
+
initialMetadataSettled = true;
|
|
1529
|
+
resolveInitialMetadata();
|
|
1530
|
+
}
|
|
1531
|
+
} catch (error) {
|
|
1532
|
+
if (!initialMetadataSettled) {
|
|
1533
|
+
initialMetadataSettled = true;
|
|
1534
|
+
rejectInitialMetadata(error);
|
|
1535
|
+
}
|
|
1536
|
+
throw error;
|
|
1537
|
+
}
|
|
1538
|
+
},
|
|
1539
|
+
failAdmission(error) {
|
|
1540
|
+
if (!initialMetadataSettled) {
|
|
1541
|
+
initialMetadataSettled = true;
|
|
1542
|
+
rejectInitialMetadata(error);
|
|
1543
|
+
}
|
|
1544
|
+
},
|
|
1545
|
+
setOwnerEventSink(next) {
|
|
1546
|
+
sink = next;
|
|
1547
|
+
},
|
|
1548
|
+
markAdmissionCommitted(generation, handoffCount) {
|
|
1549
|
+
execution.admissionCommitted = true;
|
|
1550
|
+
execution.phase = execution.phase === 'starting' ? 'running' : execution.phase;
|
|
1551
|
+
audit.leaseGeneration = generation;
|
|
1552
|
+
audit.handoffCount = handoffCount;
|
|
1553
|
+
},
|
|
1554
|
+
updateLeaseAudit(generation, handoffCount) {
|
|
1555
|
+
audit.leaseGeneration = generation;
|
|
1556
|
+
audit.handoffCount = handoffCount;
|
|
1557
|
+
},
|
|
1558
|
+
abandonReloadHandoff() {
|
|
1559
|
+
if (task.terminalPublicationState === 'pending') {
|
|
1560
|
+
task.terminalPublicationState = 'abandoned';
|
|
1561
|
+
task.terminalPublicationAbandonReason = 'reload_handoff_expired';
|
|
1562
|
+
task.terminalPublished = false;
|
|
1563
|
+
}
|
|
1564
|
+
if (task.status === 'running') {
|
|
1565
|
+
task.error = appendError(
|
|
1566
|
+
task.error,
|
|
1567
|
+
'pi_bg_reload_handoff_expired: no compatible reload activation claimed this execution',
|
|
1568
|
+
);
|
|
1569
|
+
}
|
|
1570
|
+
void writeMetadata().catch((error: unknown) => {
|
|
1571
|
+
logger.error(`[background-tasks] failed to persist handoff expiry for ${task.id}:`, error);
|
|
1572
|
+
});
|
|
1573
|
+
},
|
|
1574
|
+
beginNotification(lease) {
|
|
1575
|
+
if (execution.notificationState !== 'pending') return undefined;
|
|
1576
|
+
const token = `${String(lease.generation)}:${lease.activationNonce}:${randomNonce()}`;
|
|
1577
|
+
notificationToken = token;
|
|
1578
|
+
execution.notificationState = 'sending';
|
|
1579
|
+
return token;
|
|
1580
|
+
},
|
|
1581
|
+
finishNotification(token, delivered) {
|
|
1582
|
+
if (notificationToken !== token || execution.notificationState !== 'sending') return;
|
|
1583
|
+
notificationToken = undefined;
|
|
1584
|
+
if (delivered) {
|
|
1585
|
+
execution.notificationState = 'delivered';
|
|
1586
|
+
task.notified = true;
|
|
1587
|
+
} else {
|
|
1588
|
+
execution.notificationState = 'pending';
|
|
1589
|
+
task.notified = false;
|
|
1590
|
+
}
|
|
1591
|
+
},
|
|
1592
|
+
releaseResources() {
|
|
1593
|
+
if (task.timeoutHandle !== undefined) clearTimeout(task.timeoutHandle);
|
|
1594
|
+
if (task.killEscalationTimer !== undefined) clearTimeout(task.killEscalationTimer);
|
|
1595
|
+
if (posixTree?.verificationTimer !== undefined) clearTimeout(posixTree.verificationTimer);
|
|
1596
|
+
windowsTree?.softController?.abort();
|
|
1597
|
+
execution.child?.stdout?.off?.('data', stdoutListener);
|
|
1598
|
+
execution.child?.stderr?.off?.('data', stderrListener);
|
|
1599
|
+
execution.child?.off?.('error', childErrorListener);
|
|
1600
|
+
execution.child?.off?.('close', childCloseListener);
|
|
1601
|
+
execution.outputStream?.off('error', streamErrorListener);
|
|
1602
|
+
sink = undefined;
|
|
1603
|
+
execution.child = undefined;
|
|
1604
|
+
execution.outputStream = undefined;
|
|
1605
|
+
delete task.child;
|
|
1606
|
+
delete task.stream;
|
|
1607
|
+
delete task.timeoutHandle;
|
|
1608
|
+
delete task.killEscalationTimer;
|
|
1609
|
+
delete task.reloadExecution;
|
|
1610
|
+
execution.phase = 'released';
|
|
1611
|
+
},
|
|
1612
|
+
};
|
|
1613
|
+
|
|
1614
|
+
task.reloadExecution = execution;
|
|
1615
|
+
|
|
1616
|
+
streamErrorListener = (error) => {
|
|
1617
|
+
task.error = `Output file write failed: ${error.message}`;
|
|
1618
|
+
changed();
|
|
1619
|
+
if (task.status === 'running') {
|
|
1620
|
+
void execution.requestStop('output_cap', task.error).catch((stopError: unknown) => {
|
|
1621
|
+
logger.error(`[background-tasks] failed to stop survivor after stream error ${task.id}:`, stopError);
|
|
1622
|
+
});
|
|
1623
|
+
}
|
|
1624
|
+
};
|
|
1625
|
+
stdoutListener = (data) => {
|
|
1626
|
+
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf8');
|
|
1627
|
+
ingestTelemetry(buffer.toString('utf8'));
|
|
1628
|
+
writeBuffer(buffer);
|
|
1629
|
+
};
|
|
1630
|
+
stderrListener = (data) => {
|
|
1631
|
+
const buffer = Buffer.isBuffer(data) ? data : Buffer.from(data, 'utf8');
|
|
1632
|
+
ingestTelemetry(buffer.toString('utf8'));
|
|
1633
|
+
writeBuffer(buffer);
|
|
1634
|
+
};
|
|
1635
|
+
childErrorListener = (error) => {
|
|
1636
|
+
task.error = appendError(task.error, `Background task spawn error: ${error.message}`);
|
|
1637
|
+
writeBuffer(Buffer.from(`\n[background task spawn error: ${error.message}]\n`, 'utf8'));
|
|
1638
|
+
changed();
|
|
1639
|
+
};
|
|
1640
|
+
childCloseListener = (code, signal) => {
|
|
1641
|
+
if (execution.closeObservation !== undefined) return;
|
|
1642
|
+
finalize(code, signal);
|
|
1643
|
+
};
|
|
1644
|
+
outputStream.on('error', streamErrorListener);
|
|
1645
|
+
child.stdout?.on('data', stdoutListener);
|
|
1646
|
+
child.stderr?.on('data', stderrListener);
|
|
1647
|
+
child.on('error', childErrorListener);
|
|
1648
|
+
child.on('close', childCloseListener);
|
|
1649
|
+
|
|
1650
|
+
if (timeoutDeadlineAt !== undefined) {
|
|
1651
|
+
task.timeoutHandle = setTimeout(() => {
|
|
1652
|
+
if (task.status !== 'running' || execution.phase === 'terminal' || execution.phase === 'released') return;
|
|
1653
|
+
const message = `Timed out after ${String(task.timeoutSeconds)}s`;
|
|
1654
|
+
writeBuffer(Buffer.from(`\n[background task timeout: ${message}]\n`, 'utf8'));
|
|
1655
|
+
void execution.requestStop('timeout', message).catch((error: unknown) => {
|
|
1656
|
+
logger.error(`[background-tasks] survivor timeout stop failed for ${task.id}:`, error);
|
|
1657
|
+
});
|
|
1658
|
+
}, Math.max(0, timeoutDeadlineAt - now()));
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
return execution;
|
|
1662
|
+
}
|