pi-better-subagents 0.26.2 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -6
- package/child-operations.ts +1109 -0
- package/child-ownership.ts +12 -1
- package/cleanup.ts +11 -2
- package/docs/usage.md +44 -3
- package/finalization.ts +53 -8
- package/goal-provider.ts +42 -4
- package/index.ts +549 -14
- package/lifecycle.ts +26 -0
- package/navigator.mjs +13 -2
- package/navigator.ts +1 -0
- package/operation-bridge-types.ts +1109 -0
- package/package.json +2 -1
- package/park-checkpoint.ts +61 -0
- package/parse.ts +19 -1
- package/registry.ts +192 -6
- package/session-store.ts +3 -0
- package/shared-execution-policy.ts +2 -0
- package/shared-operation-coordinator/index.ts +113 -0
- package/shared-operation-coordinator/src/errors.ts +75 -0
- package/shared-operation-coordinator/src/operations.ts +376 -0
- package/shared-operation-coordinator/src/outbox.ts +431 -0
- package/shared-operation-coordinator/src/rows.ts +439 -0
- package/shared-operation-coordinator/src/schema.ts +244 -0
- package/shared-operation-coordinator/src/sqlite.ts +239 -0
- package/shared-operation-coordinator/src/store.ts +397 -0
- package/shared-operation-coordinator/src/types.ts +648 -0
- package/shared-operation-coordinator/src/validate.ts +239 -0
- package/shared-operation-coordinator/src/waits.ts +1378 -0
- package/shared-task-exec.ts +9 -0
- package/stop.ts +7 -3
- package/task-guard.ts +22 -3
- package/task-policy.ts +7 -0
- package/worker-operations-host.ts +1226 -0
- package/worker-waits.ts +2549 -0
package/worker-waits.ts
ADDED
|
@@ -0,0 +1,2549 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Autonomous worker park/resume for Subagents.
|
|
3
|
+
*
|
|
4
|
+
* A supervised worker conversation may launch background operations, register an
|
|
5
|
+
* all-settled wait on them, and park: its attempt completes cleanly with a
|
|
6
|
+
* durable park disposition while the assignment keeps its native conversation,
|
|
7
|
+
* launch recipe and saved history. When the dependency settles, the Background
|
|
8
|
+
* host delivers the ready wait to this module's continuation adapter, which
|
|
9
|
+
* admits exactly one continuation through the ordinary continuation launch path.
|
|
10
|
+
*
|
|
11
|
+
* Ownership split (ADR 0020, observed-waits design):
|
|
12
|
+
*
|
|
13
|
+
* - **Subagents (this module)** owns the trusted bridge host each worker child
|
|
14
|
+
* talks to, the per-run transport token, the child extension arguments,
|
|
15
|
+
* native-checkpoint validation against the trusted launch recipe, the park
|
|
16
|
+
* disposition on the run record, and resume admission/launch.
|
|
17
|
+
* - **Background Tasks** owns the durable operation/wait/outbox records, the
|
|
18
|
+
* supervisor transport, permission intersection and dispatch. This module never
|
|
19
|
+
* opens that database and never imports Background code: it consumes the
|
|
20
|
+
* service object Background publishes on its versioned event channel.
|
|
21
|
+
* - **Goal** answers `pi-better-goal:operation-resume-admission` (v1). A refused
|
|
22
|
+
* admission holds the wait; it never falls back to launching.
|
|
23
|
+
*
|
|
24
|
+
* The service contract mirror below is intentionally structural: it names only
|
|
25
|
+
* the members this module calls, so either package can load first and a missing
|
|
26
|
+
* host reads as an unavailable capability instead of a broken import.
|
|
27
|
+
*/
|
|
28
|
+
import { randomUUID } from "node:crypto";
|
|
29
|
+
import { realpathSync } from "node:fs";
|
|
30
|
+
import { join } from "node:path";
|
|
31
|
+
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
32
|
+
import {
|
|
33
|
+
OPERATIONS_ATTEMPT_ID_ENV,
|
|
34
|
+
OPERATIONS_CHILD_ID_ENV,
|
|
35
|
+
OPERATIONS_CONVERSATION_ID_ENV,
|
|
36
|
+
OPERATIONS_ENABLED_ENV,
|
|
37
|
+
OPERATIONS_ENDPOINT_ENV,
|
|
38
|
+
OPERATIONS_RUN_ID_ENV,
|
|
39
|
+
OPERATIONS_TOKEN_ENV,
|
|
40
|
+
OPERATIONS_WRITER_LEASE_ID_ENV,
|
|
41
|
+
WAIT_OPERATIONS_TOOL,
|
|
42
|
+
createToolDescriptor,
|
|
43
|
+
extractResultTaskIds,
|
|
44
|
+
normalizeWaitForOperationsResult,
|
|
45
|
+
normalizeToolDescriptors,
|
|
46
|
+
OperationBridgeError,
|
|
47
|
+
type JsonRecord,
|
|
48
|
+
type OperationBridgeCapabilities,
|
|
49
|
+
type OperationBridgeToolName,
|
|
50
|
+
type OperationToolDescriptor,
|
|
51
|
+
} from "./operation-bridge-types.ts";
|
|
52
|
+
import {
|
|
53
|
+
createOperationBridgeHost,
|
|
54
|
+
type OperationBridgeHost,
|
|
55
|
+
type OperationBridgeLogger,
|
|
56
|
+
type OperationRunHandlers,
|
|
57
|
+
type OperationRunInfo,
|
|
58
|
+
} from "./worker-operations-host.ts";
|
|
59
|
+
import { captureParkCheckpoint, verifyParkCheckpoint, type ParkCheckpoint } from "./park-checkpoint.ts";
|
|
60
|
+
import { findSavedConversation, readConversationLease } from "./continuation.ts";
|
|
61
|
+
import {
|
|
62
|
+
effectiveStatus,
|
|
63
|
+
isPendingPark,
|
|
64
|
+
listMetas,
|
|
65
|
+
listPendingParkedRuns,
|
|
66
|
+
readMeta,
|
|
67
|
+
readParkRecord,
|
|
68
|
+
processStartToken,
|
|
69
|
+
updateParkRecord,
|
|
70
|
+
writeParkRecord,
|
|
71
|
+
type ParkRecord,
|
|
72
|
+
type RunMeta,
|
|
73
|
+
} from "./registry.ts";
|
|
74
|
+
import { processExists } from "./spawn.ts";
|
|
75
|
+
import { realProcessProbe, type ProcessProbe } from "./health.ts";
|
|
76
|
+
import { stopRun } from "./stop.ts";
|
|
77
|
+
|
|
78
|
+
export type { ParkRecord };
|
|
79
|
+
|
|
80
|
+
// ---- Background service contract (versioned event bus) --------------------
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Background publishes its operation service object here and answers a request
|
|
84
|
+
* for it on the request channel. The shape mirrors `BackgroundOperationService`
|
|
85
|
+
* in `pi-better-background-tasks/src/observed-waits.ts`; only the members this
|
|
86
|
+
* module calls are declared.
|
|
87
|
+
*/
|
|
88
|
+
export const OBSERVED_WAITS_SERVICE_CHANNEL = "pi-better-background-tasks:observed-waits-service";
|
|
89
|
+
export const OBSERVED_WAITS_SERVICE_REQUEST_CHANNEL = "pi-better-background-tasks:observed-waits-service-request";
|
|
90
|
+
export const OBSERVED_WAITS_SERVICE_VERSION = 1 as const;
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Request/reply surface Background publishes for the Subagents coordinator. It
|
|
94
|
+
* carries the pieces the object service does not: run bindings for the worker's
|
|
95
|
+
* captured policy, checkpoint staging, and the resume claim/publish/release fence.
|
|
96
|
+
*/
|
|
97
|
+
export const WORKER_OPERATIONS_RPC_CHANNEL = "pi-better-background-tasks:observed-waits";
|
|
98
|
+
export const WORKER_OPERATIONS_RPC_VERSION = 1 as const;
|
|
99
|
+
|
|
100
|
+
export class WorkerOperationsRefusal extends Error {
|
|
101
|
+
readonly code: string;
|
|
102
|
+
constructor(code: string, message: string) {
|
|
103
|
+
super(message);
|
|
104
|
+
this.name = "WorkerOperationsRefusal";
|
|
105
|
+
this.code = code;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export class WorkerOperationsUnavailable extends Error {
|
|
110
|
+
constructor(message: string) {
|
|
111
|
+
super(message);
|
|
112
|
+
this.name = "WorkerOperationsUnavailable";
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export interface WorkerOperationsRpc {
|
|
117
|
+
call(method: string, params: JsonRecord, timeoutMs?: number): Promise<JsonRecord>;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Bounded request/reply client. A missing or slow host is an explicit
|
|
122
|
+
* unavailability, never an assumed success; a late reply is ignored.
|
|
123
|
+
*/
|
|
124
|
+
export function createWorkerOperationsRpc(pi: unknown, options: { timeoutMs?: number } = {}): WorkerOperationsRpc {
|
|
125
|
+
const budgetFor = (override?: number): number => Math.max(
|
|
126
|
+
50,
|
|
127
|
+
Math.min(120_000, override ?? options.timeoutMs ?? 30_000),
|
|
128
|
+
);
|
|
129
|
+
const call = (method: string, params: JsonRecord, overrideMs?: number): Promise<JsonRecord> => {
|
|
130
|
+
const bus = serviceBusOf(pi);
|
|
131
|
+
if (!bus) {
|
|
132
|
+
return Promise.reject(new WorkerOperationsUnavailable(
|
|
133
|
+
"This Pi runtime exposes no extension event bus, so worker operations are unavailable.",
|
|
134
|
+
));
|
|
135
|
+
}
|
|
136
|
+
const budget = budgetFor(overrideMs);
|
|
137
|
+
return new Promise<JsonRecord>((resolve, reject) => {
|
|
138
|
+
let settled = false;
|
|
139
|
+
const finish = (action: () => void): void => {
|
|
140
|
+
if (settled) return;
|
|
141
|
+
settled = true;
|
|
142
|
+
clearTimeout(timer);
|
|
143
|
+
action();
|
|
144
|
+
};
|
|
145
|
+
const timer = setTimeout(() => {
|
|
146
|
+
finish(() => reject(new WorkerOperationsUnavailable(
|
|
147
|
+
`The Background observed-waits host did not answer ${method} within ${budget} ms. No wait, checkpoint or resume was recorded.`,
|
|
148
|
+
)));
|
|
149
|
+
}, budget);
|
|
150
|
+
timer.unref?.();
|
|
151
|
+
const request = {
|
|
152
|
+
version: WORKER_OPERATIONS_RPC_VERSION,
|
|
153
|
+
id: `ow_${randomUUID()}`,
|
|
154
|
+
method,
|
|
155
|
+
params,
|
|
156
|
+
reply: (reply: unknown) => {
|
|
157
|
+
if (!reply || typeof reply !== "object") {
|
|
158
|
+
finish(() => reject(new WorkerOperationsUnavailable("The observed-waits host returned a malformed reply.")));
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
const value = reply as { ok?: unknown; value?: unknown; code?: unknown; reason?: unknown };
|
|
162
|
+
if (value.ok === true) {
|
|
163
|
+
finish(() => resolve(value.value && typeof value.value === "object" ? value.value as JsonRecord : {}));
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
finish(() => reject(new WorkerOperationsRefusal(
|
|
167
|
+
typeof value.code === "string" ? value.code : "refused",
|
|
168
|
+
typeof value.reason === "string" && value.reason ? value.reason : "The observed-waits host refused the request.",
|
|
169
|
+
)));
|
|
170
|
+
},
|
|
171
|
+
};
|
|
172
|
+
try {
|
|
173
|
+
bus.emit(WORKER_OPERATIONS_RPC_CHANNEL, request);
|
|
174
|
+
} catch (error) {
|
|
175
|
+
finish(() => reject(new WorkerOperationsUnavailable(
|
|
176
|
+
`The observed-waits request failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
177
|
+
)));
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
};
|
|
181
|
+
return { call };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Goal's admission gate; this module only asks, never decides. */
|
|
185
|
+
export const OPERATION_RESUME_ADMISSION_EVENT = "pi-better-goal:operation-resume-admission";
|
|
186
|
+
|
|
187
|
+
export type WorkerOwnerIdentity = {
|
|
188
|
+
kind: "worker-conversation";
|
|
189
|
+
sessionId: string;
|
|
190
|
+
workspace: string;
|
|
191
|
+
branchId?: string | null;
|
|
192
|
+
goalId?: string | null;
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
/** What one worker attempt is allowed to ask Background to record. */
|
|
196
|
+
export interface WorkerRunScope {
|
|
197
|
+
readonly runId: string;
|
|
198
|
+
readonly conversationId: string;
|
|
199
|
+
readonly workspace: string;
|
|
200
|
+
readonly authorityReference: string;
|
|
201
|
+
readonly branchId?: string | null;
|
|
202
|
+
readonly goalId?: string | null;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export interface WorkerSandboxAuthority {
|
|
206
|
+
readonly root: string;
|
|
207
|
+
readonly permissions?: Readonly<{
|
|
208
|
+
projectFiles: "off" | "read" | "write" | "read-write";
|
|
209
|
+
outsideProject: "off" | "read" | "write" | "read-write";
|
|
210
|
+
storedCredentials: "off" | "read" | "read-write";
|
|
211
|
+
commands: boolean;
|
|
212
|
+
network: boolean;
|
|
213
|
+
shell?: boolean;
|
|
214
|
+
commandTimeoutSeconds?: number | null;
|
|
215
|
+
}>;
|
|
216
|
+
readonly denyWrite?: readonly string[];
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export interface WorkerLaunchResult {
|
|
220
|
+
readonly id: string;
|
|
221
|
+
readonly operationId: string;
|
|
222
|
+
readonly kind: "process" | "watch";
|
|
223
|
+
readonly status: string;
|
|
224
|
+
readonly name?: string;
|
|
225
|
+
readonly cwd: string;
|
|
226
|
+
readonly command?: string;
|
|
227
|
+
readonly argv?: readonly string[];
|
|
228
|
+
readonly shell: boolean;
|
|
229
|
+
readonly owner: WorkerOwnerIdentity;
|
|
230
|
+
readonly started: boolean;
|
|
231
|
+
readonly claimed: boolean;
|
|
232
|
+
readonly notices: readonly string[];
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
export interface WaitDependencyView {
|
|
236
|
+
readonly taskId?: string;
|
|
237
|
+
readonly operationId: string;
|
|
238
|
+
readonly terminalRevision: number;
|
|
239
|
+
readonly terminalOutcome: string | null;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
export interface WaitEvidenceView {
|
|
243
|
+
readonly operationId: string;
|
|
244
|
+
readonly taskId?: string;
|
|
245
|
+
readonly terminalRevision: number;
|
|
246
|
+
readonly kind: string;
|
|
247
|
+
readonly ref: string;
|
|
248
|
+
readonly summary?: string;
|
|
249
|
+
readonly detailLost: boolean;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
export interface WorkerWaitView {
|
|
253
|
+
readonly waitId: string;
|
|
254
|
+
readonly generation: number;
|
|
255
|
+
readonly phase: string;
|
|
256
|
+
readonly settled: boolean;
|
|
257
|
+
readonly park: boolean;
|
|
258
|
+
/** Cancellation epoch the caller must echo; part of the synchronous release guard. */
|
|
259
|
+
readonly cancelEpoch: number;
|
|
260
|
+
readonly ownerKind: string;
|
|
261
|
+
readonly continuationNote: string;
|
|
262
|
+
readonly taskIds: readonly string[];
|
|
263
|
+
readonly dependencies: readonly WaitDependencyView[];
|
|
264
|
+
readonly evidence: readonly WaitEvidenceView[];
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export interface WorkerStatusView {
|
|
268
|
+
readonly taskId: string;
|
|
269
|
+
readonly operationId: string;
|
|
270
|
+
readonly kind: "process" | "watch";
|
|
271
|
+
readonly status: string;
|
|
272
|
+
readonly phase: string;
|
|
273
|
+
readonly exitCode: number | null;
|
|
274
|
+
readonly signal: string | null;
|
|
275
|
+
readonly reason?: string;
|
|
276
|
+
readonly terminalRevision: number;
|
|
277
|
+
readonly terminalOutcome: string | null;
|
|
278
|
+
readonly waits: readonly WorkerWaitView[];
|
|
279
|
+
readonly [key: string]: unknown;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export interface WorkerLogView {
|
|
283
|
+
readonly taskId: string;
|
|
284
|
+
readonly status: string;
|
|
285
|
+
readonly stdout: string;
|
|
286
|
+
readonly stderr: string;
|
|
287
|
+
readonly events: string;
|
|
288
|
+
readonly truncated: boolean;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
export interface WorkerHoldResumeInput {
|
|
292
|
+
waitId: string;
|
|
293
|
+
generation: number;
|
|
294
|
+
claimId: string;
|
|
295
|
+
claimToken: string;
|
|
296
|
+
attemptId: string;
|
|
297
|
+
launchToken: string;
|
|
298
|
+
proof: { writerPidGone: boolean; writerLeaseReleased: boolean };
|
|
299
|
+
reason: string;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export interface WorkerHoldResumeView {
|
|
303
|
+
readonly waitId: string;
|
|
304
|
+
readonly generation: number;
|
|
305
|
+
readonly phase: string;
|
|
306
|
+
readonly applied: boolean;
|
|
307
|
+
readonly claim: { readonly claimId: string; readonly state: string; readonly attemptId: string | null };
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
export interface WorkerAcknowledgeView {
|
|
311
|
+
readonly waitId: string;
|
|
312
|
+
readonly generation: number;
|
|
313
|
+
readonly disposition: string;
|
|
314
|
+
readonly acknowledgedOutboxIds: readonly string[];
|
|
315
|
+
readonly releasedPinIds: readonly string[];
|
|
316
|
+
readonly phase: string;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** The ready-wait delivery Background hands to its continuation adapter. */
|
|
320
|
+
export interface WorkerWaitReadyEvent {
|
|
321
|
+
readonly waitId: string;
|
|
322
|
+
readonly generation: number;
|
|
323
|
+
readonly owner: { kind: string; sessionId: string; workspace: string; branchId?: string | null; goalId?: string | null };
|
|
324
|
+
readonly parentOrigin?: { cwd: string; sessionId?: string };
|
|
325
|
+
readonly taskIds: readonly string[];
|
|
326
|
+
readonly continuationNote: string;
|
|
327
|
+
readonly evidence: readonly WaitEvidenceView[];
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
export interface WorkerContinuationAdapter {
|
|
331
|
+
readonly version: 1;
|
|
332
|
+
readonly id: string;
|
|
333
|
+
onWaitReady(event: WorkerWaitReadyEvent): void | Promise<void | "delivered" | "deferred">;
|
|
334
|
+
onWaitCancelled?(event: { waitId: string; generation: number; reason: string }): void | Promise<void>;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* The published Background operation service. Every method rechecks ownership in
|
|
339
|
+
* the durable store; the run id is an identity, never a grant.
|
|
340
|
+
*/
|
|
341
|
+
export interface WorkerOperationService {
|
|
342
|
+
readonly version: typeof OBSERVED_WAITS_SERVICE_VERSION;
|
|
343
|
+
readonly serviceId: string;
|
|
344
|
+
readonly available: true;
|
|
345
|
+
readonly continuationReady: boolean;
|
|
346
|
+
ownsTask(run: WorkerRunScope, taskId: string): boolean;
|
|
347
|
+
launchForWorker(run: WorkerRunScope, input: { kind: "process" | "watch"; params: JsonRecord; policy?: WorkerSandboxAuthority | null }): Promise<WorkerLaunchResult>;
|
|
348
|
+
statusForWorker(run: WorkerRunScope, taskId: string): WorkerStatusView;
|
|
349
|
+
logForWorker(run: WorkerRunScope, taskId: string, options?: { lines?: number }): WorkerLogView;
|
|
350
|
+
stopForWorker(run: WorkerRunScope, taskId: string): Promise<WorkerStatusView>;
|
|
351
|
+
registerWorkerWait(run: WorkerRunScope, input: {
|
|
352
|
+
taskIds: readonly string[];
|
|
353
|
+
continuationNote: string;
|
|
354
|
+
attemptId: string;
|
|
355
|
+
checkpointId: string;
|
|
356
|
+
writerLeaseId?: string | null;
|
|
357
|
+
}): WorkerWaitView;
|
|
358
|
+
checkpointWorkerWait(run: WorkerRunScope, input: {
|
|
359
|
+
waitId: string;
|
|
360
|
+
generation: number;
|
|
361
|
+
proof: { checkpointRecorded: boolean; writerLeaseReleased: boolean; writerPidGone: boolean };
|
|
362
|
+
}): Promise<WorkerWaitView>;
|
|
363
|
+
failWorkerPark(run: WorkerRunScope, input: { waitId: string; generation: number; reason: string }): WorkerWaitView;
|
|
364
|
+
acknowledgeForWorker(run: WorkerRunScope, input: {
|
|
365
|
+
waitId: string;
|
|
366
|
+
generation: number;
|
|
367
|
+
terminalRevisions: unknown;
|
|
368
|
+
disposition: "continue" | "blocked" | "assignment-complete";
|
|
369
|
+
note: string;
|
|
370
|
+
}): WorkerAcknowledgeView;
|
|
371
|
+
listWorkerWaits(run: WorkerRunScope, options?: { pendingOnly?: boolean }): readonly WorkerWaitView[];
|
|
372
|
+
cancelWorkerWait(run: WorkerRunScope, input: { waitId: string; generation: number; reason: string }): WorkerWaitView;
|
|
373
|
+
/**
|
|
374
|
+
* Hold a proven failed published attempt. Optional in older hosts; without it
|
|
375
|
+
* the caller reports the local hold and leaves the durable generation with its
|
|
376
|
+
* live claim instead of cancelling it.
|
|
377
|
+
*/
|
|
378
|
+
holdFailedResumeForWorker?(run: WorkerRunScope, input: WorkerHoldResumeInput): WorkerHoldResumeView;
|
|
379
|
+
registerContinuationAdapter(adapter: WorkerContinuationAdapter): void;
|
|
380
|
+
unregisterContinuationAdapter(id: string): void;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
interface ServiceBus {
|
|
384
|
+
on(channel: string, handler: (data: unknown) => void): unknown;
|
|
385
|
+
off?(channel: string, handler: (data: unknown) => void): unknown;
|
|
386
|
+
emit(channel: string, data: unknown): void;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
const serviceMirrors = new WeakMap<object, { service?: WorkerOperationService; reason?: string }>();
|
|
390
|
+
|
|
391
|
+
function serviceBusOf(pi: unknown): ServiceBus | undefined {
|
|
392
|
+
const events = (pi as { events?: unknown } | undefined)?.events;
|
|
393
|
+
if (!events || typeof events !== "object") return undefined;
|
|
394
|
+
const candidate = events as Partial<ServiceBus>;
|
|
395
|
+
if (typeof candidate.on !== "function" || typeof candidate.emit !== "function") return undefined;
|
|
396
|
+
return candidate as ServiceBus;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
function isPublishedService(value: unknown): value is WorkerOperationService {
|
|
400
|
+
if (!value || typeof value !== "object") return false;
|
|
401
|
+
const candidate = value as Partial<WorkerOperationService>;
|
|
402
|
+
return candidate.version === OBSERVED_WAITS_SERVICE_VERSION
|
|
403
|
+
&& candidate.available === true
|
|
404
|
+
&& typeof candidate.launchForWorker === "function"
|
|
405
|
+
&& typeof candidate.statusForWorker === "function"
|
|
406
|
+
&& typeof candidate.logForWorker === "function"
|
|
407
|
+
&& typeof candidate.registerWorkerWait === "function"
|
|
408
|
+
&& typeof candidate.checkpointWorkerWait === "function"
|
|
409
|
+
&& typeof candidate.failWorkerPark === "function"
|
|
410
|
+
&& typeof candidate.acknowledgeForWorker === "function"
|
|
411
|
+
&& typeof candidate.listWorkerWaits === "function"
|
|
412
|
+
&& typeof candidate.cancelWorkerWait === "function"
|
|
413
|
+
&& typeof candidate.ownsTask === "function"
|
|
414
|
+
&& typeof candidate.registerContinuationAdapter === "function"
|
|
415
|
+
&& (candidate.holdFailedResumeForWorker === undefined || typeof candidate.holdFailedResumeForWorker === "function");
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function mirrorOf(pi: unknown): { mirror: { service?: WorkerOperationService; reason?: string }; bus: ServiceBus } | undefined {
|
|
419
|
+
const bus = serviceBusOf(pi);
|
|
420
|
+
if (!bus) return undefined;
|
|
421
|
+
let mirror = serviceMirrors.get(bus as object);
|
|
422
|
+
if (!mirror) {
|
|
423
|
+
mirror = {};
|
|
424
|
+
serviceMirrors.set(bus as object, mirror);
|
|
425
|
+
bus.on(OBSERVED_WAITS_SERVICE_CHANNEL, (data) => {
|
|
426
|
+
const current = serviceMirrors.get(bus as object);
|
|
427
|
+
if (!current) return;
|
|
428
|
+
if (isPublishedService(data)) {
|
|
429
|
+
current.service = data;
|
|
430
|
+
current.reason = undefined;
|
|
431
|
+
return;
|
|
432
|
+
}
|
|
433
|
+
if (data && typeof data === "object"
|
|
434
|
+
&& (data as { version?: unknown }).version === OBSERVED_WAITS_SERVICE_VERSION
|
|
435
|
+
&& (data as { available?: unknown }).available === false) {
|
|
436
|
+
current.service = undefined;
|
|
437
|
+
const reason = (data as { reason?: unknown }).reason;
|
|
438
|
+
current.reason = typeof reason === "string" ? reason : "the Background operation host is unavailable";
|
|
439
|
+
}
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
return { mirror, bus };
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* Resolve the Background operation service through its versioned channel. Pi's
|
|
447
|
+
* bus dispatches synchronously, so a host that is already running answers before
|
|
448
|
+
* this returns; `undefined` means this package runs alone (or the host closed).
|
|
449
|
+
*/
|
|
450
|
+
export function getWorkerOperationService(pi: unknown): WorkerOperationService | undefined {
|
|
451
|
+
const resolved = mirrorOf(pi);
|
|
452
|
+
if (!resolved) return undefined;
|
|
453
|
+
try {
|
|
454
|
+
resolved.bus.emit(OBSERVED_WAITS_SERVICE_REQUEST_CHANNEL, { version: OBSERVED_WAITS_SERVICE_VERSION });
|
|
455
|
+
} catch {
|
|
456
|
+
return undefined;
|
|
457
|
+
}
|
|
458
|
+
return resolved.mirror.service;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/** Why the service request returned nothing, when a host refused to open. */
|
|
462
|
+
export function workerOperationsUnavailableReason(pi: unknown): string | undefined {
|
|
463
|
+
const resolved = mirrorOf(pi);
|
|
464
|
+
if (!resolved) return "this Pi runtime exposes no extension event bus";
|
|
465
|
+
if (!resolved.mirror.service) {
|
|
466
|
+
try {
|
|
467
|
+
resolved.bus.emit(OBSERVED_WAITS_SERVICE_REQUEST_CHANNEL, { version: OBSERVED_WAITS_SERVICE_VERSION });
|
|
468
|
+
} catch { /* keep the previous reason */ }
|
|
469
|
+
}
|
|
470
|
+
return resolved.mirror.reason;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
// ---- captured worker policy ----------------------------------------------
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* What the worker was launched with, captured by Subagents before the child
|
|
477
|
+
* exists. Background intersects it with the current Background profile; the
|
|
478
|
+
* child can never supply or widen it.
|
|
479
|
+
*/
|
|
480
|
+
export interface WorkerRunPolicy {
|
|
481
|
+
version: 1;
|
|
482
|
+
/** Confinement root and levels, or null for the `sandbox:false` opt-out. */
|
|
483
|
+
sandbox: WorkerSandboxAuthority | null;
|
|
484
|
+
/** Effective child tool allowlist (after exclusions). */
|
|
485
|
+
tools: readonly string[];
|
|
486
|
+
/** Clean-mode children get no ambient operation capability. */
|
|
487
|
+
clean: boolean;
|
|
488
|
+
/** Assignment deadline preserved across parking and resumption. */
|
|
489
|
+
deadlineAt: number | null;
|
|
490
|
+
/** Reference to the permission snapshot; never credential material. */
|
|
491
|
+
authorityReference: string;
|
|
492
|
+
/** Workspace the worker is confined to (canonical). */
|
|
493
|
+
workspace: string;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/** Build the policy Background needs; `taskPolicy` is the immutable task launch contract. */
|
|
497
|
+
export function captureWorkerPolicy(input: {
|
|
498
|
+
sandbox: boolean;
|
|
499
|
+
tools: readonly string[];
|
|
500
|
+
clean: boolean;
|
|
501
|
+
workspace: string;
|
|
502
|
+
authorityReference: string;
|
|
503
|
+
deadlineAt?: number | null;
|
|
504
|
+
taskPolicy?: {
|
|
505
|
+
readonly root: string;
|
|
506
|
+
readonly permissions?: WorkerSandboxAuthority["permissions"];
|
|
507
|
+
readonly denyWrite?: readonly string[];
|
|
508
|
+
} | null;
|
|
509
|
+
}): WorkerRunPolicy {
|
|
510
|
+
const sandbox = input.sandbox && input.taskPolicy
|
|
511
|
+
? {
|
|
512
|
+
root: input.taskPolicy.root,
|
|
513
|
+
...(input.taskPolicy.permissions ? { permissions: input.taskPolicy.permissions } : {}),
|
|
514
|
+
...(input.taskPolicy.denyWrite ? { denyWrite: [...input.taskPolicy.denyWrite] } : {}),
|
|
515
|
+
}
|
|
516
|
+
: null;
|
|
517
|
+
return {
|
|
518
|
+
version: 1,
|
|
519
|
+
sandbox,
|
|
520
|
+
tools: [...input.tools],
|
|
521
|
+
clean: input.clean,
|
|
522
|
+
deadlineAt: input.deadlineAt ?? null,
|
|
523
|
+
authorityReference: input.authorityReference,
|
|
524
|
+
workspace: input.workspace,
|
|
525
|
+
};
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
// ---- child capability planning -------------------------------------------
|
|
529
|
+
|
|
530
|
+
/** Fallback tool set for a supervised worker; user restrictions still subtract. */
|
|
531
|
+
export const DEFAULT_WORKER_BRIDGE_TOOLS: readonly OperationBridgeToolName[] = [
|
|
532
|
+
"bg_task_spawn",
|
|
533
|
+
"bg_task_watch",
|
|
534
|
+
"bg_task_status",
|
|
535
|
+
"bg_task_log",
|
|
536
|
+
WAIT_OPERATIONS_TOOL,
|
|
537
|
+
"acknowledge_operation_result",
|
|
538
|
+
];
|
|
539
|
+
|
|
540
|
+
export interface WorkerBridgeToolPlan {
|
|
541
|
+
tools: readonly OperationBridgeToolName[];
|
|
542
|
+
capabilities: OperationBridgeCapabilities;
|
|
543
|
+
/** Withheld tools and why. A withheld tool is never advertised to the child. */
|
|
544
|
+
withheld: readonly { name: OperationBridgeToolName; reason: string }[];
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Decide the worker's operation tools.
|
|
549
|
+
*
|
|
550
|
+
* - `clean` children keep their explicit no-ambient-capability loadout.
|
|
551
|
+
* - An explicit `tools` allowlist only passes bridge tools the caller named.
|
|
552
|
+
* - `exclude_tools` always wins.
|
|
553
|
+
* - A sandboxed child is not treated differently here: its task policy enables
|
|
554
|
+
* the guarded bridge only when the fixed bridge adapter is present, and the
|
|
555
|
+
* task guard admits the tools by exact source. Withholding them here would
|
|
556
|
+
* silently disable a capability the sandbox already supports.
|
|
557
|
+
* - Without any restriction, defaults get the full set; the Background host
|
|
558
|
+
* still intersects the run's captured task authority with the current profile
|
|
559
|
+
* before anything launches.
|
|
560
|
+
*/
|
|
561
|
+
export function planWorkerBridgeTools(input: {
|
|
562
|
+
allow: readonly string[];
|
|
563
|
+
excluded: readonly string[];
|
|
564
|
+
clean: boolean;
|
|
565
|
+
explicitTools: boolean;
|
|
566
|
+
/** Production `--tools`/tool schemas published by the parent, when known. */
|
|
567
|
+
productionSchemas?: ReadonlySet<string> | null;
|
|
568
|
+
}): WorkerBridgeToolPlan {
|
|
569
|
+
const excluded = new Set(input.excluded.map((name) => name.trim()).filter(Boolean));
|
|
570
|
+
const allow = new Set(input.allow.map((name) => name.trim()).filter(Boolean));
|
|
571
|
+
const withheld: { name: OperationBridgeToolName; reason: string }[] = [];
|
|
572
|
+
const tools: OperationBridgeToolName[] = [];
|
|
573
|
+
for (const name of DEFAULT_WORKER_BRIDGE_TOOLS) {
|
|
574
|
+
if (excluded.has(name)) {
|
|
575
|
+
withheld.push({ name, reason: "excluded by the launch's exclude_tools" });
|
|
576
|
+
continue;
|
|
577
|
+
}
|
|
578
|
+
if (input.clean) {
|
|
579
|
+
withheld.push({ name, reason: "clean mode disables ambient operation capabilities" });
|
|
580
|
+
continue;
|
|
581
|
+
}
|
|
582
|
+
if (input.explicitTools && !allow.has(name)) {
|
|
583
|
+
withheld.push({ name, reason: "not in the launch's explicit tools list" });
|
|
584
|
+
continue;
|
|
585
|
+
}
|
|
586
|
+
// Background tools must be described by the production schemas the parent
|
|
587
|
+
// publishes; the bridge's own two tools carry their own contract.
|
|
588
|
+
const backgroundTool = name !== WAIT_OPERATIONS_TOOL && name !== "acknowledge_operation_result";
|
|
589
|
+
if (backgroundTool && input.productionSchemas && !input.productionSchemas.has(name)) {
|
|
590
|
+
withheld.push({ name, reason: "the parent Background extension publishes no production schema for this tool" });
|
|
591
|
+
continue;
|
|
592
|
+
}
|
|
593
|
+
tools.push(name);
|
|
594
|
+
}
|
|
595
|
+
const has = (name: OperationBridgeToolName): boolean => tools.includes(name);
|
|
596
|
+
return {
|
|
597
|
+
tools,
|
|
598
|
+
withheld,
|
|
599
|
+
capabilities: {
|
|
600
|
+
background: {
|
|
601
|
+
spawn: has("bg_task_spawn"),
|
|
602
|
+
watch: has("bg_task_watch"),
|
|
603
|
+
status: has("bg_task_status"),
|
|
604
|
+
log: has("bg_task_log"),
|
|
605
|
+
},
|
|
606
|
+
wait: has(WAIT_OPERATIONS_TOOL),
|
|
607
|
+
acknowledge: has("acknowledge_operation_result"),
|
|
608
|
+
// The bridge derives `wait_for_operations` from `wait && parking`.
|
|
609
|
+
parking: has(WAIT_OPERATIONS_TOOL),
|
|
610
|
+
},
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
/** Production tool descriptors for bridge tools, from the parent's registrations. */
|
|
615
|
+
export function resolveProductionSchemas(
|
|
616
|
+
pi: unknown,
|
|
617
|
+
names: readonly OperationBridgeToolName[] = DEFAULT_WORKER_BRIDGE_TOOLS,
|
|
618
|
+
): { schemas: Map<string, OperationToolDescriptor>; missing: OperationBridgeToolName[] } {
|
|
619
|
+
const inventory = (pi as {
|
|
620
|
+
getAllTools?: () => Array<{ name?: unknown; label?: unknown; description?: unknown; parameters?: unknown }>;
|
|
621
|
+
}).getAllTools?.() ?? [];
|
|
622
|
+
const schemas = new Map<string, OperationToolDescriptor>();
|
|
623
|
+
const missing: OperationBridgeToolName[] = [];
|
|
624
|
+
for (const name of names) {
|
|
625
|
+
if (name === WAIT_OPERATIONS_TOOL || name === "acknowledge_operation_result") {
|
|
626
|
+
schemas.set(name, createToolDescriptor(name));
|
|
627
|
+
continue;
|
|
628
|
+
}
|
|
629
|
+
const tool = inventory.find((entry) => entry?.name === name);
|
|
630
|
+
if (!tool || !tool.parameters || typeof tool.parameters !== "object") {
|
|
631
|
+
missing.push(name);
|
|
632
|
+
continue;
|
|
633
|
+
}
|
|
634
|
+
schemas.set(name, {
|
|
635
|
+
name,
|
|
636
|
+
label: tool.label && typeof tool.label === "string" ? tool.label : createToolDescriptor(name).label,
|
|
637
|
+
description: typeof tool.description === "string" && tool.description ? tool.description : createToolDescriptor(name).description,
|
|
638
|
+
parameters: tool.parameters as JsonRecord,
|
|
639
|
+
});
|
|
640
|
+
}
|
|
641
|
+
return { schemas, missing };
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
// ---- bridge host --------------------------------------------------------
|
|
645
|
+
|
|
646
|
+
/** Checkpoint evidence supplied by the child, before the parent trusts it. */
|
|
647
|
+
export interface CheckpointEvidenceInput {
|
|
648
|
+
runId: string;
|
|
649
|
+
conversationId: string;
|
|
650
|
+
waitId: string;
|
|
651
|
+
generation: number;
|
|
652
|
+
evidence: ParkCheckpoint;
|
|
653
|
+
workspace: string;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
export type CheckpointEvidenceValidator = (input: CheckpointEvidenceInput) => ParkCheckpoint | Promise<ParkCheckpoint>;
|
|
657
|
+
|
|
658
|
+
function canonical(path: string): string {
|
|
659
|
+
try {
|
|
660
|
+
return realpathSync.native(path);
|
|
661
|
+
} catch {
|
|
662
|
+
return path;
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
/**
|
|
667
|
+
* Default validator: the persisted point must be the trusted launch recipe's own
|
|
668
|
+
* saved conversation (same directory, session id and workspace), and the captured
|
|
669
|
+
* checkpoint must match the evidence the child reported. A file path plus a hash
|
|
670
|
+
* the child chose is not proof.
|
|
671
|
+
*/
|
|
672
|
+
export const recipeCheckpointEvidenceValidator: CheckpointEvidenceValidator = (input) => {
|
|
673
|
+
const meta = readMeta(input.runId);
|
|
674
|
+
const recipe = meta?.recipe;
|
|
675
|
+
if (!recipe) {
|
|
676
|
+
throw new Error(`Cannot verify a park checkpoint for ${input.runId}: its trusted launch recipe is missing.`);
|
|
677
|
+
}
|
|
678
|
+
if (recipe.sessionId !== input.conversationId) {
|
|
679
|
+
throw new Error(
|
|
680
|
+
`Cannot verify a park checkpoint for ${input.runId}: the launch recipe belongs to conversation ${recipe.sessionId}, not ${input.conversationId}.`,
|
|
681
|
+
);
|
|
682
|
+
}
|
|
683
|
+
if (canonical(recipe.cwd) !== canonical(input.workspace)) {
|
|
684
|
+
throw new Error(
|
|
685
|
+
`Cannot verify a park checkpoint for ${input.runId}: the launch recipe belongs to workspace ${recipe.cwd}, not ${input.workspace}.`,
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
const lookup = findSavedConversation({
|
|
689
|
+
sessionDir: recipe.sessionDir,
|
|
690
|
+
sessionId: recipe.sessionId,
|
|
691
|
+
expectedCwd: recipe.cwd,
|
|
692
|
+
});
|
|
693
|
+
switch (lookup.kind) {
|
|
694
|
+
case "ok":
|
|
695
|
+
break;
|
|
696
|
+
case "missing":
|
|
697
|
+
throw new Error(`Cannot verify a park checkpoint for ${input.runId}: its saved conversation is gone.`);
|
|
698
|
+
case "ambiguous":
|
|
699
|
+
throw new Error(`Cannot verify a park checkpoint for ${input.runId}: its saved conversation is ambiguous (${lookup.paths.length} files).`);
|
|
700
|
+
case "invalid":
|
|
701
|
+
case "mismatch":
|
|
702
|
+
throw new Error(`Cannot verify a park checkpoint for ${input.runId}: ${lookup.detail}`);
|
|
703
|
+
}
|
|
704
|
+
if (canonical(lookup.conversation.path) !== canonical(input.evidence.sessionFile)) {
|
|
705
|
+
throw new Error(
|
|
706
|
+
`Cannot verify a park checkpoint for ${input.runId}: the evidence names ${input.evidence.sessionFile}, not this conversation's saved session ${lookup.conversation.path}.`,
|
|
707
|
+
);
|
|
708
|
+
}
|
|
709
|
+
if (input.evidence.sessionId !== recipe.sessionId) {
|
|
710
|
+
throw new Error(`Cannot verify a park checkpoint for ${input.runId}: the evidence names another session id.`);
|
|
711
|
+
}
|
|
712
|
+
const captured = captureParkCheckpoint({
|
|
713
|
+
sessionId: recipe.sessionId,
|
|
714
|
+
sessionFile: lookup.conversation.path,
|
|
715
|
+
toolCallId: input.evidence.toolCallId,
|
|
716
|
+
waitId: input.waitId,
|
|
717
|
+
generation: input.generation,
|
|
718
|
+
});
|
|
719
|
+
verifyParkCheckpoint(captured);
|
|
720
|
+
if (captured.prefixSha256 !== input.evidence.prefixSha256
|
|
721
|
+
|| captured.prefixBytes !== input.evidence.prefixBytes
|
|
722
|
+
|| captured.entryId !== input.evidence.entryId) {
|
|
723
|
+
throw new Error(`Cannot verify a park checkpoint for ${input.runId}: the reported evidence does not describe the persisted session.`);
|
|
724
|
+
}
|
|
725
|
+
return captured;
|
|
726
|
+
};
|
|
727
|
+
|
|
728
|
+
export interface WorkerOperationRegistration {
|
|
729
|
+
/** Subagent attempt id. Equal to the bridge run id. */
|
|
730
|
+
runId: string;
|
|
731
|
+
/** Native conversation id (the saved session). */
|
|
732
|
+
conversationId: string;
|
|
733
|
+
workspace: string;
|
|
734
|
+
authorityReference: string;
|
|
735
|
+
branchId?: string | null;
|
|
736
|
+
goalId?: string | null;
|
|
737
|
+
policy: WorkerRunPolicy;
|
|
738
|
+
capabilities: OperationBridgeCapabilities;
|
|
739
|
+
tools: readonly OperationBridgeToolName[];
|
|
740
|
+
/** Production descriptors; a tool without one is not advertised. */
|
|
741
|
+
schemas: readonly OperationToolDescriptor[];
|
|
742
|
+
/** Continuation attempt this run will become when it parks. */
|
|
743
|
+
attemptId: string;
|
|
744
|
+
/** Conversation writer lease held for this attempt, if any. */
|
|
745
|
+
writerLeaseId?: string | null;
|
|
746
|
+
/** Foreground owner recorded for resume admission. */
|
|
747
|
+
owner?: WaitOwnerBinding;
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
export interface WorkerOperationHandle {
|
|
751
|
+
runId: string;
|
|
752
|
+
conversationId: string;
|
|
753
|
+
token: string;
|
|
754
|
+
endpoint: string;
|
|
755
|
+
capabilities: OperationBridgeCapabilities;
|
|
756
|
+
tools: readonly OperationBridgeToolName[];
|
|
757
|
+
/** Child extension arguments (one `--extension <path>` pair). */
|
|
758
|
+
childExtensionArgs: string[];
|
|
759
|
+
/** Parent-to-child environment. Contains the per-run token; never persisted. */
|
|
760
|
+
env: Record<string, string>;
|
|
761
|
+
descriptorFingerprint: string;
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
export class WorkerOperationsHostUnavailable extends Error {
|
|
765
|
+
constructor(reason: string) {
|
|
766
|
+
super(reason);
|
|
767
|
+
this.name = "WorkerOperationsHostUnavailable";
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
export interface WorkerOperationsHostOptions {
|
|
772
|
+
pi: unknown;
|
|
773
|
+
/** Request/reply fence calls. Injectable for tests. */
|
|
774
|
+
rpc?: WorkerOperationsRpc;
|
|
775
|
+
/** Path to the child bridge extension inside this package. */
|
|
776
|
+
childExtensionPath: string;
|
|
777
|
+
/** Test/replacement seam for native-checkpoint validation. */
|
|
778
|
+
checkpointEvidenceValidator?: CheckpointEvidenceValidator;
|
|
779
|
+
logger?: OperationBridgeLogger;
|
|
780
|
+
/** Test seam for the transport. */
|
|
781
|
+
hostFactory?: typeof createOperationBridgeHost;
|
|
782
|
+
now?: () => number;
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
export interface WorkerOperationsHost {
|
|
786
|
+
readonly available: boolean;
|
|
787
|
+
/** The published object service, once resolved. */
|
|
788
|
+
readonly service: WorkerOperationService | undefined;
|
|
789
|
+
/** Request/reply fence calls used by resume orchestration. */
|
|
790
|
+
readonly rpc: WorkerOperationsRpc;
|
|
791
|
+
start(): Promise<boolean>;
|
|
792
|
+
register(registration: WorkerOperationRegistration): Promise<WorkerOperationHandle>;
|
|
793
|
+
release(runId: string, reason: string): void;
|
|
794
|
+
getRun(runId: string): OperationRunInfo | undefined;
|
|
795
|
+
close(): Promise<void>;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
function scopeOf(registration: {
|
|
799
|
+
runId: string;
|
|
800
|
+
conversationId: string;
|
|
801
|
+
workspace: string;
|
|
802
|
+
authorityReference: string;
|
|
803
|
+
branchId?: string | null;
|
|
804
|
+
goalId?: string | null;
|
|
805
|
+
}): WorkerRunScope {
|
|
806
|
+
return {
|
|
807
|
+
runId: registration.runId,
|
|
808
|
+
conversationId: registration.conversationId,
|
|
809
|
+
workspace: registration.workspace,
|
|
810
|
+
authorityReference: registration.authorityReference,
|
|
811
|
+
branchId: registration.branchId ?? null,
|
|
812
|
+
goalId: registration.goalId ?? null,
|
|
813
|
+
};
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/**
|
|
817
|
+
* Trusted parent-side bridge host.
|
|
818
|
+
*
|
|
819
|
+
* One host per Pi process. It owns no task runner and no database: every launch,
|
|
820
|
+
* observation, wait, checkpoint and interpretation call goes to the Background
|
|
821
|
+
* service with the run's captured policy, and ownership is re-derived from the
|
|
822
|
+
* durable store on each request (so an attempt resumed into a conversation still
|
|
823
|
+
* sees the tasks that conversation launched earlier).
|
|
824
|
+
*/
|
|
825
|
+
export function createWorkerOperationsHost(options: WorkerOperationsHostOptions): WorkerOperationsHost {
|
|
826
|
+
const created: { host?: OperationBridgeHost } = {};
|
|
827
|
+
const starting: { promise?: Promise<boolean> } = {};
|
|
828
|
+
const capability: { available: boolean; service?: WorkerOperationService } = { available: false };
|
|
829
|
+
const runs = new Map<string, WorkerRunScope>();
|
|
830
|
+
const waitNotes = new Map<string, string>();
|
|
831
|
+
const rpc = options.rpc ?? createWorkerOperationsRpc(options.pi);
|
|
832
|
+
const validate = options.checkpointEvidenceValidator ?? recipeCheckpointEvidenceValidator;
|
|
833
|
+
|
|
834
|
+
const ensureHost = async (): Promise<OperationBridgeHost> => {
|
|
835
|
+
if (created.host) return created.host;
|
|
836
|
+
const factory = options.hostFactory ?? createOperationBridgeHost;
|
|
837
|
+
created.host = await factory({
|
|
838
|
+
...(options.logger ? { logger: options.logger } : {}),
|
|
839
|
+
...(options.now ? { now: options.now } : {}),
|
|
840
|
+
});
|
|
841
|
+
return created.host;
|
|
842
|
+
};
|
|
843
|
+
|
|
844
|
+
const start = async (): Promise<boolean> => {
|
|
845
|
+
if (starting.promise) return starting.promise;
|
|
846
|
+
starting.promise = (async () => {
|
|
847
|
+
const service = getWorkerOperationService(options.pi);
|
|
848
|
+
if (!service) {
|
|
849
|
+
capability.available = false;
|
|
850
|
+
return false;
|
|
851
|
+
}
|
|
852
|
+
await ensureHost();
|
|
853
|
+
capability.service = service;
|
|
854
|
+
capability.available = true;
|
|
855
|
+
return true;
|
|
856
|
+
})();
|
|
857
|
+
return starting.promise;
|
|
858
|
+
};
|
|
859
|
+
|
|
860
|
+
const requireService = (): WorkerOperationService => {
|
|
861
|
+
if (!capability.service) {
|
|
862
|
+
throw new WorkerOperationsHostUnavailable(
|
|
863
|
+
"The Background observed-waits service is not available, so this worker cannot be given supervised operation tools or parking. Install pi-better-background-tasks, or launch without them.",
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
return capability.service;
|
|
867
|
+
};
|
|
868
|
+
|
|
869
|
+
const register = async (registration: WorkerOperationRegistration): Promise<WorkerOperationHandle> => {
|
|
870
|
+
if (!capability.available) {
|
|
871
|
+
const ready = await start();
|
|
872
|
+
if (!ready) throw new WorkerOperationsHostUnavailable(workerOperationsUnavailableReason(options.pi) ?? "The Background observed-waits service is not available.");
|
|
873
|
+
}
|
|
874
|
+
const service = requireService();
|
|
875
|
+
const host = await ensureHost();
|
|
876
|
+
const scope = scopeOf(registration);
|
|
877
|
+
runs.set(registration.runId, scope);
|
|
878
|
+
const owner = {
|
|
879
|
+
kind: "worker-conversation" as const,
|
|
880
|
+
sessionId: registration.conversationId,
|
|
881
|
+
workspace: registration.workspace,
|
|
882
|
+
branchId: registration.branchId ?? null,
|
|
883
|
+
goalId: registration.goalId ?? null,
|
|
884
|
+
};
|
|
885
|
+
// The durable run binding captures the worker's task authority and the
|
|
886
|
+
// conversation identity. Every later tool call, checkpoint and resume fence
|
|
887
|
+
// is checked against it by the Background host.
|
|
888
|
+
await rpc.call("register-run", {
|
|
889
|
+
runId: registration.runId,
|
|
890
|
+
conversationId: registration.conversationId,
|
|
891
|
+
attemptId: registration.attemptId,
|
|
892
|
+
workspace: registration.workspace,
|
|
893
|
+
authorityReference: registration.authorityReference,
|
|
894
|
+
policy: registration.policy as unknown as JsonRecord,
|
|
895
|
+
tools: [...registration.tools],
|
|
896
|
+
capabilities: registration.capabilities as unknown as JsonRecord,
|
|
897
|
+
owner,
|
|
898
|
+
...(registration.writerLeaseId ? { writerLeaseId: registration.writerLeaseId } : {}),
|
|
899
|
+
});
|
|
900
|
+
|
|
901
|
+
/**
|
|
902
|
+
* Ownership is re-derived from the durable store (by conversation), so an
|
|
903
|
+
* attempt resumed into this conversation still owns the tasks an earlier
|
|
904
|
+
* attempt launched. This is a transport guard; Background rechecks too.
|
|
905
|
+
*/
|
|
906
|
+
const ownsTask = (taskId: string): boolean => {
|
|
907
|
+
try {
|
|
908
|
+
return service.ownsTask(scope, taskId);
|
|
909
|
+
} catch {
|
|
910
|
+
return false;
|
|
911
|
+
}
|
|
912
|
+
};
|
|
913
|
+
const handlers: OperationRunHandlers = {};
|
|
914
|
+
const execute = (tool: string, params: JsonRecord): Promise<JsonRecord> => rpc.call("execute-tool", {
|
|
915
|
+
runId: registration.runId,
|
|
916
|
+
conversationId: registration.conversationId,
|
|
917
|
+
tool,
|
|
918
|
+
params,
|
|
919
|
+
});
|
|
920
|
+
if (registration.capabilities.background.spawn) {
|
|
921
|
+
handlers.bg_task_spawn = async (params) => execute("bg_task_spawn", params);
|
|
922
|
+
}
|
|
923
|
+
if (registration.capabilities.background.watch) {
|
|
924
|
+
handlers.bg_task_watch = async (params) => execute("bg_task_watch", params);
|
|
925
|
+
}
|
|
926
|
+
if (registration.capabilities.background.status) {
|
|
927
|
+
handlers.bg_task_status = async (params) => execute("bg_task_status", params);
|
|
928
|
+
}
|
|
929
|
+
if (registration.capabilities.background.log) {
|
|
930
|
+
handlers.bg_task_log = async (params) => execute("bg_task_log", params);
|
|
931
|
+
}
|
|
932
|
+
if (registration.capabilities.wait && registration.capabilities.parking) {
|
|
933
|
+
handlers.wait_for_operations = async (params: JsonRecord) => {
|
|
934
|
+
const taskIds = Array.isArray(params.task_ids)
|
|
935
|
+
? params.task_ids.filter((value): value is string => typeof value === "string")
|
|
936
|
+
: [];
|
|
937
|
+
if (taskIds.length === 0) {
|
|
938
|
+
throw new WorkerOperationsRefusal("task-invalid", "wait_for_operations needs at least one task id");
|
|
939
|
+
}
|
|
940
|
+
const note = typeof params.continuation_note === "string" ? params.continuation_note : "";
|
|
941
|
+
if (!note) {
|
|
942
|
+
throw new WorkerOperationsRefusal("task-invalid", "wait_for_operations needs a continuation note");
|
|
943
|
+
}
|
|
944
|
+
const view = await rpc.call("wait-admission", {
|
|
945
|
+
runId: registration.runId,
|
|
946
|
+
conversationId: registration.conversationId,
|
|
947
|
+
taskIds,
|
|
948
|
+
continuationNote: note,
|
|
949
|
+
attemptId: registration.attemptId,
|
|
950
|
+
owner,
|
|
951
|
+
}) as unknown as WorkerWaitView;
|
|
952
|
+
if (!view.park) {
|
|
953
|
+
throw new WorkerOperationsRefusal("wait-invalid", "the Background host did not register a park wait for this worker");
|
|
954
|
+
}
|
|
955
|
+
waitNotes.set(`${view.waitId}:${view.generation}`, note);
|
|
956
|
+
const normalized = normalizeWaitForOperationsResult({ ...view, park: true, taskIds: view.taskIds.length ? view.taskIds : taskIds });
|
|
957
|
+
if (!normalized.ok) {
|
|
958
|
+
throw new WorkerOperationsRefusal("wait-invalid", `wait admission returned an invalid result: ${normalized.message}`);
|
|
959
|
+
}
|
|
960
|
+
return { ...view, ...normalized.result, park: true };
|
|
961
|
+
};
|
|
962
|
+
}
|
|
963
|
+
if (registration.capabilities.acknowledge) {
|
|
964
|
+
handlers.acknowledge_operation_result = async (params: JsonRecord) => {
|
|
965
|
+
const waitId = typeof params.wait_id === "string" ? params.wait_id : "";
|
|
966
|
+
const generation = typeof params.generation === "number" ? params.generation : 0;
|
|
967
|
+
const disposition = params.disposition;
|
|
968
|
+
if (!waitId || !generation || typeof disposition !== "string" || !["continue", "blocked", "assignment-complete"].includes(disposition)) {
|
|
969
|
+
throw new WorkerOperationsRefusal("schema-invalid", "acknowledge_operation_result needs wait_id, generation and a disposition");
|
|
970
|
+
}
|
|
971
|
+
const view = await rpc.call("acknowledge", {
|
|
972
|
+
runId: registration.runId,
|
|
973
|
+
conversationId: registration.conversationId,
|
|
974
|
+
waitId,
|
|
975
|
+
generation,
|
|
976
|
+
terminalRevisions: params.terminal_revisions,
|
|
977
|
+
disposition: disposition as "continue" | "blocked" | "assignment-complete",
|
|
978
|
+
note: typeof params.note === "string" ? params.note : "",
|
|
979
|
+
owner,
|
|
980
|
+
}) as unknown as WorkerAcknowledgeView;
|
|
981
|
+
const parked = parkedWaitByGeneration(waitId, generation);
|
|
982
|
+
if (parked) updateParkRecord(parked.runId, { phase: "consumed" });
|
|
983
|
+
return view;
|
|
984
|
+
};
|
|
985
|
+
}
|
|
986
|
+
if (registration.capabilities.parking) {
|
|
987
|
+
handlers.checkpoint = async (params: JsonRecord) => {
|
|
988
|
+
const waitId = typeof params.wait_id === "string" ? params.wait_id : "";
|
|
989
|
+
const generation = typeof params.generation === "number" ? params.generation : 0;
|
|
990
|
+
const evidence = params.evidence as ParkCheckpoint | undefined;
|
|
991
|
+
if (!waitId || !generation || !evidence || typeof evidence !== "object") {
|
|
992
|
+
throw new WorkerOperationsRefusal("checkpoint-invalid", "checkpoint needs wait_id, generation and evidence");
|
|
993
|
+
}
|
|
994
|
+
// Validate against the trusted recipe (session location, id and
|
|
995
|
+
// workspace) and re-capture the checkpoint from the file itself.
|
|
996
|
+
let captured: ParkCheckpoint;
|
|
997
|
+
try {
|
|
998
|
+
captured = await validate({
|
|
999
|
+
runId: registration.runId,
|
|
1000
|
+
conversationId: registration.conversationId,
|
|
1001
|
+
waitId,
|
|
1002
|
+
generation,
|
|
1003
|
+
evidence,
|
|
1004
|
+
workspace: registration.workspace,
|
|
1005
|
+
});
|
|
1006
|
+
} catch (error) {
|
|
1007
|
+
throw new OperationBridgeError("checkpoint-invalid", reasonOf(error));
|
|
1008
|
+
}
|
|
1009
|
+
const note = waitNotes.get(`${waitId}:${generation}`);
|
|
1010
|
+
const at = options.now?.() ?? Date.now();
|
|
1011
|
+
// STAGED: the original writer is still alive, so the generation stays
|
|
1012
|
+
// `parking`; Background records the evidence and the exit path proves
|
|
1013
|
+
// writer exit before the wait can leave parking.
|
|
1014
|
+
const staged = await rpc.call("checkpoint", {
|
|
1015
|
+
runId: registration.runId,
|
|
1016
|
+
conversationId: registration.conversationId,
|
|
1017
|
+
waitId,
|
|
1018
|
+
generation,
|
|
1019
|
+
toolCallId: captured.toolCallId,
|
|
1020
|
+
attemptId: registration.attemptId,
|
|
1021
|
+
evidence: captured as unknown as JsonRecord,
|
|
1022
|
+
});
|
|
1023
|
+
if (staged.phase !== "parking" && staged.staged !== true) {
|
|
1024
|
+
throw new WorkerOperationsRefusal("checkpoint-invalid", `the Background host reported ${String(staged.phase)} for a staged park checkpoint`);
|
|
1025
|
+
}
|
|
1026
|
+
writeParkRecord(registration.runId, {
|
|
1027
|
+
version: 1,
|
|
1028
|
+
waitId,
|
|
1029
|
+
generation,
|
|
1030
|
+
phase: "parking",
|
|
1031
|
+
attemptId: registration.attemptId,
|
|
1032
|
+
...(note ? { continuationNote: note } : {}),
|
|
1033
|
+
...(registration.owner ? {
|
|
1034
|
+
ownerSessionId: registration.owner.sessionId,
|
|
1035
|
+
ownerWorkspace: registration.owner.workspace,
|
|
1036
|
+
ownerBranchId: registration.owner.branchId,
|
|
1037
|
+
ownerGoalId: registration.owner.goalId,
|
|
1038
|
+
} : {}),
|
|
1039
|
+
checkpoint: {
|
|
1040
|
+
sessionId: captured.sessionId,
|
|
1041
|
+
sessionFile: captured.sessionFile,
|
|
1042
|
+
toolCallId: captured.toolCallId,
|
|
1043
|
+
entryId: captured.entryId,
|
|
1044
|
+
prefixBytes: captured.prefixBytes,
|
|
1045
|
+
prefixSha256: captured.prefixSha256,
|
|
1046
|
+
},
|
|
1047
|
+
parkedAt: at,
|
|
1048
|
+
updatedAt: at,
|
|
1049
|
+
});
|
|
1050
|
+
return { checkpointed: true, staged: true, phase: "parking", waitId, generation };
|
|
1051
|
+
};
|
|
1052
|
+
handlers.park_failed = async (params: JsonRecord) => {
|
|
1053
|
+
const waitId = typeof params.wait_id === "string" ? params.wait_id : "";
|
|
1054
|
+
const generation = typeof params.generation === "number" ? params.generation : 0;
|
|
1055
|
+
const reason = typeof params.reason === "string" && params.reason ? params.reason : "the worker could not certify its park checkpoint";
|
|
1056
|
+
if (!waitId || !generation) {
|
|
1057
|
+
throw new WorkerOperationsRefusal("schema-invalid", "park_failed needs wait_id and generation");
|
|
1058
|
+
}
|
|
1059
|
+
const view = await rpc.call("park-failed", {
|
|
1060
|
+
runId: registration.runId,
|
|
1061
|
+
conversationId: registration.conversationId,
|
|
1062
|
+
waitId,
|
|
1063
|
+
generation,
|
|
1064
|
+
reason,
|
|
1065
|
+
});
|
|
1066
|
+
updateParkRecord(registration.runId, { phase: phaseOf(String(view.phase)), heldReason: reason });
|
|
1067
|
+
return { held: true, phase: view.phase };
|
|
1068
|
+
};
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
const handle = host.registerRun({
|
|
1072
|
+
runId: registration.runId,
|
|
1073
|
+
conversationId: registration.conversationId,
|
|
1074
|
+
workspace: registration.workspace,
|
|
1075
|
+
authorityReference: registration.authorityReference,
|
|
1076
|
+
capabilities: registration.capabilities,
|
|
1077
|
+
tools: normalizeToolDescriptors(registration.schemas, registration.capabilities),
|
|
1078
|
+
ownership: (taskId) => ownsTask(taskId),
|
|
1079
|
+
handlers,
|
|
1080
|
+
});
|
|
1081
|
+
return {
|
|
1082
|
+
runId: registration.runId,
|
|
1083
|
+
conversationId: registration.conversationId,
|
|
1084
|
+
token: handle.token,
|
|
1085
|
+
endpoint: host.endpoint,
|
|
1086
|
+
capabilities: registration.capabilities,
|
|
1087
|
+
tools: registration.tools,
|
|
1088
|
+
childExtensionArgs: ["--extension", options.childExtensionPath],
|
|
1089
|
+
env: childEnvironment(handle.token, host.endpoint, registration),
|
|
1090
|
+
descriptorFingerprint: handle.descriptor.fingerprint,
|
|
1091
|
+
};
|
|
1092
|
+
};
|
|
1093
|
+
|
|
1094
|
+
const release = (runId: string, reason: string): void => {
|
|
1095
|
+
runs.delete(runId);
|
|
1096
|
+
try { created.host?.unregisterRun(runId); } catch { /* already gone */ }
|
|
1097
|
+
void rpc.call("release-run", { runId, reason }).catch(() => {});
|
|
1098
|
+
};
|
|
1099
|
+
|
|
1100
|
+
const close = async (): Promise<void> => {
|
|
1101
|
+
const host = created.host;
|
|
1102
|
+
created.host = undefined;
|
|
1103
|
+
starting.promise = undefined;
|
|
1104
|
+
capability.available = false;
|
|
1105
|
+
capability.service = undefined;
|
|
1106
|
+
runs.clear();
|
|
1107
|
+
waitNotes.clear();
|
|
1108
|
+
if (!host) return;
|
|
1109
|
+
await host.close();
|
|
1110
|
+
};
|
|
1111
|
+
|
|
1112
|
+
return {
|
|
1113
|
+
get available() {
|
|
1114
|
+
return capability.available;
|
|
1115
|
+
},
|
|
1116
|
+
get service() {
|
|
1117
|
+
return capability.service;
|
|
1118
|
+
},
|
|
1119
|
+
rpc,
|
|
1120
|
+
start,
|
|
1121
|
+
register,
|
|
1122
|
+
release,
|
|
1123
|
+
getRun: (runId) => created.host?.getRun(runId),
|
|
1124
|
+
close,
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
/** Environment handed to the child; the token is the only secret and is never persisted. */
|
|
1129
|
+
export function childEnvironment(
|
|
1130
|
+
token: string,
|
|
1131
|
+
endpoint: string,
|
|
1132
|
+
registration: { runId: string; conversationId: string; attemptId: string },
|
|
1133
|
+
): Record<string, string> {
|
|
1134
|
+
return {
|
|
1135
|
+
[OPERATIONS_ENABLED_ENV]: "1",
|
|
1136
|
+
[OPERATIONS_ENDPOINT_ENV]: endpoint,
|
|
1137
|
+
[OPERATIONS_TOKEN_ENV]: token,
|
|
1138
|
+
[OPERATIONS_RUN_ID_ENV]: registration.runId,
|
|
1139
|
+
[OPERATIONS_CONVERSATION_ID_ENV]: registration.conversationId,
|
|
1140
|
+
[OPERATIONS_ATTEMPT_ID_ENV]: registration.attemptId,
|
|
1141
|
+
[OPERATIONS_CHILD_ID_ENV]: `${registration.runId}:${randomUUID()}`,
|
|
1142
|
+
};
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1145
|
+
/** Writer-lease id for a parked attempt, when the continuation path holds one. */
|
|
1146
|
+
export function childWriterLeaseEnv(leaseId: string | undefined): Record<string, string> {
|
|
1147
|
+
return leaseId ? { [OPERATIONS_WRITER_LEASE_ID_ENV]: leaseId } : {};
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
// ---- park record, disposition and post-exit finalization ------------------
|
|
1151
|
+
|
|
1152
|
+
/** Phases a wait can report, mirrored into the park record. */
|
|
1153
|
+
export function phaseOf(phase: string): ParkRecord["phase"] {
|
|
1154
|
+
switch (phase) {
|
|
1155
|
+
case "parking":
|
|
1156
|
+
case "waiting":
|
|
1157
|
+
case "ready":
|
|
1158
|
+
case "resuming":
|
|
1159
|
+
case "held":
|
|
1160
|
+
case "consumed":
|
|
1161
|
+
case "cancelled":
|
|
1162
|
+
return phase;
|
|
1163
|
+
default:
|
|
1164
|
+
return "held";
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
export interface ParkDisposition {
|
|
1169
|
+
/** True when this exit is a clean park: the attempt finished, not the assignment. */
|
|
1170
|
+
parked: boolean;
|
|
1171
|
+
record?: ParkRecord;
|
|
1172
|
+
reason?: string;
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
/** True when the recorded exit is a clean completed attempt. */
|
|
1176
|
+
export function isCleanParkExit(meta: RunMeta | undefined): boolean {
|
|
1177
|
+
if (!meta) return false;
|
|
1178
|
+
if (meta.exitCode !== 0) return false;
|
|
1179
|
+
if (meta.failureReason !== undefined) return false;
|
|
1180
|
+
if (meta.lifecycleClassification === "incomplete_no_terminal_event") return false;
|
|
1181
|
+
if (meta.lifecycleClassification === "incomplete_open_tools") return false;
|
|
1182
|
+
if (meta.lifecycleClassification === "failed_model_error") return false;
|
|
1183
|
+
return meta.status === "completed";
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
/**
|
|
1187
|
+
* Decide the exit disposition for a run.
|
|
1188
|
+
*
|
|
1189
|
+
* A park is recognized only when the exit was clean, a durable park record
|
|
1190
|
+
* exists, its wait generation is still live, and the recorded checkpoint still
|
|
1191
|
+
* verifies against the saved conversation. A staged record (`parking`) counts:
|
|
1192
|
+
* the durable wait is finalized separately, after the writer exit is proven.
|
|
1193
|
+
*/
|
|
1194
|
+
export function evaluateParkDisposition(input: {
|
|
1195
|
+
runId: string;
|
|
1196
|
+
cleanExit: boolean;
|
|
1197
|
+
verifyCheckpoint?: (checkpoint: ParkCheckpoint) => void;
|
|
1198
|
+
}): ParkDisposition {
|
|
1199
|
+
const record = readParkRecord(input.runId);
|
|
1200
|
+
if (!record) return { parked: false };
|
|
1201
|
+
if (!isPendingPark(record)) {
|
|
1202
|
+
return { parked: false, reason: `wait ${record.waitId} generation ${record.generation} is ${record.phase}` };
|
|
1203
|
+
}
|
|
1204
|
+
if (record.phase === "held" || record.phase === "resuming") {
|
|
1205
|
+
return { parked: false, record, reason: record.heldReason ?? `the wait is ${record.phase}` };
|
|
1206
|
+
}
|
|
1207
|
+
if (!input.cleanExit) {
|
|
1208
|
+
return { parked: false, record, reason: "the child did not exit cleanly" };
|
|
1209
|
+
}
|
|
1210
|
+
if (!record.checkpoint) return { parked: false, record, reason: "no verified park checkpoint was recorded" };
|
|
1211
|
+
const verify = input.verifyCheckpoint ?? verifyParkCheckpoint;
|
|
1212
|
+
try {
|
|
1213
|
+
verify({
|
|
1214
|
+
version: 1,
|
|
1215
|
+
sessionId: record.checkpoint.sessionId,
|
|
1216
|
+
sessionFile: record.checkpoint.sessionFile,
|
|
1217
|
+
toolCallId: record.checkpoint.toolCallId,
|
|
1218
|
+
waitId: record.waitId,
|
|
1219
|
+
generation: record.generation,
|
|
1220
|
+
entryId: record.checkpoint.entryId,
|
|
1221
|
+
prefixBytes: record.checkpoint.prefixBytes,
|
|
1222
|
+
prefixSha256: record.checkpoint.prefixSha256,
|
|
1223
|
+
});
|
|
1224
|
+
} catch (error) {
|
|
1225
|
+
return { parked: false, record, reason: `checkpoint no longer verifies: ${reasonOf(error)}` };
|
|
1226
|
+
}
|
|
1227
|
+
return { parked: true, record };
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
export interface ParkFinalizationInput {
|
|
1231
|
+
runId: string;
|
|
1232
|
+
service: WorkerOperationService | undefined;
|
|
1233
|
+
/** True when the caller knows the conversation writer lease was released. */
|
|
1234
|
+
writerLeaseReleased: boolean;
|
|
1235
|
+
/** Defaults to the recorded pid no longer existing. */
|
|
1236
|
+
writerPidGone?: boolean;
|
|
1237
|
+
/**
|
|
1238
|
+
* Restart recovery: the parent that observed the exit is gone, so the exit
|
|
1239
|
+
* code may be unknown. A staged checkpoint whose writer is proven gone and
|
|
1240
|
+
* whose lease is free still describes a finished attempt; the assignment
|
|
1241
|
+
* result is unchanged (nothing is reported as succeeded).
|
|
1242
|
+
*/
|
|
1243
|
+
interrupted?: boolean;
|
|
1244
|
+
now?: () => number;
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
export interface ParkFinalization {
|
|
1248
|
+
outcome: "parked" | "held" | "cancelled" | "skipped";
|
|
1249
|
+
reason?: string;
|
|
1250
|
+
wait?: WorkerWaitView;
|
|
1251
|
+
proof?: { checkpointRecorded: boolean; writerLeaseReleased: boolean; writerPidGone: boolean };
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
/**
|
|
1255
|
+
* The Background owner scope for a parked worker attempt.
|
|
1256
|
+
*
|
|
1257
|
+
* `record.ownerWorkspace` belongs to the FOREGROUND parent that owns the
|
|
1258
|
+
* assignment and is only used for resume admission. Background registered this
|
|
1259
|
+
* worker conversation under the WORKER's workspace (the `sandbox_dir` or clone the
|
|
1260
|
+
* child actually ran in), so ownership must use that; mixing the two makes every
|
|
1261
|
+
* post-exit operation foreign to the durable record.
|
|
1262
|
+
*/
|
|
1263
|
+
export function backgroundScopeForMeta(meta: RunMeta, record: ParkRecord): WorkerRunScope | null {
|
|
1264
|
+
const operations = meta.operations;
|
|
1265
|
+
const conversationId = operations?.conversationId ?? meta.sessionId;
|
|
1266
|
+
const workspace = operations?.workspace ?? meta.cwd;
|
|
1267
|
+
if (!conversationId || !workspace) return null;
|
|
1268
|
+
return {
|
|
1269
|
+
runId: meta.id,
|
|
1270
|
+
conversationId,
|
|
1271
|
+
workspace,
|
|
1272
|
+
authorityReference: operations?.authorityReference ?? `worker:${meta.id}`,
|
|
1273
|
+
branchId: record.ownerBranchId ?? null,
|
|
1274
|
+
goalId: record.ownerGoalId ?? null,
|
|
1275
|
+
};
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
/**
|
|
1279
|
+
* Finalize a staged park AFTER the child exited.
|
|
1280
|
+
*
|
|
1281
|
+
* Background requires three facts before a generation may leave `parking`:
|
|
1282
|
+
* the checkpoint was recorded (staged by the bridge while the child was alive),
|
|
1283
|
+
* the writer lease was released, and the original writer process is gone. This is
|
|
1284
|
+
* the only place that sends that proof; a missing or unproven fact holds the
|
|
1285
|
+
* generation instead of claiming a park.
|
|
1286
|
+
*/
|
|
1287
|
+
export async function finalizeWorkerParkAfterExit(input: ParkFinalizationInput): Promise<ParkFinalization> {
|
|
1288
|
+
const meta = readMeta(input.runId);
|
|
1289
|
+
const record = readParkRecord(input.runId);
|
|
1290
|
+
if (!meta || !record) return { outcome: "skipped" };
|
|
1291
|
+
const service = input.service;
|
|
1292
|
+
const hold = (reason: string): ParkFinalization => {
|
|
1293
|
+
updateParkRecord(input.runId, { phase: "held", heldReason: reason });
|
|
1294
|
+
return { outcome: "held", reason };
|
|
1295
|
+
};
|
|
1296
|
+
if (!service) return hold("the Background operation service is unavailable; the park wait cannot be finalized");
|
|
1297
|
+
|
|
1298
|
+
const scope = backgroundScopeForMeta(meta, record);
|
|
1299
|
+
if (!scope) return hold("this run records no conversation workspace for its park wait");
|
|
1300
|
+
|
|
1301
|
+
// A restart has no exit observation. The staged checkpoint plus a proven-dead
|
|
1302
|
+
// writer is the strongest available evidence; an unknown exit code alone is
|
|
1303
|
+
// never treated as success, but it does not strand a finished attempt either.
|
|
1304
|
+
const writerGone = input.writerPidGone ?? !processExists(meta.pid);
|
|
1305
|
+
const cleanExit = isCleanParkExit(meta) || (input.interrupted === true && writerGone);
|
|
1306
|
+
const disposition = evaluateParkDisposition({ runId: input.runId, cleanExit });
|
|
1307
|
+
if (!disposition.parked) {
|
|
1308
|
+
const reason = disposition.reason ?? "the attempt did not park cleanly";
|
|
1309
|
+
try {
|
|
1310
|
+
const view = service.failWorkerPark(scope, { waitId: record.waitId, generation: record.generation, reason });
|
|
1311
|
+
const phase = phaseOf(view.phase);
|
|
1312
|
+
updateParkRecord(input.runId, { phase, heldReason: phase === "cancelled" ? reason : undefined });
|
|
1313
|
+
return { outcome: phase === "cancelled" ? "cancelled" : "held", reason, wait: view };
|
|
1314
|
+
} catch (error) {
|
|
1315
|
+
return hold(`${reason}; releasing the wait failed: ${reasonOf(error)}`);
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
const proof = {
|
|
1320
|
+
checkpointRecorded: record.checkpoint !== undefined,
|
|
1321
|
+
writerLeaseReleased: input.writerLeaseReleased,
|
|
1322
|
+
writerPidGone: input.writerPidGone ?? !processExists(meta.pid),
|
|
1323
|
+
};
|
|
1324
|
+
if (!proof.checkpointRecorded || !proof.writerLeaseReleased || !proof.writerPidGone) {
|
|
1325
|
+
const missing = [
|
|
1326
|
+
proof.checkpointRecorded ? undefined : "the checkpoint was not recorded",
|
|
1327
|
+
proof.writerLeaseReleased ? undefined : "the writer lease was not released",
|
|
1328
|
+
proof.writerPidGone ? undefined : "the original writer process is still alive",
|
|
1329
|
+
].filter(Boolean).join("; ");
|
|
1330
|
+
return hold(`the parked wait needs proven exit: ${missing}`);
|
|
1331
|
+
}
|
|
1332
|
+
try {
|
|
1333
|
+
const view = await service.checkpointWorkerWait(scope, {
|
|
1334
|
+
waitId: record.waitId,
|
|
1335
|
+
generation: record.generation,
|
|
1336
|
+
proof,
|
|
1337
|
+
});
|
|
1338
|
+
updateParkRecord(input.runId, {
|
|
1339
|
+
phase: phaseOf(view.phase),
|
|
1340
|
+
heldReason: view.phase === "held" ? "the Background host held the wait after checkout" : undefined,
|
|
1341
|
+
});
|
|
1342
|
+
return { outcome: "parked", wait: view, proof };
|
|
1343
|
+
} catch (error) {
|
|
1344
|
+
return hold(`the Background host refused the park checkpoint: ${reasonOf(error)}`);
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
|
|
1348
|
+
// ---- resume orchestration ------------------------------------------------
|
|
1349
|
+
|
|
1350
|
+
export interface WaitOwnerBinding {
|
|
1351
|
+
sessionId: string;
|
|
1352
|
+
workspace: string;
|
|
1353
|
+
branchId: string | null;
|
|
1354
|
+
goalId: string | null;
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
/** Read-only mirror of Goal's public session-entry shape (no Goal import). */
|
|
1358
|
+
const GOAL_SESSION_ENTRY = "pi-better-goal";
|
|
1359
|
+
const LEGACY_GOAL_SESSION_ENTRY = "pi-codex-goal";
|
|
1360
|
+
|
|
1361
|
+
/**
|
|
1362
|
+
* Current Goal binding for a foreground context, or null when no goal is
|
|
1363
|
+
* recorded. Only the id is read: status, holds and workflow gates stay Goal's
|
|
1364
|
+
* decision, answered through the admission event. An unreadable branch yields
|
|
1365
|
+
* `null`, so admission refuses instead of resuming against a guessed goal.
|
|
1366
|
+
*/
|
|
1367
|
+
export function readGoalBinding(ctx: ExtensionContext | undefined): { goalId: string | null; branchId: string | null } {
|
|
1368
|
+
if (!ctx) return { goalId: null, branchId: null };
|
|
1369
|
+
try {
|
|
1370
|
+
const branch = ctx.sessionManager.getBranch() as Array<{ id?: unknown; type?: unknown; customType?: unknown; data?: unknown }>;
|
|
1371
|
+
let goalId: string | null = null;
|
|
1372
|
+
for (const entry of branch) {
|
|
1373
|
+
if (entry?.type !== "custom") continue;
|
|
1374
|
+
if (entry.customType !== GOAL_SESSION_ENTRY && entry.customType !== LEGACY_GOAL_SESSION_ENTRY) continue;
|
|
1375
|
+
const data = entry.data as { kind?: unknown; goal?: unknown } | undefined;
|
|
1376
|
+
if (!data || typeof data !== "object") continue;
|
|
1377
|
+
if (data.kind === "clear") goalId = null;
|
|
1378
|
+
if (data.kind === "set") {
|
|
1379
|
+
const goal = data.goal as { goalId?: unknown } | undefined;
|
|
1380
|
+
goalId = typeof goal?.goalId === "string" ? goal.goalId : goalId;
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
const last = branch.at(-1);
|
|
1384
|
+
return { goalId, branchId: typeof last?.id === "string" ? last.id : null };
|
|
1385
|
+
} catch {
|
|
1386
|
+
return { goalId: null, branchId: null };
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
/** True when the recorded branch anchor is still on the current branch. */
|
|
1391
|
+
export function branchContains(ctx: ExtensionContext, anchor: string): boolean {
|
|
1392
|
+
try {
|
|
1393
|
+
return (ctx.sessionManager.getBranch() as Array<{ id?: unknown }>).some((entry) => entry.id === anchor);
|
|
1394
|
+
} catch {
|
|
1395
|
+
return false;
|
|
1396
|
+
}
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
/** Owner binding recorded at launch for a foreground context. */
|
|
1400
|
+
export function ownerBindingFromContext(ctx: ExtensionContext): WaitOwnerBinding {
|
|
1401
|
+
const { goalId, branchId } = readGoalBinding(ctx);
|
|
1402
|
+
let sessionId: string;
|
|
1403
|
+
try {
|
|
1404
|
+
sessionId = ctx.sessionManager.getSessionId();
|
|
1405
|
+
} catch {
|
|
1406
|
+
sessionId = "";
|
|
1407
|
+
}
|
|
1408
|
+
return { sessionId, workspace: ctx.cwd, branchId, goalId };
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
export interface ResumeAdmissionRequest {
|
|
1412
|
+
version: 1;
|
|
1413
|
+
owner: { sessionId: string; workspace: string; branchId?: string | null; goalId?: string | null };
|
|
1414
|
+
reply(gate: { allowed: boolean; reason?: string; revision: number; isCurrent?: () => boolean }): void;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
1417
|
+
export type ResumeAdmission =
|
|
1418
|
+
| { allowed: true; revision: number; isCurrent?: () => boolean }
|
|
1419
|
+
| { allowed: false; reason: string; revision: number };
|
|
1420
|
+
|
|
1421
|
+
/**
|
|
1422
|
+
* Ask Goal whether this owner may resume now. The owner binding is the one
|
|
1423
|
+
* CAPTURED when the worker parked, so an active Goal is matched by id instead of
|
|
1424
|
+
* by whatever goal happens to be current; a no-goal session records `null`, which
|
|
1425
|
+
* is exactly what Goal compares against when no goal exists.
|
|
1426
|
+
*/
|
|
1427
|
+
export async function requestResumeAdmission(
|
|
1428
|
+
pi: unknown,
|
|
1429
|
+
owner: WaitOwnerBinding,
|
|
1430
|
+
options: { timeoutMs?: number } = {},
|
|
1431
|
+
): Promise<ResumeAdmission> {
|
|
1432
|
+
const bus = serviceBusOf(pi);
|
|
1433
|
+
if (!bus) {
|
|
1434
|
+
return { allowed: false, reason: "No Goal admission channel is available; a resume needs an active owning parent.", revision: -1 };
|
|
1435
|
+
}
|
|
1436
|
+
const timeoutMs = Math.max(50, Math.min(60_000, options.timeoutMs ?? 10_000));
|
|
1437
|
+
const answer = await new Promise<ResumeAdmission | null>((resolve) => {
|
|
1438
|
+
let settled = false;
|
|
1439
|
+
const finish = (value: ResumeAdmission | null): void => {
|
|
1440
|
+
if (settled) return;
|
|
1441
|
+
settled = true;
|
|
1442
|
+
clearTimeout(timer);
|
|
1443
|
+
resolve(value);
|
|
1444
|
+
};
|
|
1445
|
+
const timer = setTimeout(() => finish(null), timeoutMs);
|
|
1446
|
+
timer.unref?.();
|
|
1447
|
+
const request: ResumeAdmissionRequest = {
|
|
1448
|
+
version: 1,
|
|
1449
|
+
owner: {
|
|
1450
|
+
sessionId: owner.sessionId,
|
|
1451
|
+
workspace: owner.workspace,
|
|
1452
|
+
branchId: owner.branchId,
|
|
1453
|
+
goalId: owner.goalId,
|
|
1454
|
+
},
|
|
1455
|
+
reply: (gate) => {
|
|
1456
|
+
if (!gate || typeof gate.allowed !== "boolean") {
|
|
1457
|
+
finish({ allowed: false, reason: "Goal returned a malformed resume admission", revision: -1 });
|
|
1458
|
+
return;
|
|
1459
|
+
}
|
|
1460
|
+
finish(gate.allowed
|
|
1461
|
+
? { allowed: true, revision: typeof gate.revision === "number" ? gate.revision : 0,
|
|
1462
|
+
...(typeof gate.isCurrent === "function" ? { isCurrent: gate.isCurrent } : {}) }
|
|
1463
|
+
: {
|
|
1464
|
+
allowed: false,
|
|
1465
|
+
reason: typeof gate.reason === "string" && gate.reason ? gate.reason : "Goal refused the resume",
|
|
1466
|
+
revision: typeof gate.revision === "number" ? gate.revision : 0,
|
|
1467
|
+
});
|
|
1468
|
+
},
|
|
1469
|
+
};
|
|
1470
|
+
try {
|
|
1471
|
+
bus.emit(OPERATION_RESUME_ADMISSION_EVENT, request);
|
|
1472
|
+
} catch {
|
|
1473
|
+
finish({ allowed: false, reason: "The Goal admission channel failed", revision: -1 });
|
|
1474
|
+
}
|
|
1475
|
+
});
|
|
1476
|
+
if (answer) return answer;
|
|
1477
|
+
return {
|
|
1478
|
+
allowed: false,
|
|
1479
|
+
reason: "No Goal admission answer arrived; a resume needs an active owning parent and Goal gate.",
|
|
1480
|
+
revision: -1,
|
|
1481
|
+
};
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
/** A parked generation this process may resume. */
|
|
1485
|
+
export interface ParkedWait {
|
|
1486
|
+
runId: string;
|
|
1487
|
+
conversationId: string;
|
|
1488
|
+
attemptId: string;
|
|
1489
|
+
waitId: string;
|
|
1490
|
+
generation: number;
|
|
1491
|
+
phase: ParkRecord["phase"];
|
|
1492
|
+
meta: RunMeta;
|
|
1493
|
+
record: ParkRecord;
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
function parkedEntry(meta: RunMeta, record: ParkRecord): ParkedWait {
|
|
1497
|
+
return {
|
|
1498
|
+
runId: meta.id,
|
|
1499
|
+
conversationId: meta.sessionId,
|
|
1500
|
+
attemptId: meta.id,
|
|
1501
|
+
waitId: record.waitId,
|
|
1502
|
+
generation: record.generation,
|
|
1503
|
+
phase: record.phase,
|
|
1504
|
+
meta,
|
|
1505
|
+
record,
|
|
1506
|
+
};
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
/** Parked runs whose wait can still resume (including held ones). */
|
|
1510
|
+
export function parkedWaits(): ParkedWait[] {
|
|
1511
|
+
const parked: ParkedWait[] = [];
|
|
1512
|
+
for (const meta of listPendingParkedRuns()) {
|
|
1513
|
+
const record = readParkRecord(meta.id);
|
|
1514
|
+
if (!record || !isPendingPark(record)) continue;
|
|
1515
|
+
parked.push(parkedEntry(meta, record));
|
|
1516
|
+
}
|
|
1517
|
+
return parked;
|
|
1518
|
+
}
|
|
1519
|
+
|
|
1520
|
+
export function parkedWaitForRun(runId: string): ParkedWait | null {
|
|
1521
|
+
return parkedWaits().find((entry) => entry.runId === runId) ?? null;
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
/** A parked attempt's record for one wait generation, in any phase. */
|
|
1525
|
+
export function parkedWaitByGeneration(waitId: string, generation: number): ParkedWait | null {
|
|
1526
|
+
for (const meta of listMetas()) {
|
|
1527
|
+
const record = readParkRecord(meta.id);
|
|
1528
|
+
if (!record || record.waitId !== waitId || record.generation !== generation) continue;
|
|
1529
|
+
return parkedEntry(meta, record);
|
|
1530
|
+
}
|
|
1531
|
+
return null;
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
/** Refusal text for a manual continuation of a conversation parked on a wait. */
|
|
1535
|
+
export function manualResumeRefusal(parked: ParkedWait): string {
|
|
1536
|
+
return `Cannot continue ${parked.runId} manually: it is parked on wait ${parked.waitId} generation ${parked.generation} (${parked.phase}) and will resume automatically when its dependency settles. ` +
|
|
1537
|
+
`To take over, run subagent_stop ${parked.runId} first; that cancels the pending automatic resume.`;
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
/** Capacity admission injected by the launcher (the ordinary Subagents gate). */
|
|
1541
|
+
export interface ResumeCapacity {
|
|
1542
|
+
tryReserve(): boolean;
|
|
1543
|
+
/** Conservative recheck after reserving, for a cap reduced in the meantime. */
|
|
1544
|
+
recheck(): boolean;
|
|
1545
|
+
commit(): void;
|
|
1546
|
+
release(): void;
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
export interface ResumeLaunchInput {
|
|
1550
|
+
park: ParkedWait;
|
|
1551
|
+
/** Attempt id and token published durably before the child is spawned. */
|
|
1552
|
+
assignedRunId: string;
|
|
1553
|
+
launchToken: string;
|
|
1554
|
+
/** Durable claim identity the publication must still match at dispatch. */
|
|
1555
|
+
claim: { claimId: string; claimToken: string; cancelEpoch: number; attemptId: string; launchToken: string };
|
|
1556
|
+
/** Goal admission revision the owner was admitted at. */
|
|
1557
|
+
admissionRevision: number;
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
/**
|
|
1561
|
+
* Stop one attempt this process owns. Implemented by the host with the ordinary
|
|
1562
|
+
* `subagent_stop` path; it must verify ownership itself and must never signal a
|
|
1563
|
+
* pid it cannot attribute (return `"foreign"` instead).
|
|
1564
|
+
*/
|
|
1565
|
+
export type StopPublishedAttempt = (input: {
|
|
1566
|
+
attemptId: string;
|
|
1567
|
+
/** Token persisted with the attempt before spawn; identifies the publication. */
|
|
1568
|
+
launchToken?: string;
|
|
1569
|
+
waitId: string;
|
|
1570
|
+
generation: number;
|
|
1571
|
+
reason: string;
|
|
1572
|
+
}) => Promise<"aborted" | "stopped" | "foreign"> | "aborted" | "stopped" | "foreign";
|
|
1573
|
+
|
|
1574
|
+
/**
|
|
1575
|
+
* Default owner-checked stopper: only a child this parent spawned, whose recorded
|
|
1576
|
+
* publication token matches, may be stopped. Anything else is reported as
|
|
1577
|
+
* `foreign` so no pid is signalled on a guess.
|
|
1578
|
+
*/
|
|
1579
|
+
export function createOwnedAttemptStopper(options: {
|
|
1580
|
+
parentPid?: number;
|
|
1581
|
+
/** Replacement stop implementation (tests, or a host with different semantics). */
|
|
1582
|
+
stop?: (attemptId: string) => { action: string };
|
|
1583
|
+
} = {}): StopPublishedAttempt {
|
|
1584
|
+
const parentPid = options.parentPid ?? process.pid;
|
|
1585
|
+
const stop = options.stop ?? ((attemptId: string) => stopRun(attemptId));
|
|
1586
|
+
return (input) => {
|
|
1587
|
+
const meta = readMeta(input.attemptId);
|
|
1588
|
+
if (!meta) return "aborted";
|
|
1589
|
+
if (meta.spawnPid !== parentPid) return "foreign";
|
|
1590
|
+
if (input.launchToken && meta.operations?.resumeLaunchToken !== input.launchToken) return "foreign";
|
|
1591
|
+
const outcome = stop(input.attemptId);
|
|
1592
|
+
return outcome.action === "stopped" || outcome.action === "finalized" ? "stopped" : "aborted";
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
export interface ResumeOrchestrationOptions {
|
|
1597
|
+
pi: unknown;
|
|
1598
|
+
service: () => WorkerOperationService | undefined;
|
|
1599
|
+
rpc: WorkerOperationsRpc;
|
|
1600
|
+
context(): ExtensionContext | undefined;
|
|
1601
|
+
capacity: ResumeCapacity;
|
|
1602
|
+
/** Launch one continuation through the existing admission path. */
|
|
1603
|
+
launch(input: ResumeLaunchInput): Promise<void>;
|
|
1604
|
+
/** Follow a cancelled generation to the attempt it already published. */
|
|
1605
|
+
stopAttempt?: StopPublishedAttempt;
|
|
1606
|
+
notify?(message: string, level: "info" | "warning"): void;
|
|
1607
|
+
now?: () => number;
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
export interface ResumeOrchestration {
|
|
1611
|
+
/** Try to resume one parked generation. */
|
|
1612
|
+
resume(park: ParkedWait): Promise<"resumed" | "held" | "skipped">;
|
|
1613
|
+
/** Ready/cancelled delivery adapter to register with the Background host. */
|
|
1614
|
+
readonly adapter: WorkerContinuationAdapter;
|
|
1615
|
+
stop(): void;
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1618
|
+
/**
|
|
1619
|
+
* A continuation refused before any child could exist (planning, capacity,
|
|
1620
|
+
* current permission admission, writer lease). The launcher throws this only for
|
|
1621
|
+
* a proven pre-spawn failure; anything else is ambiguous and is held.
|
|
1622
|
+
*/
|
|
1623
|
+
export class PreSpawnResumeError extends Error {
|
|
1624
|
+
constructor(message: string) {
|
|
1625
|
+
super(message);
|
|
1626
|
+
this.name = "PreSpawnResumeError";
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
/**
|
|
1631
|
+
* Release one child: the synchronous dispatch guard runs with no await (and no
|
|
1632
|
+
* other code) between it and the spawn, so nothing can slip in after the guard.
|
|
1633
|
+
*/
|
|
1634
|
+
export function releaseChild<T>(guard: DispatchGuard | undefined | void, spawn: () => T): T {
|
|
1635
|
+
if (typeof guard === "function") guard();
|
|
1636
|
+
return spawn();
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
/** A proven pre-spawn refusal from the final dispatch fence. */
|
|
1640
|
+
export class FinalDispatchRefusal extends PreSpawnResumeError {
|
|
1641
|
+
constructor(message: string) {
|
|
1642
|
+
super(message);
|
|
1643
|
+
this.name = "FinalDispatchRefusal";
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
/** Synchronous guard invoked immediately before the child is released. */
|
|
1648
|
+
export type DispatchGuard = () => void;
|
|
1649
|
+
|
|
1650
|
+
export interface FinalDispatchInput {
|
|
1651
|
+
pi: unknown;
|
|
1652
|
+
rpc: WorkerOperationsRpc;
|
|
1653
|
+
/** The object service, read synchronously for the final durable confirmation. */
|
|
1654
|
+
service: () => WorkerOperationService | undefined;
|
|
1655
|
+
/** Live foreground context getter: never a value captured before the awaits. */
|
|
1656
|
+
context: () => ExtensionContext | undefined;
|
|
1657
|
+
park: ParkedWait;
|
|
1658
|
+
claim: { claimId: string; claimToken: string; cancelEpoch: number; attemptId: string; launchToken: string };
|
|
1659
|
+
/** Foreground owner binding captured when the worker parked. */
|
|
1660
|
+
owner: WaitOwnerBinding;
|
|
1661
|
+
/** Goal admission revision the resume was admitted at. */
|
|
1662
|
+
admissionRevision: number;
|
|
1663
|
+
/** Fresh permission resolution compared with the captured one. */
|
|
1664
|
+
permissionsUnchanged(): boolean;
|
|
1665
|
+
/** Ordinary capacity gate recheck. */
|
|
1666
|
+
capacityAvailable(): boolean;
|
|
1667
|
+
admissionTimeoutMs?: number;
|
|
1668
|
+
/** Extra host-specific synchronous check (for example a generation counter). */
|
|
1669
|
+
extraCheck?: () => void;
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/**
|
|
1673
|
+
* Final dispatch fence.
|
|
1674
|
+
*
|
|
1675
|
+
* The launch path awaits extension resolution, permission re-resolution, the
|
|
1676
|
+
* bridge registration and the writer lease. Any of those can take long enough for
|
|
1677
|
+
* the parent to switch branch, pause Goal, tighten permissions, cancel the wait or
|
|
1678
|
+
* shrink capacity. Immediately before a child may start, this re-reads the durable
|
|
1679
|
+
* claim, the cancellation epoch and the Goal gate, and re-checks the synchronous
|
|
1680
|
+
* parent identity and the captured policy. A refusal is a proven pre-spawn
|
|
1681
|
+
* failure: nothing was executed, and the caller releases the claim/capacity.
|
|
1682
|
+
*/
|
|
1683
|
+
export async function assertFinalResumeDispatch(input: FinalDispatchInput): Promise<DispatchGuard> {
|
|
1684
|
+
const refusal = (message: string): never => {
|
|
1685
|
+
throw new FinalDispatchRefusal(message);
|
|
1686
|
+
};
|
|
1687
|
+
const identityNow = (): WaitOwnerBinding => {
|
|
1688
|
+
const ctx = input.context();
|
|
1689
|
+
if (!ctx) refusal("The owning foreground session is no longer active");
|
|
1690
|
+
return ownerBindingFromContext(ctx as ExtensionContext);
|
|
1691
|
+
};
|
|
1692
|
+
const assertIdentity = (phase: string): void => {
|
|
1693
|
+
const ctx = input.context();
|
|
1694
|
+
if (!ctx) refusal("The owning foreground session is no longer active");
|
|
1695
|
+
const current = identityNow();
|
|
1696
|
+
if (current.sessionId !== input.owner.sessionId) {
|
|
1697
|
+
refusal(`The owning parent session changed during ${phase}; the resume was not executed`);
|
|
1698
|
+
}
|
|
1699
|
+
if (canonical(current.workspace) !== canonical(input.owner.workspace)) {
|
|
1700
|
+
refusal(`The foreground workspace changed during ${phase}; the resume was not executed`);
|
|
1701
|
+
}
|
|
1702
|
+
if (input.owner.branchId && !branchContains(ctx as ExtensionContext, input.owner.branchId)) {
|
|
1703
|
+
refusal(`The owning branch is no longer active during ${phase}; the resume was not executed`);
|
|
1704
|
+
}
|
|
1705
|
+
if ((current.goalId ?? null) !== (input.owner.goalId ?? null)) {
|
|
1706
|
+
refusal(`The owning Goal binding changed during ${phase}; the resume was not executed`);
|
|
1707
|
+
}
|
|
1708
|
+
};
|
|
1709
|
+
/**
|
|
1710
|
+
* The only durable confirmation that matters at release time.
|
|
1711
|
+
*
|
|
1712
|
+
* `listWorkerWaits` is synchronous, so it runs after every await and directly
|
|
1713
|
+
* before the spawn with no window for cancellation, a claim change or a parent
|
|
1714
|
+
* switch to hide inside. The store enforces one active claim per generation, and
|
|
1715
|
+
* the async part above already replayed `publish-resume` with the exact token,
|
|
1716
|
+
* so phase + cancellation epoch identify this publication.
|
|
1717
|
+
*/
|
|
1718
|
+
const durableGuard = (): void => {
|
|
1719
|
+
const service = input.service();
|
|
1720
|
+
if (!service) refusal("The Background operation service is unavailable at dispatch; the continuation was not started");
|
|
1721
|
+
const scope = backgroundScopeForMeta(input.park.meta, input.park.record);
|
|
1722
|
+
if (!scope) refusal("The parked attempt records no worker conversation scope; the continuation was not started");
|
|
1723
|
+
let view: WorkerWaitView | undefined;
|
|
1724
|
+
try {
|
|
1725
|
+
view = (service as WorkerOperationService).listWorkerWaits(scope!, { pendingOnly: true })
|
|
1726
|
+
.find((entry) => entry.waitId === input.park.waitId && entry.generation === input.park.generation);
|
|
1727
|
+
} catch (error) {
|
|
1728
|
+
refusal(`The durable wait could not be read at dispatch: ${reasonOf(error)}`);
|
|
1729
|
+
}
|
|
1730
|
+
if (!view) {
|
|
1731
|
+
refusal("The durable wait is gone at dispatch (consumed or cancelled); the continuation was not started");
|
|
1732
|
+
}
|
|
1733
|
+
if (view!.phase !== "resuming") {
|
|
1734
|
+
refusal(`The durable wait is ${view!.phase} at dispatch, not resuming; the continuation was not started`);
|
|
1735
|
+
}
|
|
1736
|
+
if (view!.cancelEpoch !== input.claim.cancelEpoch) {
|
|
1737
|
+
refusal("The wait was cancelled or replaced at dispatch; the continuation was not started");
|
|
1738
|
+
}
|
|
1739
|
+
};
|
|
1740
|
+
let goalProof: (() => boolean) | undefined;
|
|
1741
|
+
const guard = (): void => {
|
|
1742
|
+
if (goalProof && !goalProof()) refusal("Goal admission changed before execution release; the continuation was not started");
|
|
1743
|
+
assertIdentity("dispatch");
|
|
1744
|
+
if (!input.permissionsUnchanged()) {
|
|
1745
|
+
refusal("The worker's permissions changed before dispatch; the continuation was not started");
|
|
1746
|
+
}
|
|
1747
|
+
if (!input.capacityAvailable()) {
|
|
1748
|
+
refusal("Capacity changed before dispatch; the continuation was not started");
|
|
1749
|
+
}
|
|
1750
|
+
durableGuard();
|
|
1751
|
+
input.extraCheck?.();
|
|
1752
|
+
// Identity can only change synchronously through this process, so one final
|
|
1753
|
+
// read after the durable check closes the guard.
|
|
1754
|
+
assertIdentity("release");
|
|
1755
|
+
};
|
|
1756
|
+
|
|
1757
|
+
// Synchronous identity and captured policy first: cheap and fail-closed.
|
|
1758
|
+
assertIdentity("preparation");
|
|
1759
|
+
if (!input.permissionsUnchanged()) {
|
|
1760
|
+
refusal("The worker's permissions changed while the resume was being prepared; the continuation was not started");
|
|
1761
|
+
}
|
|
1762
|
+
if (!input.capacityAvailable()) {
|
|
1763
|
+
refusal("Capacity changed while the resume was being prepared; the continuation was not started");
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
// Durable claim, phase and cancellation epoch (async pre-check).
|
|
1767
|
+
const snapshot = await input.rpc.call("wait-snapshot", {
|
|
1768
|
+
waitId: input.park.waitId,
|
|
1769
|
+
generation: input.park.generation,
|
|
1770
|
+
});
|
|
1771
|
+
if (snapshot.phase !== "resuming") {
|
|
1772
|
+
refusal(`The durable wait is ${String(snapshot.phase)}, not resuming; the continuation was not started`);
|
|
1773
|
+
}
|
|
1774
|
+
if (Number(snapshot.cancelEpoch ?? -1) !== input.claim.cancelEpoch) {
|
|
1775
|
+
refusal("The wait was cancelled or replaced while the resume was being prepared; the continuation was not started");
|
|
1776
|
+
}
|
|
1777
|
+
const active = snapshot.claim as { state?: unknown; attemptId?: unknown; claimId?: unknown } | null | undefined;
|
|
1778
|
+
if (!active || active.state !== "published" || active.attemptId !== input.claim.attemptId) {
|
|
1779
|
+
refusal("The published resume claim no longer names this attempt; the continuation was not started");
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
// Exact launch token: replaying the publication is idempotent only for the
|
|
1783
|
+
// same (claim, token, attempt, launch token) tuple.
|
|
1784
|
+
await input.rpc.call("publish-resume", {
|
|
1785
|
+
runId: input.park.runId,
|
|
1786
|
+
waitId: input.park.waitId,
|
|
1787
|
+
generation: input.park.generation,
|
|
1788
|
+
claimId: input.claim.claimId,
|
|
1789
|
+
claimToken: input.claim.claimToken,
|
|
1790
|
+
attemptId: input.claim.attemptId,
|
|
1791
|
+
launchToken: input.claim.launchToken,
|
|
1792
|
+
});
|
|
1793
|
+
|
|
1794
|
+
// Goal gate last, because it is the only asynchronous authority check. Its
|
|
1795
|
+
// revision must not move during the fence, and the durable confirmation is
|
|
1796
|
+
// repeated synchronously afterwards: a cancellation performed inside the Goal
|
|
1797
|
+
// handler is caught here, not accepted on an earlier read.
|
|
1798
|
+
const admission = await requestResumeAdmission(input.pi, input.owner, {
|
|
1799
|
+
...(input.admissionTimeoutMs !== undefined ? { timeoutMs: input.admissionTimeoutMs } : {}),
|
|
1800
|
+
});
|
|
1801
|
+
if (!admission.allowed) {
|
|
1802
|
+
refusal(`Goal refused the resume at dispatch: ${admission.reason}`);
|
|
1803
|
+
}
|
|
1804
|
+
if (admission.revision !== input.admissionRevision) {
|
|
1805
|
+
refusal(`Goal admission advanced from revision ${input.admissionRevision} to ${admission.revision} during preparation; the continuation was not started`);
|
|
1806
|
+
}
|
|
1807
|
+
goalProof = (admission as { isCurrent?: () => boolean }).isCurrent;
|
|
1808
|
+
// Validate the synchronous guard once here (so the returned guard is known to
|
|
1809
|
+
// hold after the last await) and hand it back for the release point.
|
|
1810
|
+
guard();
|
|
1811
|
+
return guard;
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
/**
|
|
1815
|
+
* One resume at a time, through the existing continuation launch path.
|
|
1816
|
+
*
|
|
1817
|
+
* Order is fixed: capacity is reserved with the ordinary gate, the owner is
|
|
1818
|
+
* admitted (captured parent session/branch binding + Goal), the durable wait is
|
|
1819
|
+
* re-read, the attempt id/token are written to the park record, and only then is
|
|
1820
|
+
* a child spawned. A full capacity defers delivery without touching the wait, so
|
|
1821
|
+
* a queued resume never becomes "held". A proven pre-spawn failure holds the wait
|
|
1822
|
+
* with its reason; an ambiguous failure holds without relaunching.
|
|
1823
|
+
*/
|
|
1824
|
+
export function createResumeOrchestration(options: ResumeOrchestrationOptions): ResumeOrchestration {
|
|
1825
|
+
const inFlight = new Set<string>();
|
|
1826
|
+
const now = options.now ?? (() => Date.now());
|
|
1827
|
+
|
|
1828
|
+
const hold = (park: ParkedWait, reason: string): "held" => {
|
|
1829
|
+
updateParkRecord(park.runId, { phase: "held", heldReason: reason });
|
|
1830
|
+
try {
|
|
1831
|
+
options.service()?.failWorkerPark(backgroundScopeForMeta(park.meta, park.record) ?? {
|
|
1832
|
+
runId: park.runId,
|
|
1833
|
+
conversationId: park.conversationId,
|
|
1834
|
+
workspace: park.meta.operations?.workspace ?? park.meta.cwd,
|
|
1835
|
+
authorityReference: park.meta.operations?.authorityReference ?? `worker:${park.runId}`,
|
|
1836
|
+
}, { waitId: park.waitId, generation: park.generation, reason });
|
|
1837
|
+
} catch { /* the park record still shows the hold */ }
|
|
1838
|
+
try {
|
|
1839
|
+
options.notify?.(`Subagent ${park.runId} wait ${park.waitId} is held: ${reason}`, "warning");
|
|
1840
|
+
} catch { /* ignore UI failures */ }
|
|
1841
|
+
return "held";
|
|
1842
|
+
};
|
|
1843
|
+
|
|
1844
|
+
const resume = async (park: ParkedWait): Promise<"resumed" | "held" | "skipped"> => {
|
|
1845
|
+
const key = `${park.waitId}:${park.generation}`;
|
|
1846
|
+
if (inFlight.has(key)) return "skipped";
|
|
1847
|
+
inFlight.add(key);
|
|
1848
|
+
let reserved = false;
|
|
1849
|
+
try {
|
|
1850
|
+
// The durable wait is the authority for readiness (checked below). A
|
|
1851
|
+
// record that is in flight or already held must never start another
|
|
1852
|
+
// automatic attempt: `resuming` has a published attempt, and `held`
|
|
1853
|
+
// waits for its owner's interpretation.
|
|
1854
|
+
if (park.record.phase === "resuming" || park.record.phase === "held") return "skipped";
|
|
1855
|
+
const ctx = options.context();
|
|
1856
|
+
if (!ctx) return "skipped";
|
|
1857
|
+
const current = ownerBindingFromContext(ctx);
|
|
1858
|
+
if (park.record.ownerSessionId && current.sessionId !== park.record.ownerSessionId) return "skipped";
|
|
1859
|
+
if (park.record.ownerWorkspace && canonical(park.record.ownerWorkspace) !== canonical(current.workspace)) return "skipped";
|
|
1860
|
+
if (park.record.ownerBranchId && !branchContains(ctx, park.record.ownerBranchId)) return "skipped";
|
|
1861
|
+
const prior = readMeta(park.runId);
|
|
1862
|
+
if (prior && prior.status === "running" && processExists(prior.pid)) return "skipped";
|
|
1863
|
+
|
|
1864
|
+
const service = options.service();
|
|
1865
|
+
if (!service) return "skipped";
|
|
1866
|
+
const scope = backgroundScopeForMeta(park.meta, park.record);
|
|
1867
|
+
if (!scope) return "skipped";
|
|
1868
|
+
// Capacity FIRST: a saturated pool leaves the generation ready and the
|
|
1869
|
+
// delivery pending, so a queued resume never becomes "held" and no
|
|
1870
|
+
// unresumable claim is published.
|
|
1871
|
+
if (!options.capacity.tryReserve()) return "skipped";
|
|
1872
|
+
reserved = true;
|
|
1873
|
+
if (!options.capacity.recheck()) {
|
|
1874
|
+
options.capacity.release();
|
|
1875
|
+
reserved = false;
|
|
1876
|
+
return "skipped";
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
// The captured binding is the owner: an active Goal must match by id,
|
|
1880
|
+
// and a no-goal session records null.
|
|
1881
|
+
const owner: WaitOwnerBinding = {
|
|
1882
|
+
sessionId: park.record.ownerSessionId ?? current.sessionId,
|
|
1883
|
+
workspace: park.record.ownerWorkspace ?? current.workspace,
|
|
1884
|
+
branchId: park.record.ownerBranchId ?? current.branchId,
|
|
1885
|
+
goalId: park.record.ownerGoalId ?? null,
|
|
1886
|
+
};
|
|
1887
|
+
const admission = await requestResumeAdmission(options.pi, owner);
|
|
1888
|
+
if (!admission.allowed) {
|
|
1889
|
+
options.capacity.release();
|
|
1890
|
+
reserved = false;
|
|
1891
|
+
return hold(park, admission.reason);
|
|
1892
|
+
}
|
|
1893
|
+
|
|
1894
|
+
// Rehydrate the parked attempt's run binding: the claim fence is
|
|
1895
|
+
// addressed by run id and requires a binding for this conversation.
|
|
1896
|
+
await options.rpc.call("register-run", {
|
|
1897
|
+
runId: park.runId,
|
|
1898
|
+
conversationId: scope.conversationId,
|
|
1899
|
+
attemptId: park.runId,
|
|
1900
|
+
workspace: scope.workspace,
|
|
1901
|
+
authorityReference: scope.authorityReference,
|
|
1902
|
+
policy: {
|
|
1903
|
+
version: 1,
|
|
1904
|
+
sandbox: park.meta.operations?.policy?.sandbox ?? null,
|
|
1905
|
+
tools: park.meta.operations?.policy?.tools ?? [],
|
|
1906
|
+
clean: park.meta.operations?.policy?.clean === true,
|
|
1907
|
+
deadlineAt: park.meta.operations?.policy?.deadlineAt ?? null,
|
|
1908
|
+
authorityReference: scope.authorityReference,
|
|
1909
|
+
workspace: scope.workspace,
|
|
1910
|
+
},
|
|
1911
|
+
tools: park.meta.operations?.tools ?? [],
|
|
1912
|
+
capabilities: {},
|
|
1913
|
+
owner: {
|
|
1914
|
+
kind: "worker-conversation",
|
|
1915
|
+
sessionId: scope.conversationId,
|
|
1916
|
+
workspace: scope.workspace,
|
|
1917
|
+
branchId: owner.branchId,
|
|
1918
|
+
goalId: owner.goalId,
|
|
1919
|
+
},
|
|
1920
|
+
});
|
|
1921
|
+
const snapshot = await options.rpc.call("wait-snapshot", {
|
|
1922
|
+
waitId: park.waitId,
|
|
1923
|
+
generation: park.generation,
|
|
1924
|
+
});
|
|
1925
|
+
if (snapshot.phase !== "ready") {
|
|
1926
|
+
options.capacity.release();
|
|
1927
|
+
reserved = false;
|
|
1928
|
+
if (snapshot.phase === "waiting" || snapshot.phase === "parking" || snapshot.phase === "resuming") {
|
|
1929
|
+
if (snapshot.phase !== park.phase) updateParkRecord(park.runId, { phase: phaseOf(String(snapshot.phase)) });
|
|
1930
|
+
return "skipped";
|
|
1931
|
+
}
|
|
1932
|
+
updateParkRecord(park.runId, { phase: phaseOf(String(snapshot.phase)) });
|
|
1933
|
+
return "skipped";
|
|
1934
|
+
}
|
|
1935
|
+
if (snapshot.claim) {
|
|
1936
|
+
// Another supervisor holds or published this generation.
|
|
1937
|
+
options.capacity.release();
|
|
1938
|
+
reserved = false;
|
|
1939
|
+
return "skipped";
|
|
1940
|
+
}
|
|
1941
|
+
const expectedCancelEpoch = typeof snapshot.cancelEpoch === "number" ? snapshot.cancelEpoch : 0;
|
|
1942
|
+
const claim = await options.rpc.call("claim-resume", {
|
|
1943
|
+
runId: park.runId,
|
|
1944
|
+
waitId: park.waitId,
|
|
1945
|
+
generation: park.generation,
|
|
1946
|
+
expectedCancelEpoch,
|
|
1947
|
+
lease: { ownerId: `subagents:${process.pid}`, pid: process.pid },
|
|
1948
|
+
reservedCheckpointId: park.record.checkpoint?.entryId ?? park.waitId,
|
|
1949
|
+
});
|
|
1950
|
+
const claimId = String(claim.claimId ?? "");
|
|
1951
|
+
const claimToken = String(claim.claimToken ?? "");
|
|
1952
|
+
const assignedRunId = `sa_${now().toString(36)}_${randomUUID().replace(/-/g, "").slice(0, 8)}`;
|
|
1953
|
+
const launchToken = `slt_${randomUUID()}`;
|
|
1954
|
+
if (!claimId || !claimToken) {
|
|
1955
|
+
options.capacity.release();
|
|
1956
|
+
reserved = false;
|
|
1957
|
+
return hold(park, "the Background host returned no usable resume claim");
|
|
1958
|
+
}
|
|
1959
|
+
try {
|
|
1960
|
+
await options.rpc.call("publish-resume", {
|
|
1961
|
+
runId: park.runId,
|
|
1962
|
+
waitId: park.waitId,
|
|
1963
|
+
generation: park.generation,
|
|
1964
|
+
claimId,
|
|
1965
|
+
claimToken,
|
|
1966
|
+
attemptId: assignedRunId,
|
|
1967
|
+
launchToken,
|
|
1968
|
+
});
|
|
1969
|
+
} catch (error) {
|
|
1970
|
+
options.capacity.release();
|
|
1971
|
+
reserved = false;
|
|
1972
|
+
try {
|
|
1973
|
+
await options.rpc.call("release-resume", {
|
|
1974
|
+
runId: park.runId,
|
|
1975
|
+
waitId: park.waitId,
|
|
1976
|
+
generation: park.generation,
|
|
1977
|
+
claimId,
|
|
1978
|
+
claimToken,
|
|
1979
|
+
outcome: "ready",
|
|
1980
|
+
reason: reasonOf(error),
|
|
1981
|
+
});
|
|
1982
|
+
} catch { /* the claim stays for reconciliation */ }
|
|
1983
|
+
return hold(park, reasonOf(error));
|
|
1984
|
+
}
|
|
1985
|
+
// Mirror the published attempt identity into the park record. The store
|
|
1986
|
+
// claim is the cross-process fence; this keeps the local record readable.
|
|
1987
|
+
updateParkRecord(park.runId, {
|
|
1988
|
+
phase: "resuming",
|
|
1989
|
+
resumedAt: now(),
|
|
1990
|
+
resumedBy: assignedRunId,
|
|
1991
|
+
resumeLaunchToken: launchToken,
|
|
1992
|
+
resumeClaimId: claimId,
|
|
1993
|
+
resumeClaimToken: claimToken,
|
|
1994
|
+
heldReason: undefined,
|
|
1995
|
+
});
|
|
1996
|
+
try {
|
|
1997
|
+
await options.launch({
|
|
1998
|
+
park,
|
|
1999
|
+
assignedRunId,
|
|
2000
|
+
launchToken,
|
|
2001
|
+
claim: {
|
|
2002
|
+
claimId,
|
|
2003
|
+
claimToken,
|
|
2004
|
+
cancelEpoch: expectedCancelEpoch,
|
|
2005
|
+
attemptId: assignedRunId,
|
|
2006
|
+
launchToken,
|
|
2007
|
+
},
|
|
2008
|
+
admissionRevision: admission.revision,
|
|
2009
|
+
});
|
|
2010
|
+
} catch (error) {
|
|
2011
|
+
const reason = reasonOf(error);
|
|
2012
|
+
// A published claim cannot be released; the durable record holds it
|
|
2013
|
+
// for recovery. Capacity and admission already succeeded, so this is a
|
|
2014
|
+
// spawn-time failure, not a queue condition.
|
|
2015
|
+
updateParkRecord(park.runId, { phase: "held", heldReason: reason });
|
|
2016
|
+
return "held";
|
|
2017
|
+
}
|
|
2018
|
+
options.capacity.commit();
|
|
2019
|
+
reserved = false;
|
|
2020
|
+
void options.rpc.call("release-run", { runId: park.runId, reason: "resumed into a new attempt" }).catch(() => {});
|
|
2021
|
+
return "resumed";
|
|
2022
|
+
} catch (error) {
|
|
2023
|
+
return hold(park, reasonOf(error));
|
|
2024
|
+
} finally {
|
|
2025
|
+
if (reserved) options.capacity.release();
|
|
2026
|
+
inFlight.delete(key);
|
|
2027
|
+
}
|
|
2028
|
+
};
|
|
2029
|
+
|
|
2030
|
+
const adapter: WorkerContinuationAdapter = {
|
|
2031
|
+
version: 1,
|
|
2032
|
+
id: `subagents-continuation:${randomUUID()}`,
|
|
2033
|
+
async onWaitReady(event) {
|
|
2034
|
+
const park = parkedWaitByGeneration(event.waitId, event.generation);
|
|
2035
|
+
if (!park) return "delivered";
|
|
2036
|
+
const outcome = await resume(park);
|
|
2037
|
+
return outcome === "skipped" ? "deferred" : "delivered";
|
|
2038
|
+
},
|
|
2039
|
+
async onWaitCancelled(event) {
|
|
2040
|
+
const park = parkedWaitByGeneration(event.waitId, event.generation);
|
|
2041
|
+
if (!park || park.record.phase === "cancelled") return;
|
|
2042
|
+
const cancellation = await stopPublishedAttempt(park, event.reason, options.stopAttempt, options.now);
|
|
2043
|
+
updateParkRecord(park.runId, {
|
|
2044
|
+
phase: "cancelled",
|
|
2045
|
+
cancelReason: cancellation === "stopped"
|
|
2046
|
+
? `${event.reason} (the published continuation ${park.record.resumedBy} was stopped)`
|
|
2047
|
+
: cancellation === "foreign"
|
|
2048
|
+
? `${event.reason} (the published continuation could not be attributed to this parent and was left running)`
|
|
2049
|
+
: event.reason,
|
|
2050
|
+
cancelledAt: now(),
|
|
2051
|
+
});
|
|
2052
|
+
},
|
|
2053
|
+
};
|
|
2054
|
+
|
|
2055
|
+
return {
|
|
2056
|
+
resume,
|
|
2057
|
+
adapter,
|
|
2058
|
+
stop() {
|
|
2059
|
+
try { options.service()?.unregisterContinuationAdapter(adapter.id); } catch { /* host already gone */ }
|
|
2060
|
+
},
|
|
2061
|
+
};
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
function reasonOf(error: unknown): string {
|
|
2065
|
+
return error instanceof Error ? error.message : String(error);
|
|
2066
|
+
}
|
|
2067
|
+
|
|
2068
|
+
/**
|
|
2069
|
+
* Cancel a parked assignment's automatic resume. Called by `subagent_stop`.
|
|
2070
|
+
* The durable wait is cancelled first; the run record is marked so nothing
|
|
2071
|
+
* resumes it.
|
|
2072
|
+
*/
|
|
2073
|
+
export async function cancelParkedWait(
|
|
2074
|
+
service: WorkerOperationService | undefined,
|
|
2075
|
+
park: ParkedWait,
|
|
2076
|
+
reason: string,
|
|
2077
|
+
options: { now?: () => number; stopAttempt?: StopPublishedAttempt } = {},
|
|
2078
|
+
): Promise<void> {
|
|
2079
|
+
// Cancel the durable wait first: no new claim can be published after this.
|
|
2080
|
+
// The registered continuation adapter is notified by the host and performs the
|
|
2081
|
+
// same follow-up, so an already-recorded cancellation is left alone.
|
|
2082
|
+
const scope = backgroundScopeForMeta(park.meta, park.record);
|
|
2083
|
+
const at = (options.now ?? Date.now)();
|
|
2084
|
+
if (service && scope) {
|
|
2085
|
+
let view;
|
|
2086
|
+
try {
|
|
2087
|
+
view = service.cancelWorkerWait(scope, { waitId: park.waitId, generation: park.generation, reason });
|
|
2088
|
+
} catch (error) {
|
|
2089
|
+
updateParkRecord(park.runId, {
|
|
2090
|
+
phase: "held",
|
|
2091
|
+
cancelReason: `${reason} (the durable cancellation failed: ${reasonOf(error)})`,
|
|
2092
|
+
cancelledAt: at,
|
|
2093
|
+
});
|
|
2094
|
+
return;
|
|
2095
|
+
}
|
|
2096
|
+
const settled = readParkRecord(park.runId);
|
|
2097
|
+
if (settled?.phase === "cancelled" && settled.cancelReason) return;
|
|
2098
|
+
const cancellation = await stopPublishedAttempt(park, reason, options.stopAttempt, options.now);
|
|
2099
|
+
updateParkRecord(park.runId, {
|
|
2100
|
+
phase: phaseOf(view.phase),
|
|
2101
|
+
cancelReason: cancellation === "stopped"
|
|
2102
|
+
? `${reason} (the published continuation ${park.record.resumedBy} was stopped)`
|
|
2103
|
+
: cancellation === "foreign"
|
|
2104
|
+
? `${reason} (the published continuation could not be attributed to this parent and was left running)`
|
|
2105
|
+
: reason,
|
|
2106
|
+
cancelledAt: at,
|
|
2107
|
+
});
|
|
2108
|
+
return;
|
|
2109
|
+
}
|
|
2110
|
+
const cancellation = await stopPublishedAttempt(park, reason, options.stopAttempt, options.now);
|
|
2111
|
+
updateParkRecord(park.runId, {
|
|
2112
|
+
phase: "cancelled",
|
|
2113
|
+
cancelReason: cancellation === "foreign"
|
|
2114
|
+
? `${reason} (the published continuation could not be attributed to this parent and was left running)`
|
|
2115
|
+
: reason,
|
|
2116
|
+
cancelledAt: at,
|
|
2117
|
+
});
|
|
2118
|
+
}
|
|
2119
|
+
|
|
2120
|
+
/**
|
|
2121
|
+
* Follow a cancelled generation to the attempt it already published.
|
|
2122
|
+
*
|
|
2123
|
+
* A `resuming` record with no registered attempt is a pre-dispatch abort: nothing
|
|
2124
|
+
* was spawned. A record whose attempt exists is stopped only through the injected
|
|
2125
|
+
* owner-checked seam; an unattributable identity is reported as `foreign` and left
|
|
2126
|
+
* alone, never signalled by pid guesswork.
|
|
2127
|
+
*/
|
|
2128
|
+
async function stopPublishedAttempt(
|
|
2129
|
+
park: ParkedWait,
|
|
2130
|
+
reason: string,
|
|
2131
|
+
stopAttempt: StopPublishedAttempt | undefined,
|
|
2132
|
+
now: (() => number) | undefined,
|
|
2133
|
+
): Promise<"aborted" | "stopped" | "foreign" | "none"> {
|
|
2134
|
+
const attemptId = park.record.resumedBy;
|
|
2135
|
+
if (!attemptId) return "none";
|
|
2136
|
+
const meta = readMeta(attemptId);
|
|
2137
|
+
if (!meta) {
|
|
2138
|
+
updateParkRecord(park.runId, { phase: "cancelled", cancelReason: reason, cancelledAt: (now ?? Date.now)() });
|
|
2139
|
+
return "aborted";
|
|
2140
|
+
}
|
|
2141
|
+
if (!stopAttempt) return "foreign";
|
|
2142
|
+
try {
|
|
2143
|
+
return await stopAttempt({
|
|
2144
|
+
attemptId,
|
|
2145
|
+
launchToken: park.record.resumeLaunchToken,
|
|
2146
|
+
waitId: park.waitId,
|
|
2147
|
+
generation: park.generation,
|
|
2148
|
+
reason,
|
|
2149
|
+
});
|
|
2150
|
+
} catch {
|
|
2151
|
+
return "foreign";
|
|
2152
|
+
}
|
|
2153
|
+
}
|
|
2154
|
+
|
|
2155
|
+
/**
|
|
2156
|
+
* Reconcile parked generations at session start.
|
|
2157
|
+
*
|
|
2158
|
+
* A `resuming` record with no attempt metadata is the crash window between the
|
|
2159
|
+
* durable pre-spawn write and the spawn: it is held, never relaunched. A ready
|
|
2160
|
+
* generation whose delivery was missed (inactive owner, host restart) is resumed
|
|
2161
|
+
* here. Everything else mirrors the durable phase.
|
|
2162
|
+
*/
|
|
2163
|
+
/**
|
|
2164
|
+
* Retry a staged park after a restart.
|
|
2165
|
+
*
|
|
2166
|
+
* Evidence required: the staged checkpoint still verifies, the recorded writer
|
|
2167
|
+
* process is gone, and no live conversation writer lease remains. A live writer,
|
|
2168
|
+
* a lease held by a live process, or a missing/foreign identity is reported as an
|
|
2169
|
+
* explicit hold, never left indistinguishable from a healthy `parking` wait.
|
|
2170
|
+
*/
|
|
2171
|
+
/**
|
|
2172
|
+
* Whether the recorded writer process is still alive, proven independently of the
|
|
2173
|
+
* recorded status. `unverified` means a live pid exists but its process-start
|
|
2174
|
+
* identity cannot be compared, so neither "gone" nor "the same writer" is proven.
|
|
2175
|
+
*/
|
|
2176
|
+
export function writerPresence(meta: RunMeta, probe: ProcessProbe = realProcessProbe): "absent" | "alive" | "unverified" {
|
|
2177
|
+
if (!Number.isInteger(meta.pid) || meta.pid <= 0) return "absent";
|
|
2178
|
+
if (!probe.pidExists(meta.pid)) return "absent";
|
|
2179
|
+
if (meta.pidStartTime === undefined) return "unverified";
|
|
2180
|
+
const token = probe.startToken?.(meta.pid);
|
|
2181
|
+
if (token === undefined) return "unverified";
|
|
2182
|
+
// A different start token is a recycled pid: the recorded writer is gone.
|
|
2183
|
+
return token === meta.pidStartTime ? "alive" : "absent";
|
|
2184
|
+
}
|
|
2185
|
+
|
|
2186
|
+
export async function recoverStagedPark(
|
|
2187
|
+
park: ParkedWait,
|
|
2188
|
+
service: WorkerOperationService,
|
|
2189
|
+
options: {
|
|
2190
|
+
writerLeaseLocked?: (sessionId: string) => boolean;
|
|
2191
|
+
probe?: ProcessProbe;
|
|
2192
|
+
} = {},
|
|
2193
|
+
): Promise<"parked" | "ready" | "held" | "skipped"> {
|
|
2194
|
+
const writer = writerPresence(park.meta, options.probe ?? realProcessProbe);
|
|
2195
|
+
if (writer === "alive") {
|
|
2196
|
+
// A supervised running attempt still has its exit handler; anything else
|
|
2197
|
+
// may still be appending to the saved conversation, and a fresh writer has
|
|
2198
|
+
// no lease, so a live process is the only reliable evidence.
|
|
2199
|
+
if (park.meta.status === "running") return "skipped";
|
|
2200
|
+
updateParkRecord(park.runId, {
|
|
2201
|
+
phase: "held",
|
|
2202
|
+
heldReason: `the staged park cannot be finalized while writer pid ${park.meta.pid} is still alive (${park.meta.status})`,
|
|
2203
|
+
});
|
|
2204
|
+
return "held";
|
|
2205
|
+
}
|
|
2206
|
+
if (writer === "unverified") {
|
|
2207
|
+
updateParkRecord(park.runId, {
|
|
2208
|
+
phase: "held",
|
|
2209
|
+
heldReason: `the recorded writer pid ${park.meta.pid} is alive but its process identity cannot be verified, so its exit is unproven`,
|
|
2210
|
+
});
|
|
2211
|
+
return "held";
|
|
2212
|
+
}
|
|
2213
|
+
const lease = readConversationLease(park.meta.sessionId);
|
|
2214
|
+
const leaseStart = lease ? processStartToken(lease.pid) : undefined;
|
|
2215
|
+
const leaseLive = lease !== undefined
|
|
2216
|
+
&& (options.writerLeaseLocked?.(park.meta.sessionId)
|
|
2217
|
+
?? (processExists(lease.pid) && (lease.start === undefined || leaseStart === undefined || lease.start === leaseStart)));
|
|
2218
|
+
if (leaseLive) {
|
|
2219
|
+
updateParkRecord(park.runId, {
|
|
2220
|
+
phase: "held",
|
|
2221
|
+
heldReason: "the staged park cannot be finalized while a live conversation writer lease is held",
|
|
2222
|
+
});
|
|
2223
|
+
return "held";
|
|
2224
|
+
}
|
|
2225
|
+
const scope = backgroundScopeForMeta(park.meta, park.record);
|
|
2226
|
+
if (!scope) {
|
|
2227
|
+
updateParkRecord(park.runId, { phase: "held", heldReason: "the staged park records no worker conversation workspace" });
|
|
2228
|
+
return "held";
|
|
2229
|
+
}
|
|
2230
|
+
const result = await finalizeWorkerParkAfterExit({
|
|
2231
|
+
runId: park.runId,
|
|
2232
|
+
service,
|
|
2233
|
+
writerLeaseReleased: true,
|
|
2234
|
+
writerPidGone: true,
|
|
2235
|
+
interrupted: true,
|
|
2236
|
+
});
|
|
2237
|
+
if (result.outcome !== "parked") return "held";
|
|
2238
|
+
if (result.wait?.phase === "ready") return "ready";
|
|
2239
|
+
return "parked";
|
|
2240
|
+
}
|
|
2241
|
+
|
|
2242
|
+
/** Every park record (any phase) that published `attemptId` as its resume. */
|
|
2243
|
+
export function parkedWaitsForResumedAttempt(attemptId: string): ParkedWait[] {
|
|
2244
|
+
const found: ParkedWait[] = [];
|
|
2245
|
+
for (const meta of listMetas()) {
|
|
2246
|
+
const record = readParkRecord(meta.id);
|
|
2247
|
+
if (!record || record.resumedBy !== attemptId) continue;
|
|
2248
|
+
found.push(parkedEntry(meta, record));
|
|
2249
|
+
}
|
|
2250
|
+
return found;
|
|
2251
|
+
}
|
|
2252
|
+
|
|
2253
|
+
/** Terminal for the purposes of "this attempt will never interpret the wait again". */
|
|
2254
|
+
export function isTerminalAttempt(meta: RunMeta): boolean {
|
|
2255
|
+
const status = effectiveStatus(meta);
|
|
2256
|
+
return status !== "running" && status !== "orphaned";
|
|
2257
|
+
}
|
|
2258
|
+
|
|
2259
|
+
export interface ResumedAttemptExitInput {
|
|
2260
|
+
/** The attempt id published by the resume claim (NOT the parked attempt). */
|
|
2261
|
+
attemptId: string;
|
|
2262
|
+
service: WorkerOperationService | undefined;
|
|
2263
|
+
/** Recorded status for the reason text. */
|
|
2264
|
+
status?: string;
|
|
2265
|
+
/** Overrides the generated reason. */
|
|
2266
|
+
reason?: string;
|
|
2267
|
+
/** The exit observer already released the writer lease before calling. */
|
|
2268
|
+
writerLeaseReleased?: boolean;
|
|
2269
|
+
notify?(message: string, level: "warning"): void;
|
|
2270
|
+
now?: () => number;
|
|
2271
|
+
}
|
|
2272
|
+
|
|
2273
|
+
export interface ResumedAttemptExitResult {
|
|
2274
|
+
outcome: "settled" | "ignored" | "foreign" | "deferred";
|
|
2275
|
+
/** Durable wait phase, when it was read or changed. */
|
|
2276
|
+
waitPhase?: string;
|
|
2277
|
+
reason?: string;
|
|
2278
|
+
}
|
|
2279
|
+
|
|
2280
|
+
/** True when a lease payload still names a live writer for the conversation. */
|
|
2281
|
+
export function conversationWriterLeaseLive(
|
|
2282
|
+
sessionId: string,
|
|
2283
|
+
probe: Pick<ProcessProbe, "pidExists" | "startToken"> = { pidExists: processExists, startToken: processStartToken },
|
|
2284
|
+
): boolean {
|
|
2285
|
+
const lease = readConversationLease(sessionId);
|
|
2286
|
+
if (!lease) return false;
|
|
2287
|
+
if (!probe.pidExists(lease.pid)) return false;
|
|
2288
|
+
if (lease.start === undefined) return true;
|
|
2289
|
+
const current = probe.startToken?.(lease.pid);
|
|
2290
|
+
return current === undefined || current === lease.start;
|
|
2291
|
+
}
|
|
2292
|
+
|
|
2293
|
+
/**
|
|
2294
|
+
* Hold the unconsumed generation that a published resume attempt was admitted for,
|
|
2295
|
+
* after that attempt exited without interpreting it.
|
|
2296
|
+
*
|
|
2297
|
+
* This is a HOLD, not a cancellation: the store keeps the cancellation epoch and
|
|
2298
|
+
* the evidence pins, records the failure reason, and invalidates the publish-only
|
|
2299
|
+
* claim so a second automatic attempt can never be published. It requires the exact
|
|
2300
|
+
* published identity (generation, claim id/token persisted with the attempt,
|
|
2301
|
+
* attempt id, launch token) and proven writer exit plus a free writer lease. An
|
|
2302
|
+
* ambiguous identity (live or unverifiable writer, live lease, missing claim
|
|
2303
|
+
* identity, missing host capability) is reported as a local hold and the durable
|
|
2304
|
+
* generation is left untouched with its live writer — never a fallback cancel.
|
|
2305
|
+
*/
|
|
2306
|
+
export async function settleResumedAttemptExit(input: ResumedAttemptExitInput): Promise<ResumedAttemptExitResult> {
|
|
2307
|
+
const candidates = parkedWaitsForResumedAttempt(input.attemptId);
|
|
2308
|
+
if (candidates.length === 0) return { outcome: "ignored" };
|
|
2309
|
+
const attempt = readMeta(input.attemptId);
|
|
2310
|
+
const at = (input.now ?? Date.now)();
|
|
2311
|
+
let result: ResumedAttemptExitResult = { outcome: "ignored" };
|
|
2312
|
+
for (const park of candidates) {
|
|
2313
|
+
if (park.record.phase === "consumed" || park.record.phase === "cancelled") {
|
|
2314
|
+
result = { outcome: "ignored", waitPhase: park.record.phase };
|
|
2315
|
+
continue;
|
|
2316
|
+
}
|
|
2317
|
+
const reason = input.reason
|
|
2318
|
+
?? `the published resume attempt ${input.attemptId} exited (${input.status ?? attempt?.status ?? "unknown"}) without interpreting the wait`;
|
|
2319
|
+
const holdLocally = (cause: string, outcome: "deferred" | "settled" = "deferred"): void => {
|
|
2320
|
+
updateParkRecord(park.runId, { phase: "held", heldReason: `${reason}; ${cause}` });
|
|
2321
|
+
result = { outcome, reason };
|
|
2322
|
+
};
|
|
2323
|
+
// The publication token is what identifies THIS resume of the record.
|
|
2324
|
+
const recordedToken = park.record.resumeLaunchToken;
|
|
2325
|
+
const attemptToken = attempt?.operations?.resumeLaunchToken;
|
|
2326
|
+
if (recordedToken && attemptToken && recordedToken !== attemptToken) {
|
|
2327
|
+
updateParkRecord(park.runId, {
|
|
2328
|
+
phase: "held",
|
|
2329
|
+
heldReason: `attempt ${input.attemptId} does not match the launch token published for this wait; the wait was left for its owner`,
|
|
2330
|
+
});
|
|
2331
|
+
input.notify?.(`Subagent ${park.runId}: resume attempt ${input.attemptId} does not match the published launch token; the wait is held for inspection.`, "warning");
|
|
2332
|
+
result = { outcome: "foreign" };
|
|
2333
|
+
continue;
|
|
2334
|
+
}
|
|
2335
|
+
if (!input.service) {
|
|
2336
|
+
holdLocally("the Background operation service is unavailable");
|
|
2337
|
+
continue;
|
|
2338
|
+
}
|
|
2339
|
+
if (typeof input.service.holdFailedResumeForWorker !== "function") {
|
|
2340
|
+
holdLocally("this Background host has no proven-exit hold capability yet");
|
|
2341
|
+
continue;
|
|
2342
|
+
}
|
|
2343
|
+
const claimId = park.record.resumeClaimId;
|
|
2344
|
+
const claimToken = park.record.resumeClaimToken;
|
|
2345
|
+
if (!claimId || !claimToken) {
|
|
2346
|
+
holdLocally("the resume claim identity was not recorded with the attempt, so the durable claim cannot be matched");
|
|
2347
|
+
continue;
|
|
2348
|
+
}
|
|
2349
|
+
if (!attempt) {
|
|
2350
|
+
holdLocally("the published continuation's metadata is missing; its writer identity and exit cannot be proved");
|
|
2351
|
+
continue;
|
|
2352
|
+
}
|
|
2353
|
+
const writer = writerPresence(attempt);
|
|
2354
|
+
const writerPidGone = writer === "absent";
|
|
2355
|
+
if (!writerPidGone) {
|
|
2356
|
+
holdLocally(`writer pid ${attempt?.pid} is still alive, so its exit is unproven`);
|
|
2357
|
+
continue;
|
|
2358
|
+
}
|
|
2359
|
+
if (writer === "unverified" && attempt) {
|
|
2360
|
+
holdLocally(`the recorded writer pid ${attempt.pid} is alive but its process identity cannot be verified`);
|
|
2361
|
+
continue;
|
|
2362
|
+
}
|
|
2363
|
+
const writerLeaseReleased = input.writerLeaseReleased
|
|
2364
|
+
?? !conversationWriterLeaseLive(attempt?.sessionId ?? park.conversationId);
|
|
2365
|
+
if (!writerLeaseReleased) {
|
|
2366
|
+
holdLocally("a live conversation writer lease still holds the history, so writer exit is unproven");
|
|
2367
|
+
continue;
|
|
2368
|
+
}
|
|
2369
|
+
const scope = backgroundScopeForMeta(park.meta, park.record);
|
|
2370
|
+
if (!scope) {
|
|
2371
|
+
holdLocally("the parked attempt records no worker conversation scope");
|
|
2372
|
+
continue;
|
|
2373
|
+
}
|
|
2374
|
+
try {
|
|
2375
|
+
const held = input.service.holdFailedResumeForWorker(scope, {
|
|
2376
|
+
waitId: park.waitId,
|
|
2377
|
+
generation: park.generation,
|
|
2378
|
+
claimId,
|
|
2379
|
+
claimToken,
|
|
2380
|
+
attemptId: input.attemptId,
|
|
2381
|
+
launchToken: park.record.resumeLaunchToken ?? attemptToken ?? "",
|
|
2382
|
+
proof: { writerPidGone: writerPidGone && writer !== "unverified", writerLeaseReleased },
|
|
2383
|
+
reason,
|
|
2384
|
+
});
|
|
2385
|
+
const phase = phaseOf(held.phase);
|
|
2386
|
+
updateParkRecord(park.runId, {
|
|
2387
|
+
phase,
|
|
2388
|
+
...(phase === "held" ? { heldReason: reason } : {}),
|
|
2389
|
+
});
|
|
2390
|
+
input.notify?.(`Subagent ${park.runId}: ${reason}. The wait is ${phase}; it will not resume automatically.`, "warning");
|
|
2391
|
+
result = { outcome: "settled", waitPhase: phase, reason };
|
|
2392
|
+
} catch (error) {
|
|
2393
|
+
// The store refused (stale identity, terminal generation, unproven exit).
|
|
2394
|
+
// Never fall back to cancellation: report the local hold and leave the
|
|
2395
|
+
// durable generation exactly as it is.
|
|
2396
|
+
let durablePhase: string | undefined;
|
|
2397
|
+
try {
|
|
2398
|
+
durablePhase = input.service.listWorkerWaits(scope)
|
|
2399
|
+
.find((entry) => entry.waitId === park.waitId && entry.generation === park.generation)?.phase;
|
|
2400
|
+
} catch { /* durable state unreadable */ }
|
|
2401
|
+
if (durablePhase === "consumed" || durablePhase === "cancelled") {
|
|
2402
|
+
updateParkRecord(park.runId, { phase: phaseOf(durablePhase) });
|
|
2403
|
+
result = { outcome: "ignored", waitPhase: durablePhase };
|
|
2404
|
+
continue;
|
|
2405
|
+
}
|
|
2406
|
+
holdLocally(`the durable hold was refused (${reasonOf(error)})`);
|
|
2407
|
+
if (durablePhase) result = { ...result, waitPhase: durablePhase };
|
|
2408
|
+
}
|
|
2409
|
+
}
|
|
2410
|
+
void at;
|
|
2411
|
+
return result;
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
export async function reconcileParkedWaits(
|
|
2415
|
+
service: WorkerOperationService | undefined,
|
|
2416
|
+
options: {
|
|
2417
|
+
resume(park: ParkedWait): Promise<"resumed" | "held" | "skipped">;
|
|
2418
|
+
notify?(message: string, level: "warning"): void;
|
|
2419
|
+
},
|
|
2420
|
+
): Promise<{ resumed: number; held: number; skipped: number }> {
|
|
2421
|
+
let resumed = 0;
|
|
2422
|
+
let held = 0;
|
|
2423
|
+
let skipped = 0;
|
|
2424
|
+
for (const park of parkedWaits()) {
|
|
2425
|
+
try {
|
|
2426
|
+
if (park.record.phase === "resuming") {
|
|
2427
|
+
const attemptId = park.record.resumedBy ?? "";
|
|
2428
|
+
const attempt = attemptId ? readMeta(attemptId) : undefined;
|
|
2429
|
+
if (!attemptId || !attempt) {
|
|
2430
|
+
updateParkRecord(park.runId, {
|
|
2431
|
+
phase: "held",
|
|
2432
|
+
heldReason: `Resume attempt ${attemptId || "(unknown)"} was recorded before spawn but never started; reconcile it manually.`,
|
|
2433
|
+
});
|
|
2434
|
+
options.notify?.(
|
|
2435
|
+
`Subagent ${park.runId}: resume attempt ${attemptId || "(unknown)"} was recorded but never started. The wait is held; take over with subagent_stop ${park.runId}.`,
|
|
2436
|
+
"warning",
|
|
2437
|
+
);
|
|
2438
|
+
held += 1;
|
|
2439
|
+
continue;
|
|
2440
|
+
}
|
|
2441
|
+
// The published attempt already ended without acknowledging: settle the
|
|
2442
|
+
// unconsumed generation now instead of leaving it `resuming` forever.
|
|
2443
|
+
if (isTerminalAttempt(attempt)) {
|
|
2444
|
+
const settled = await settleResumedAttemptExit({
|
|
2445
|
+
attemptId,
|
|
2446
|
+
service,
|
|
2447
|
+
status: effectiveStatus(attempt),
|
|
2448
|
+
...(options.notify ? { notify: options.notify } : {}),
|
|
2449
|
+
});
|
|
2450
|
+
if (settled.outcome === "settled") held += 1;
|
|
2451
|
+
else skipped += 1;
|
|
2452
|
+
continue;
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
if (!service) {
|
|
2456
|
+
skipped += 1;
|
|
2457
|
+
continue;
|
|
2458
|
+
}
|
|
2459
|
+
const scope = backgroundScopeForMeta(park.meta, park.record);
|
|
2460
|
+
if (!scope) {
|
|
2461
|
+
skipped += 1;
|
|
2462
|
+
continue;
|
|
2463
|
+
}
|
|
2464
|
+
const wait = service.listWorkerWaits(scope, { pendingOnly: true })
|
|
2465
|
+
.find((entry) => entry.waitId === park.waitId && entry.generation === park.generation);
|
|
2466
|
+
if (!wait) {
|
|
2467
|
+
// The generation was consumed, cancelled or superseded elsewhere.
|
|
2468
|
+
updateParkRecord(park.runId, { phase: "consumed" });
|
|
2469
|
+
skipped += 1;
|
|
2470
|
+
continue;
|
|
2471
|
+
}
|
|
2472
|
+
if (wait.phase === "resuming") {
|
|
2473
|
+
// A published attempt owns this generation. If it already ended (or
|
|
2474
|
+
// its record vanished), the unconsumed generation is settled
|
|
2475
|
+
// explicitly; nothing else may resume or mirror it.
|
|
2476
|
+
const attemptId = park.record.resumedBy ?? "";
|
|
2477
|
+
const attempt = attemptId ? readMeta(attemptId) : undefined;
|
|
2478
|
+
if (!attempt || isTerminalAttempt(attempt)) {
|
|
2479
|
+
const settled = await settleResumedAttemptExit({
|
|
2480
|
+
attemptId: attemptId || park.runId,
|
|
2481
|
+
service,
|
|
2482
|
+
status: attempt ? effectiveStatus(attempt) : "missing",
|
|
2483
|
+
...(options.notify ? { notify: options.notify } : {}),
|
|
2484
|
+
});
|
|
2485
|
+
// `deferred` means the local record was held while the durable
|
|
2486
|
+
// generation could not be proven (ambiguous identity/capability);
|
|
2487
|
+
// it is never a silent success.
|
|
2488
|
+
if (settled.outcome === "settled" || settled.outcome === "deferred") held += 1;
|
|
2489
|
+
else skipped += 1;
|
|
2490
|
+
continue;
|
|
2491
|
+
}
|
|
2492
|
+
// The attempt is still live; it will acknowledge or exit.
|
|
2493
|
+
skipped += 1;
|
|
2494
|
+
continue;
|
|
2495
|
+
}
|
|
2496
|
+
if (wait.phase === "ready") {
|
|
2497
|
+
const outcome = await options.resume(park);
|
|
2498
|
+
if (outcome === "resumed") resumed += 1;
|
|
2499
|
+
else if (outcome === "held") held += 1;
|
|
2500
|
+
else skipped += 1;
|
|
2501
|
+
continue;
|
|
2502
|
+
}
|
|
2503
|
+
if (wait.phase === "parking") {
|
|
2504
|
+
// The exit proof is normally sent by the run's exit handler. After a
|
|
2505
|
+
// restart that handler is gone, so the staged checkpoint is retried
|
|
2506
|
+
// here against real writer and lease evidence. Anything ambiguous is
|
|
2507
|
+
// held explicitly instead of parking forever.
|
|
2508
|
+
const recovered = await recoverStagedPark(park, service);
|
|
2509
|
+
if (recovered === "parked" || recovered === "ready") resumed += 1;
|
|
2510
|
+
else if (recovered === "held") held += 1;
|
|
2511
|
+
else skipped += 1;
|
|
2512
|
+
continue;
|
|
2513
|
+
}
|
|
2514
|
+
if (phaseOf(wait.phase) !== park.phase) updateParkRecord(park.runId, { phase: phaseOf(wait.phase) });
|
|
2515
|
+
skipped += 1;
|
|
2516
|
+
} catch (error) {
|
|
2517
|
+
updateParkRecord(park.runId, { phase: "held", heldReason: reasonOf(error) });
|
|
2518
|
+
options.notify?.(`Subagent ${park.runId}: wait reconciliation failed (${reasonOf(error)}).`, "warning");
|
|
2519
|
+
held += 1;
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
2522
|
+
return { resumed, held, skipped };
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
/**
|
|
2526
|
+
* Prompt for the continuation attempt. It restates the durable wait identity and
|
|
2527
|
+
* the next reasoning step only: it changes no tool, model, role, permission,
|
|
2528
|
+
* workspace or authorization, and it does not tell the worker the outcome.
|
|
2529
|
+
*/
|
|
2530
|
+
export function resumePrompt(park: ParkedWait): string {
|
|
2531
|
+
const note = park.record.continuationNote ?? "Continue the assignment you were working on.";
|
|
2532
|
+
return [
|
|
2533
|
+
"Your previous execution attempt parked while it waited on background operations, and they have now all reached a terminal outcome.",
|
|
2534
|
+
`Durable wait: ${park.waitId} generation ${park.generation}.`,
|
|
2535
|
+
"Read the terminal evidence for every task listed in that wait (bg_task_status / bg_task_log), interpret it yourself, and then acknowledge the result with acknowledge_operation_result naming the exact terminal revisions you read.",
|
|
2536
|
+
"Then continue the original assignment. The evidence decides what to do; it does not by itself mean the assignment succeeded.",
|
|
2537
|
+
`Next reasoning step recorded when you parked: ${note}`,
|
|
2538
|
+
].join("\n");
|
|
2539
|
+
}
|
|
2540
|
+
|
|
2541
|
+
/** Child bridge extension path inside this package. */
|
|
2542
|
+
export function childBridgeExtensionPath(packageDir: string): string {
|
|
2543
|
+
return join(packageDir, "child-operations.ts");
|
|
2544
|
+
}
|
|
2545
|
+
|
|
2546
|
+
/** True when `taskId` appeared in a launch result (used by the host's ownership seam). */
|
|
2547
|
+
export function launchedTaskIds(result: unknown): string[] {
|
|
2548
|
+
return extractResultTaskIds(result);
|
|
2549
|
+
}
|