pi-famulus 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +31 -12
- package/src/agents/builtins.ts +5 -22
- package/src/agents/definition.ts +16 -12
- package/src/bash-override.ts +8 -21
- package/src/behavior-guidelines.ts +40 -15
- package/src/clock.ts +5 -0
- package/src/comms/comms.ts +7 -3
- package/src/comms/tools.ts +11 -36
- package/src/config.ts +25 -1
- package/src/format.ts +83 -5
- package/src/index.ts +155 -30
- package/src/manager-client.ts +265 -114
- package/src/monitor.ts +14 -33
- package/src/notify.ts +7 -7
- package/src/prompts.generated.ts +183 -0
- package/src/shell-exec.ts +6 -2
- package/src/subagent/admission.ts +224 -0
- package/src/subagent/agent-records.ts +66 -3
- package/src/subagent/child-bash.ts +65 -56
- package/src/subagent/model-spec.ts +134 -16
- package/src/subagent/overrun.ts +191 -0
- package/src/subagent/pi-runtime.ts +118 -33
- package/src/subagent/registry.ts +235 -34
- package/src/subagent/runner.ts +565 -99
- package/src/subagent/tool.ts +111 -61
- package/src/subagent/types.ts +71 -7
- package/src/subagent/usage.ts +48 -0
- package/src/task-tools.ts +15 -25
- package/src/thinking-levels.ts +26 -0
- package/src/tui/message-renderers.ts +52 -2
- package/src/wake-delivery.ts +137 -0
- package/src/wake.ts +270 -7
- package/src/work-index.ts +4 -0
package/src/subagent/runner.ts
CHANGED
|
@@ -9,7 +9,13 @@
|
|
|
9
9
|
* starts a new generation on the same session):
|
|
10
10
|
* - a per-generation admission slot is acquired via the optional `acquire`
|
|
11
11
|
* hook (the registry uses it for the global concurrency cap);
|
|
12
|
-
* - `timeoutMs` is a
|
|
12
|
+
* - `timeoutMs` is a SOFT deadline per user turn: reaching it calls
|
|
13
|
+
* `onOverrun` (the parent gets a subagent-overrun wake) and repeats every
|
|
14
|
+
* `overrunRepeatMs`; the child is not aborted and its shell keeps running.
|
|
15
|
+
* `extend()` re-arms the deadline; steer/followUp after a reminder postpone
|
|
16
|
+
* the next one. See design.md §4.6 "Child lifecycle".
|
|
17
|
+
* - `hardTimeoutMs` (opt-in, default off) is the abort ceiling per turn:
|
|
18
|
+
* abort -> result {status:"interrupted", error:"timeout"};
|
|
13
19
|
* - a stall watchdog aborts the child after `stallMs` (default 10min) without
|
|
14
20
|
* any session event. A stall is treated as transient (a silently dropped
|
|
15
21
|
* provider stream): the child is aborted and auto-resumed on the SAME
|
|
@@ -21,9 +27,10 @@
|
|
|
21
27
|
* rejects prompt() while a run is still active ("Agent is already
|
|
22
28
|
* processing"). The wait is bounded by `stallMs` — an abort that never
|
|
23
29
|
* completes means the stream ignored it and the session is dead.
|
|
24
|
-
* -
|
|
25
|
-
*
|
|
26
|
-
*
|
|
30
|
+
* - deadlines belong to the whole user turn: stall retries re-arm the soft
|
|
31
|
+
* deadline / reminder schedule and the hard ceiling from turn-level fields,
|
|
32
|
+
* never from the retry. A reminder landing during a retry delay is sent and
|
|
33
|
+
* the retry still runs; a hard ceiling landing there still settles.
|
|
27
34
|
* - session/prompt exceptions -> {status:"failed", error}.
|
|
28
35
|
*
|
|
29
36
|
* Note on pi semantics: AgentSession.prompt() resolves only after the whole
|
|
@@ -32,12 +39,15 @@
|
|
|
32
39
|
* steer/followUp processing.
|
|
33
40
|
*/
|
|
34
41
|
import { realClock, TimerScope, type Clock, type ClockTimer } from "../clock";
|
|
42
|
+
import type { OverrunTick } from "./overrun";
|
|
35
43
|
import type {
|
|
44
|
+
ChildHandle,
|
|
36
45
|
ChildResult,
|
|
37
46
|
ChildRunRequest,
|
|
38
47
|
ChildRunner,
|
|
39
48
|
ChildSessionAdapter,
|
|
40
49
|
ChildStatus,
|
|
50
|
+
ChildWallUsage,
|
|
41
51
|
CreateSessionFn,
|
|
42
52
|
DisposableChildHandle,
|
|
43
53
|
} from "./types";
|
|
@@ -48,6 +58,8 @@ export const DEFAULT_STALL_MS = 5 * 60 * 1000;
|
|
|
48
58
|
export const DEFAULT_STALL_RETRIES = 1;
|
|
49
59
|
/** Pause between the stall abort and the retry prompt (ms). */
|
|
50
60
|
export const DEFAULT_STALL_RETRY_DELAY_MS = 5_000;
|
|
61
|
+
/** Overrun reminder interval while a child stays past its soft budget (ms). */
|
|
62
|
+
export const DEFAULT_OVERRUN_REPEAT_MS = 10 * 60 * 1000;
|
|
51
63
|
|
|
52
64
|
/** Continuation prompt for a stall retry: the transcript holds the context. */
|
|
53
65
|
export function stallRetryPrompt(stallMs: number): string {
|
|
@@ -57,6 +69,14 @@ export function stallRetryPrompt(stallMs: number): string {
|
|
|
57
69
|
);
|
|
58
70
|
}
|
|
59
71
|
|
|
72
|
+
/** Identity of one generation's slot request (see InProcessRunnerOptions.acquire). */
|
|
73
|
+
export interface AdmissionTicket {
|
|
74
|
+
/** False once the requesting generation settled or was superseded. */
|
|
75
|
+
current: () => boolean;
|
|
76
|
+
/** Aborted when this generation settles while its admission is queued. */
|
|
77
|
+
signal: AbortSignal;
|
|
78
|
+
}
|
|
79
|
+
|
|
60
80
|
export interface InProcessRunnerOptions {
|
|
61
81
|
createSession: CreateSessionFn;
|
|
62
82
|
/** Stall watchdog timeout (ms). Default 10 minutes. */
|
|
@@ -67,14 +87,24 @@ export interface InProcessRunnerOptions {
|
|
|
67
87
|
stallRetryDelayMs?: number;
|
|
68
88
|
/** Called on every stall detection (before any retry), with the 1-based attempt. */
|
|
69
89
|
onStall?: (childId: string, attempt: number) => void;
|
|
90
|
+
/** Overrun reminder interval (ms). Default 10 minutes. */
|
|
91
|
+
overrunRepeatMs?: number;
|
|
92
|
+
/** Abort ceiling per user turn (ms). 0/absent = off: the soft deadline never aborts. */
|
|
93
|
+
hardTimeoutMs?: number;
|
|
94
|
+
/**
|
|
95
|
+
* The child passed its soft deadline (first reminder) or is still past it
|
|
96
|
+
* (later reminders). The child keeps running; the parent decides.
|
|
97
|
+
*/
|
|
98
|
+
onOverrun?: (tick: OverrunTick) => void;
|
|
70
99
|
/** Shared time source and scheduler. */
|
|
71
100
|
clock?: Clock;
|
|
72
101
|
/**
|
|
73
102
|
* Per-generation admission hook. Awaited before each (re)start; the
|
|
74
|
-
*
|
|
75
|
-
*
|
|
103
|
+
* releaser gets terminal=false for a resumable interruption so its local
|
|
104
|
+
* slot can be returned while the child-owned machine permit is retained.
|
|
105
|
+
* Rejecting cancels the generation as {status:"interrupted", error}.
|
|
76
106
|
*/
|
|
77
|
-
acquire?: (req: ChildRunRequest) => Promise<() => void>;
|
|
107
|
+
acquire?: (req: ChildRunRequest, ticket: AdmissionTicket) => Promise<(terminal?: boolean) => void>;
|
|
78
108
|
/**
|
|
79
109
|
* Called after the child's conversation may have changed (a message or
|
|
80
110
|
* tool finished, or the generation settled). Used to persist transcripts.
|
|
@@ -93,8 +123,9 @@ export class InProcessRunner implements ChildRunner {
|
|
|
93
123
|
this.opts = opts;
|
|
94
124
|
}
|
|
95
125
|
|
|
96
|
-
async start(req: ChildRunRequest): Promise<DisposableChildHandle> {
|
|
126
|
+
async start(req: ChildRunRequest, onHandle?: (handle: ChildHandle) => void): Promise<DisposableChildHandle> {
|
|
97
127
|
const handle = new InProcessChildHandle(req, this.opts);
|
|
128
|
+
onHandle?.(handle);
|
|
98
129
|
await handle.launch();
|
|
99
130
|
return handle;
|
|
100
131
|
}
|
|
@@ -107,25 +138,39 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
107
138
|
private readonly stallRetries: number;
|
|
108
139
|
private readonly stallRetryDelayMs: number;
|
|
109
140
|
private readonly onStall?: (childId: string, attempt: number) => void;
|
|
141
|
+
private readonly overrunRepeatMs: number;
|
|
142
|
+
private readonly hardTimeoutMs: number;
|
|
143
|
+
private readonly onOverrun?: (tick: OverrunTick) => void;
|
|
110
144
|
private readonly clock: Clock;
|
|
111
|
-
private readonly acquire?: (req: ChildRunRequest) => Promise<() => void>;
|
|
145
|
+
private readonly acquire?: (req: ChildRunRequest, ticket: AdmissionTicket) => Promise<(terminal?: boolean) => void>;
|
|
112
146
|
private readonly onActivity?: (childId: string) => void;
|
|
113
147
|
|
|
114
148
|
private session: ChildSessionAdapter | null = null;
|
|
149
|
+
private sessionCreation: Promise<ChildSessionAdapter> | null = null;
|
|
115
150
|
private resolvedModel_: string | undefined;
|
|
116
151
|
private unsubscribe: (() => void) | null = null;
|
|
117
152
|
private status_: ChildStatus = "pending";
|
|
118
153
|
private lastEvent: number;
|
|
119
|
-
/**
|
|
154
|
+
/** Session start, set once and preserved across user resumes. */
|
|
120
155
|
private startedAt: number;
|
|
121
156
|
/** User-turn start: reset on launch and user resume(), not on stall retries. */
|
|
122
157
|
private runStartedAt: number;
|
|
158
|
+
/** Timestamp when this generation acquired admission; durationMs is measured from here. */
|
|
159
|
+
private admittedAt: number | null;
|
|
123
160
|
/**
|
|
124
|
-
* When the
|
|
161
|
+
* When the turn's deadlines start counting. Set after admission and session
|
|
125
162
|
* creation (queue wait must not eat the budget), and NOT reset by stall
|
|
126
163
|
* retries — a stall already consumes budget time by definition.
|
|
127
164
|
*/
|
|
128
165
|
private turnBudgetStart: number;
|
|
166
|
+
/** This turn's soft budget: spawn timeoutMs, or resume()'s timeoutMs. */
|
|
167
|
+
private turnBudgetMs: number;
|
|
168
|
+
/** Soft deadline (absolute). Moved by extend(). Null = no soft deadline. */
|
|
169
|
+
private softDeadlineAt: number | null = null;
|
|
170
|
+
/** When the soft timer fires next: softDeadlineAt, then each reminder. */
|
|
171
|
+
private nextReminderAt: number | null = null;
|
|
172
|
+
/** Overrun reminders sent this turn. */
|
|
173
|
+
private reminders = 0;
|
|
129
174
|
/** Abort kicked off by the latest stall detection; awaited (bounded) by the retry. */
|
|
130
175
|
private abortPromise: Promise<void> | null = null;
|
|
131
176
|
private generation = 0;
|
|
@@ -140,7 +185,8 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
140
185
|
* the previous turn's settle by definition).
|
|
141
186
|
*/
|
|
142
187
|
private settleWaiters: Array<() => void> = [];
|
|
143
|
-
private
|
|
188
|
+
private softTimer: ClockTimer | null = null;
|
|
189
|
+
private hardTimer: ClockTimer | null = null;
|
|
144
190
|
private stallTimer: ClockTimer | null = null;
|
|
145
191
|
private retryTimer: { scope: TimerScope; id: ClockTimer } | null = null;
|
|
146
192
|
private timerScope: TimerScope | null = null;
|
|
@@ -148,10 +194,29 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
148
194
|
private stallAttempts = 0;
|
|
149
195
|
/** Generation retired by a stall detection awaiting its retry. */
|
|
150
196
|
private retiredGen: number | null = null;
|
|
151
|
-
private releaseSlot: (() => void) | null = null;
|
|
197
|
+
private releaseSlot: ((terminal?: boolean) => void) | null = null;
|
|
198
|
+
private admissionAbort: AbortController | null = null;
|
|
199
|
+
/** Whether the current generation's settled result is terminal for this child. */
|
|
200
|
+
private terminalSettle = false;
|
|
152
201
|
private disposed = false;
|
|
153
|
-
|
|
202
|
+
private disposalPromise: Promise<void> | undefined;
|
|
203
|
+
/** Nested current-generation tool calls. Stall stays paused while > 0. */
|
|
154
204
|
private toolDepth = 0;
|
|
205
|
+
/** Start generation by Pi toolCallId so late end events cannot close a new interval. */
|
|
206
|
+
private readonly toolCallGenerations = new Map<string, number>();
|
|
207
|
+
/** FIFO fallback for adapters that omit Pi's toolCallId (marked approximate). */
|
|
208
|
+
private readonly anonymousToolGenerations: number[] = [];
|
|
209
|
+
private wall: ChildWallUsage = { llmMs: 0, toolMs: 0, queueMs: 0, otherMs: 0, approximate: false };
|
|
210
|
+
private generationAt: number | null = null;
|
|
211
|
+
private generationLlmMs = 0;
|
|
212
|
+
private generationToolMs = 0;
|
|
213
|
+
private generationQueueMs = 0;
|
|
214
|
+
private queueStartedAt: number | null = null;
|
|
215
|
+
private queueGeneration: number | null = null;
|
|
216
|
+
/** Pending-abort drain before resume is unclassified wall time. */
|
|
217
|
+
private otherStartedAt: number | null = null;
|
|
218
|
+
private llmStartedAt: number | null = null;
|
|
219
|
+
private toolStartedAt: number | null = null;
|
|
155
220
|
/** contact_supervisor need_decision. Stall stays paused while true. */
|
|
156
221
|
private decisionPaused = false;
|
|
157
222
|
|
|
@@ -168,11 +233,21 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
168
233
|
? Math.max(0, Math.floor(rawDelay))
|
|
169
234
|
: DEFAULT_STALL_RETRY_DELAY_MS;
|
|
170
235
|
this.onStall = opts.onStall;
|
|
236
|
+
const rawRepeat = opts.overrunRepeatMs ?? DEFAULT_OVERRUN_REPEAT_MS;
|
|
237
|
+
this.overrunRepeatMs =
|
|
238
|
+
Number.isFinite(rawRepeat) && rawRepeat > 0
|
|
239
|
+
? Math.max(1, Math.floor(rawRepeat))
|
|
240
|
+
: DEFAULT_OVERRUN_REPEAT_MS;
|
|
241
|
+
const rawHard = opts.hardTimeoutMs ?? 0;
|
|
242
|
+
this.hardTimeoutMs = Number.isFinite(rawHard) && rawHard > 0 ? Math.floor(rawHard) : 0;
|
|
243
|
+
this.onOverrun = opts.onOverrun;
|
|
244
|
+
this.turnBudgetMs = req.timeoutMs;
|
|
171
245
|
this.clock = opts.clock ?? realClock;
|
|
172
246
|
this.acquire = opts.acquire;
|
|
173
247
|
this.onActivity = opts.onActivity;
|
|
174
248
|
this.startedAt = this.clock.now();
|
|
175
249
|
this.runStartedAt = this.startedAt;
|
|
250
|
+
this.admittedAt = null;
|
|
176
251
|
this.turnBudgetStart = this.startedAt;
|
|
177
252
|
this.lastEvent = this.startedAt;
|
|
178
253
|
this.resultPromise = new Promise((resolve) => {
|
|
@@ -201,7 +276,14 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
201
276
|
return this.resolvedModel_ ?? this.session?.resolvedModel;
|
|
202
277
|
}
|
|
203
278
|
|
|
204
|
-
|
|
279
|
+
effectiveThinkingLevel(): string | undefined {
|
|
280
|
+
return this.session?.effectiveThinkingLevel;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* need_decision pending: pause the stall watchdog (nested with tool
|
|
285
|
+
* execution) and hold overrun reminders. The soft budget keeps counting.
|
|
286
|
+
*/
|
|
205
287
|
pauseStall(): void {
|
|
206
288
|
this.decisionPaused = true;
|
|
207
289
|
this.clearStall();
|
|
@@ -211,12 +293,39 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
211
293
|
this.decisionPaused = false;
|
|
212
294
|
this.lastEvent = this.clock.now();
|
|
213
295
|
if (this.status_ === "running" && this.toolDepth === 0) this.armStall(this.generation);
|
|
296
|
+
// Deliver a reminder held during the decision (due now, so it fires at
|
|
297
|
+
// once); otherwise this re-arms the unchanged schedule.
|
|
298
|
+
if (this.status_ === "running" && !this.settledFlag) this.armSoftDeadline(this.generation);
|
|
214
299
|
}
|
|
215
300
|
|
|
216
301
|
conversation() {
|
|
217
302
|
return this.session?.getConversation() ?? [];
|
|
218
303
|
}
|
|
219
304
|
|
|
305
|
+
tokenUsage() {
|
|
306
|
+
return this.session?.tokenUsage() ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
wallUsage(): ChildWallUsage {
|
|
310
|
+
const now = this.clock.now();
|
|
311
|
+
const queueMs = this.queueStartedAt === null ? 0 : Math.max(0, now - this.queueStartedAt);
|
|
312
|
+
const llmMs = this.llmStartedAt === null ? 0 : Math.max(0, now - this.llmStartedAt);
|
|
313
|
+
const toolMs = this.toolStartedAt === null ? 0 : Math.max(0, now - this.toolStartedAt);
|
|
314
|
+
const elapsed = this.generationAt === null ? 0 : Math.max(0, now - this.generationAt);
|
|
315
|
+
const accounted = this.generationQueueMs + queueMs + this.generationLlmMs + llmMs + this.generationToolMs + toolMs;
|
|
316
|
+
const otherMs =
|
|
317
|
+
this.wall.otherMs +
|
|
318
|
+
Math.max(0, elapsed - accounted) +
|
|
319
|
+
(this.otherStartedAt === null ? 0 : Math.max(0, now - this.otherStartedAt));
|
|
320
|
+
return {
|
|
321
|
+
llmMs: this.wall.llmMs + llmMs,
|
|
322
|
+
toolMs: this.wall.toolMs + toolMs,
|
|
323
|
+
queueMs: this.wall.queueMs + queueMs,
|
|
324
|
+
otherMs,
|
|
325
|
+
approximate: this.wall.approximate || otherMs > this.wall.otherMs,
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
|
|
220
329
|
/** Launch generation 1. Resolves once the prompt is issued (not completed). */
|
|
221
330
|
async launch(): Promise<void> {
|
|
222
331
|
await this.beginGeneration(this.req.prompt, true);
|
|
@@ -230,7 +339,9 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
230
339
|
// The previous generation was aborted; delivery would be undefined.
|
|
231
340
|
throw new Error(`subagent ${this.req.childId} is restarting after a stall; retry shortly`);
|
|
232
341
|
}
|
|
342
|
+
const gen = this.generation;
|
|
233
343
|
await this.session.steer(message);
|
|
344
|
+
this.parentActed(gen);
|
|
234
345
|
}
|
|
235
346
|
|
|
236
347
|
async followUp(message: string): Promise<void> {
|
|
@@ -240,10 +351,57 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
240
351
|
if (this.retiredGen !== null) {
|
|
241
352
|
throw new Error(`subagent ${this.req.childId} is restarting after a stall; retry shortly`);
|
|
242
353
|
}
|
|
354
|
+
const gen = this.generation;
|
|
243
355
|
await this.session.followUp(message);
|
|
356
|
+
this.parentActed(gen);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
* Parent action on an overrun: move the soft deadline to now + timeoutMs
|
|
361
|
+
* (default: the spawn budget). The reminder count continues; the hard
|
|
362
|
+
* ceiling (if any) does not move.
|
|
363
|
+
*/
|
|
364
|
+
extend(timeoutMs?: number): { deadlineAt: number; hardDeadlineAt: number | null } {
|
|
365
|
+
if (this.status_ !== "running") {
|
|
366
|
+
throw new Error(`subagent ${this.req.childId} is not running (status: ${this.status_})`);
|
|
367
|
+
}
|
|
368
|
+
const ms = timeoutMs ?? this.req.timeoutMs;
|
|
369
|
+
if (!(ms > 0)) throw new Error("extend needs a positive timeout_ms");
|
|
370
|
+
this.softDeadlineAt = this.now() + ms;
|
|
371
|
+
this.nextReminderAt = this.softDeadlineAt;
|
|
372
|
+
this.armSoftDeadline(this.generation);
|
|
373
|
+
return {
|
|
374
|
+
deadlineAt: this.softDeadlineAt,
|
|
375
|
+
hardDeadlineAt: this.hardTimeoutMs > 0 ? this.turnBudgetStart + this.hardTimeoutMs : null,
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* steer/followUp after a reminder: the parent is handling the overrun, so
|
|
381
|
+
* the next reminder waits a full interval. Before the first reminder the
|
|
382
|
+
* deadline is untouched, and so is a deadline that extend() moved into the
|
|
383
|
+
* future: the child is inside its budget again, and the next wake belongs
|
|
384
|
+
* at that deadline, not one repeat from now.
|
|
385
|
+
*/
|
|
386
|
+
private parentActed(gen: number): void {
|
|
387
|
+
// The delivery was awaited: if its turn settled (and maybe a resume
|
|
388
|
+
// started a new one) meanwhile, it must not move the new schedule.
|
|
389
|
+
if (gen !== this.generation || this.settledFlag) return;
|
|
390
|
+
if (this.reminders === 0 || this.status_ !== "running") return;
|
|
391
|
+
const now = this.now();
|
|
392
|
+
if (this.softDeadlineAt !== null && now < this.softDeadlineAt) return;
|
|
393
|
+
this.nextReminderAt = now + this.overrunRepeatMs;
|
|
394
|
+
this.armSoftDeadline(this.generation);
|
|
244
395
|
}
|
|
245
396
|
|
|
246
|
-
|
|
397
|
+
/**
|
|
398
|
+
* Start a new user turn. Resolves once the request is accepted, not when
|
|
399
|
+
* the turn starts: draining a still-unwinding abort and waiting for an
|
|
400
|
+
* admission slot run in the background, so the parent's tool call never
|
|
401
|
+
* waits for a slot. Until admission the child is "pending" (queued).
|
|
402
|
+
* Failures after acceptance settle the new turn instead of rejecting.
|
|
403
|
+
*/
|
|
404
|
+
async resume(message: string, opts: { timeoutMs?: number } = {}): Promise<void> {
|
|
247
405
|
if (this.status_ === "running" || this.status_ === "pending") {
|
|
248
406
|
throw new Error(
|
|
249
407
|
`subagent ${this.req.childId} is still ${this.status_}; use steer for a running subagent`,
|
|
@@ -255,35 +413,62 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
255
413
|
if (!this.session) {
|
|
256
414
|
throw new Error(`subagent ${this.req.childId} has no session to resume`);
|
|
257
415
|
}
|
|
258
|
-
//
|
|
259
|
-
//
|
|
260
|
-
// the session is dead and cannot be resumed.
|
|
261
|
-
const pendingAbort = this.abortPromise;
|
|
262
|
-
this.abortPromise = null;
|
|
263
|
-
if (pendingAbort && !(await this.awaitAbortBounded(pendingAbort))) {
|
|
264
|
-
throw new Error(
|
|
265
|
-
`subagent ${this.req.childId} session did not go idle after the abort; cannot resume`,
|
|
266
|
-
);
|
|
267
|
-
}
|
|
268
|
-
if (this.disposed) {
|
|
269
|
-
throw new Error(`subagent ${this.req.childId} has been disposed`);
|
|
270
|
-
}
|
|
271
|
-
// A new user turn gets a fresh stall budget and a fresh timeout budget.
|
|
416
|
+
// A new user turn gets a fresh stall budget and a fresh soft budget:
|
|
417
|
+
// the one passed with resume, else the spawn budget.
|
|
272
418
|
this.stallAttempts = 0;
|
|
419
|
+
this.turnBudgetMs =
|
|
420
|
+
opts.timeoutMs !== undefined && opts.timeoutMs > 0 ? opts.timeoutMs : this.req.timeoutMs;
|
|
421
|
+
this.reminders = 0;
|
|
273
422
|
this.runStartedAt = this.clock.now();
|
|
274
|
-
|
|
275
|
-
//
|
|
423
|
+
this.admittedAt = null;
|
|
424
|
+
// The turn's generation exists from here: interrupt()/dispose() while it
|
|
425
|
+
// is queued settle it, and its result promise is observable through
|
|
426
|
+
// registry.getResult() before admission.
|
|
427
|
+
const gen = ++this.generation;
|
|
428
|
+
this.settledFlag = false;
|
|
429
|
+
this.retiredGen = null;
|
|
430
|
+
this.status_ = "pending";
|
|
431
|
+
this.lastEvent = this.runStartedAt;
|
|
276
432
|
this.resultPromise = new Promise((resolve) => {
|
|
277
433
|
this.resolveResult = resolve;
|
|
278
434
|
});
|
|
279
|
-
|
|
435
|
+
const pendingAbort = this.abortPromise;
|
|
436
|
+
this.abortPromise = null;
|
|
437
|
+
void this.startResumedTurn(gen, message, pendingAbort);
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
/**
|
|
441
|
+
* Background half of resume(): drain a stall/timeout abort that is still
|
|
442
|
+
* unwinding (real pi rejects prompt() while the aborted run is active),
|
|
443
|
+
* then admission and prompt. Bounded: a hung abort means the session is
|
|
444
|
+
* dead, and the turn settles failed.
|
|
445
|
+
*/
|
|
446
|
+
private async startResumedTurn(gen: number, message: string, pendingAbort: Promise<void> | null): Promise<void> {
|
|
447
|
+
if (pendingAbort) {
|
|
448
|
+
this.otherStartedAt = this.now();
|
|
449
|
+
const idle = await this.awaitAbortBounded(pendingAbort);
|
|
450
|
+
this.closeOther(this.now());
|
|
451
|
+
if (!idle) {
|
|
452
|
+
this.settle(gen, {
|
|
453
|
+
status: "failed",
|
|
454
|
+
text: "",
|
|
455
|
+
error: `subagent ${this.req.childId} session did not go idle after the abort; cannot resume`,
|
|
456
|
+
durationMs: this.now() - this.runStartedAt,
|
|
457
|
+
});
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
if (this.disposed || this.isSettled(gen)) return;
|
|
462
|
+
await this.beginGeneration(message, false, false, gen);
|
|
280
463
|
}
|
|
281
464
|
|
|
282
465
|
async interrupt(): Promise<void> {
|
|
283
466
|
if (this.status_ !== "running" && this.status_ !== "pending") return;
|
|
284
467
|
this.settle(this.generation, {
|
|
285
468
|
status: "interrupted",
|
|
286
|
-
text
|
|
469
|
+
// A pending turn has produced nothing yet; the last assistant text
|
|
470
|
+
// belongs to the previous turn.
|
|
471
|
+
text: this.status_ === "pending" ? "" : this.partialText(),
|
|
287
472
|
durationMs: this.now() - this.startedAt,
|
|
288
473
|
});
|
|
289
474
|
try {
|
|
@@ -293,7 +478,8 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
293
478
|
}
|
|
294
479
|
}
|
|
295
480
|
|
|
296
|
-
dispose(): void {
|
|
481
|
+
dispose(): void | Promise<void> {
|
|
482
|
+
if (this.disposed) return this.disposalPromise;
|
|
297
483
|
this.disposed = true;
|
|
298
484
|
this.clearTimers();
|
|
299
485
|
if (this.unsubscribe) {
|
|
@@ -306,15 +492,18 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
306
492
|
}
|
|
307
493
|
const session = this.session;
|
|
308
494
|
this.session = null;
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
session.dispose()
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
495
|
+
try {
|
|
496
|
+
this.disposalPromise = session
|
|
497
|
+
? session.dispose() || undefined
|
|
498
|
+
: this.sessionCreation?.then((created) => created.dispose(), () => {});
|
|
499
|
+
return this.disposalPromise;
|
|
500
|
+
} finally {
|
|
501
|
+
// Never leave result waiters hanging, even when cleanup throws or awaits.
|
|
502
|
+
this.settle(this.generation, { status: "interrupted", text: "", error: "disposed", durationMs: 0 });
|
|
503
|
+
// settle() can be a no-op if the child was already interrupted; disposal
|
|
504
|
+
// is terminal and must still return its retained machine permit.
|
|
505
|
+
this.release(true);
|
|
315
506
|
}
|
|
316
|
-
// Never leave result waiters hanging.
|
|
317
|
-
this.settle(this.generation, { status: "interrupted", text: "", error: "disposed", durationMs: 0 });
|
|
318
507
|
}
|
|
319
508
|
|
|
320
509
|
// -------------------------------------------------------------------------
|
|
@@ -326,26 +515,62 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
326
515
|
* slot acquired by the stalled generation is still held (no settle happened),
|
|
327
516
|
* so admission is not re-run and the slot is not double-counted.
|
|
328
517
|
*/
|
|
329
|
-
private async beginGeneration(
|
|
330
|
-
|
|
518
|
+
private async beginGeneration(
|
|
519
|
+
prompt: string,
|
|
520
|
+
first: boolean,
|
|
521
|
+
reuseSlot = false,
|
|
522
|
+
resumedGen?: number,
|
|
523
|
+
): Promise<void> {
|
|
524
|
+
if (this.disposed) return;
|
|
525
|
+
// Stall retries replace a generation without settling the user turn.
|
|
526
|
+
if (reuseSlot) this.pruneToolCalls(this.generation);
|
|
527
|
+
// A resumed turn allocated its generation when it was requested.
|
|
528
|
+
const gen = resumedGen ?? ++this.generation;
|
|
331
529
|
this.retiredGen = null;
|
|
332
530
|
this.clearRetryTimer();
|
|
333
531
|
this.timerScope?.dispose();
|
|
334
532
|
this.timerScope = new TimerScope(this.clock);
|
|
335
533
|
this.settledFlag = false;
|
|
534
|
+
this.terminalSettle = false;
|
|
336
535
|
this.status_ = "pending";
|
|
337
536
|
this.startedAt = this.clock.now();
|
|
338
537
|
this.lastEvent = this.startedAt;
|
|
538
|
+
if (!reuseSlot) {
|
|
539
|
+
this.generationAt = this.startedAt;
|
|
540
|
+
this.generationLlmMs = 0;
|
|
541
|
+
this.generationToolMs = 0;
|
|
542
|
+
this.generationQueueMs = 0;
|
|
543
|
+
this.queueStartedAt = this.acquire ? this.startedAt : null;
|
|
544
|
+
this.queueGeneration = this.queueStartedAt === null ? null : gen;
|
|
545
|
+
this.llmStartedAt = null;
|
|
546
|
+
this.toolStartedAt = null;
|
|
547
|
+
}
|
|
339
548
|
// A tool_execution_end from the previous generation may have been dropped
|
|
340
549
|
// after settle. Don't carry that depth (or a pending decision) into this one.
|
|
341
550
|
this.toolDepth = 0;
|
|
342
551
|
this.decisionPaused = false;
|
|
343
552
|
|
|
553
|
+
if (!reuseSlot) this.generationQueueMs = 0;
|
|
344
554
|
if (this.acquire && !reuseSlot) {
|
|
555
|
+
const admissionAbort = new AbortController();
|
|
556
|
+
this.admissionAbort = admissionAbort;
|
|
345
557
|
try {
|
|
346
|
-
|
|
558
|
+
// The ticket ties the slot request to THIS generation: a request
|
|
559
|
+
// whose generation settled while queued (interrupt, then a new
|
|
560
|
+
// resume) must not admit the child's next generation.
|
|
561
|
+
const release = await this.acquire(this.req, {
|
|
562
|
+
current: () => !this.disposed && !this.isSettled(gen),
|
|
563
|
+
signal: admissionAbort.signal,
|
|
564
|
+
});
|
|
565
|
+
if (this.disposed || this.isSettled(gen)) {
|
|
566
|
+
// A queued interruption is resumable; only disposal is terminal.
|
|
567
|
+
release(this.disposed);
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
this.releaseSlot = release;
|
|
347
571
|
} catch (err) {
|
|
348
572
|
// Admission denied (e.g. fail_fast cancellation while queued).
|
|
573
|
+
this.closeQueue(this.now(), gen);
|
|
349
574
|
this.settle(gen, {
|
|
350
575
|
status: "interrupted",
|
|
351
576
|
text: "",
|
|
@@ -353,18 +578,30 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
353
578
|
durationMs: this.now() - this.startedAt,
|
|
354
579
|
});
|
|
355
580
|
return;
|
|
581
|
+
} finally {
|
|
582
|
+
if (gen === this.generation) {
|
|
583
|
+
// closeQueue()/settle() accounts for this interval exactly once;
|
|
584
|
+
// the finally block only clears state owned by the current generation.
|
|
585
|
+
if (this.admissionAbort === admissionAbort) this.admissionAbort = null;
|
|
586
|
+
}
|
|
356
587
|
}
|
|
588
|
+
this.closeQueue(this.now(), gen);
|
|
357
589
|
}
|
|
358
590
|
if (this.disposed || this.isSettled(gen)) {
|
|
359
|
-
this.release();
|
|
591
|
+
this.release(this.terminalSettle);
|
|
360
592
|
return;
|
|
361
593
|
}
|
|
594
|
+
if (!reuseSlot) this.admittedAt = this.now();
|
|
362
595
|
this.status_ = "running";
|
|
363
596
|
|
|
364
597
|
if (first) {
|
|
365
598
|
try {
|
|
366
|
-
this.
|
|
367
|
-
|
|
599
|
+
this.sessionCreation = this.createSession(this.req);
|
|
600
|
+
const created = await this.sessionCreation;
|
|
601
|
+
// dispose() owns cleanup of an adapter that resolves after teardown.
|
|
602
|
+
if (this.disposed) return;
|
|
603
|
+
this.session = created;
|
|
604
|
+
this.resolvedModel_ = created.resolvedModel;
|
|
368
605
|
} catch (err) {
|
|
369
606
|
this.settle(gen, {
|
|
370
607
|
status: "failed",
|
|
@@ -373,34 +610,14 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
373
610
|
durationMs: this.now() - this.startedAt,
|
|
374
611
|
});
|
|
375
612
|
return;
|
|
613
|
+
} finally {
|
|
614
|
+
this.sessionCreation = null;
|
|
376
615
|
}
|
|
377
616
|
if (this.disposed || this.isSettled(gen)) {
|
|
378
617
|
this.release();
|
|
379
618
|
return;
|
|
380
619
|
}
|
|
381
|
-
this.unsubscribe = this.session.subscribe((event) =>
|
|
382
|
-
if (event.type === "message_end" || event.type === "tool_execution_end" || event.type === "agent_end") {
|
|
383
|
-
this.notifyActivity();
|
|
384
|
-
}
|
|
385
|
-
// Track depth even after settle. A late tool_execution_end must not
|
|
386
|
-
// leak into the next resume, and must not rearm a stale generation.
|
|
387
|
-
if (event.type === "tool_execution_start") {
|
|
388
|
-
this.toolDepth++;
|
|
389
|
-
if (this.status_ === "running") this.clearStall();
|
|
390
|
-
return;
|
|
391
|
-
}
|
|
392
|
-
if (event.type === "tool_execution_end") {
|
|
393
|
-
this.toolDepth = Math.max(0, this.toolDepth - 1);
|
|
394
|
-
this.lastEvent = this.now();
|
|
395
|
-
if (this.status_ === "running" && this.toolDepth === 0 && !this.decisionPaused) {
|
|
396
|
-
this.armStall(this.generation);
|
|
397
|
-
}
|
|
398
|
-
return;
|
|
399
|
-
}
|
|
400
|
-
if (this.status_ !== "running") return;
|
|
401
|
-
this.lastEvent = this.now();
|
|
402
|
-
if (this.toolDepth === 0 && !this.decisionPaused) this.armStall(this.generation);
|
|
403
|
-
});
|
|
620
|
+
this.unsubscribe = this.session.subscribe((event) => this.handleSessionEvent(event));
|
|
404
621
|
}
|
|
405
622
|
|
|
406
623
|
const session = this.session;
|
|
@@ -414,13 +631,19 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
414
631
|
return;
|
|
415
632
|
}
|
|
416
633
|
|
|
417
|
-
// The
|
|
634
|
+
// The turn's deadlines start only after admission and session creation:
|
|
418
635
|
// time spent queued for an admission slot is not the child's budget.
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
636
|
+
// Stall retries (reuseSlot) keep the turn's deadlines and schedule.
|
|
637
|
+
if (!reuseSlot) {
|
|
638
|
+
this.turnBudgetStart = this.now();
|
|
639
|
+
this.softDeadlineAt = this.turnBudgetMs > 0 ? this.turnBudgetStart + this.turnBudgetMs : null;
|
|
640
|
+
this.nextReminderAt = this.softDeadlineAt;
|
|
641
|
+
}
|
|
642
|
+
this.armHardCeiling(gen);
|
|
643
|
+
// A stall can spend the whole hard budget before this generation starts;
|
|
644
|
+
// armHardCeiling settles in that case and prompting must not proceed.
|
|
423
645
|
if (this.isSettled(gen)) return;
|
|
646
|
+
this.armSoftDeadline(gen);
|
|
424
647
|
this.armStall(gen);
|
|
425
648
|
// Tell the child which model it is — otherwise only the parent/fleet knows.
|
|
426
649
|
const prompted =
|
|
@@ -495,7 +718,19 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
495
718
|
|
|
496
719
|
private settle(gen: number, result: ChildResult): void {
|
|
497
720
|
if (gen !== this.generation || this.settledFlag) return;
|
|
721
|
+
const at = this.now();
|
|
722
|
+
this.pruneToolCalls(gen);
|
|
723
|
+
const queueMs = this.generationQueueMs + (
|
|
724
|
+
this.queueStartedAt !== null && this.queueGeneration === gen
|
|
725
|
+
? Math.max(0, at - this.queueStartedAt)
|
|
726
|
+
: 0
|
|
727
|
+
);
|
|
728
|
+
if (result.queueMs === undefined && queueMs > 0) result.queueMs = queueMs;
|
|
729
|
+
this.closeOther(at);
|
|
730
|
+
this.finishWallGeneration(at, gen);
|
|
498
731
|
this.settledFlag = true;
|
|
732
|
+
this.admissionAbort?.abort();
|
|
733
|
+
this.admissionAbort = null;
|
|
499
734
|
this.clearTimers();
|
|
500
735
|
// Generations run: 1 + resumes + stall retries. Forensics for the
|
|
501
736
|
// incident class this exists for (a stalled child that needed retries).
|
|
@@ -503,9 +738,14 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
503
738
|
if (result.attempts === undefined && this.generation > 1) {
|
|
504
739
|
result.attempts = this.generation;
|
|
505
740
|
}
|
|
506
|
-
// durationMs covers
|
|
507
|
-
//
|
|
508
|
-
if (result.error !== "disposed")
|
|
741
|
+
// durationMs covers admitted execution (including stalls and retry delays),
|
|
742
|
+
// but excludes time waiting for an admission slot. dispose keeps its 0.
|
|
743
|
+
if (result.error !== "disposed") {
|
|
744
|
+
result.durationMs = this.admittedAt === null ? 0 : Math.max(0, this.now() - this.admittedAt);
|
|
745
|
+
}
|
|
746
|
+
result.queueMs = this.admittedAt === null
|
|
747
|
+
? Math.max(0, this.now() - this.runStartedAt)
|
|
748
|
+
: Math.max(0, this.admittedAt - this.runStartedAt);
|
|
509
749
|
if (result.stalls === undefined && this.stallAttempts > 0) {
|
|
510
750
|
result.stalls = this.stallAttempts;
|
|
511
751
|
}
|
|
@@ -516,7 +756,8 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
516
756
|
result.warning = this.session.warning;
|
|
517
757
|
}
|
|
518
758
|
this.status_ = result.status;
|
|
519
|
-
this.
|
|
759
|
+
this.terminalSettle = result.status !== "interrupted" || result.error === "disposed";
|
|
760
|
+
this.release(this.terminalSettle);
|
|
520
761
|
this.notifyActivity();
|
|
521
762
|
this.resolveResult(result);
|
|
522
763
|
}
|
|
@@ -529,28 +770,245 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
529
770
|
}
|
|
530
771
|
}
|
|
531
772
|
|
|
532
|
-
private
|
|
773
|
+
private pruneToolCalls(gen: number): void {
|
|
774
|
+
for (const [toolCallId, generation] of this.toolCallGenerations) {
|
|
775
|
+
if (generation === gen) this.toolCallGenerations.delete(toolCallId);
|
|
776
|
+
}
|
|
777
|
+
for (let i = this.anonymousToolGenerations.length - 1; i >= 0; i--) {
|
|
778
|
+
if (this.anonymousToolGenerations[i] === gen) this.anonymousToolGenerations.splice(i, 1);
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
private closeQueue(at: number, gen: number): void {
|
|
783
|
+
if (this.queueStartedAt === null || this.queueGeneration !== gen) return;
|
|
784
|
+
const elapsed = Math.max(0, at - this.queueStartedAt);
|
|
785
|
+
this.wall.queueMs += elapsed;
|
|
786
|
+
this.generationQueueMs += elapsed;
|
|
787
|
+
this.queueStartedAt = null;
|
|
788
|
+
this.queueGeneration = null;
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
private closeLlm(at: number): void {
|
|
792
|
+
if (this.llmStartedAt === null) return;
|
|
793
|
+
const elapsed = Math.max(0, at - this.llmStartedAt);
|
|
794
|
+
this.wall.llmMs += elapsed;
|
|
795
|
+
this.generationLlmMs += elapsed;
|
|
796
|
+
this.llmStartedAt = null;
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
private closeTool(at: number): void {
|
|
800
|
+
if (this.toolStartedAt === null) return;
|
|
801
|
+
const elapsed = Math.max(0, at - this.toolStartedAt);
|
|
802
|
+
this.wall.toolMs += elapsed;
|
|
803
|
+
this.generationToolMs += elapsed;
|
|
804
|
+
this.toolStartedAt = null;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
private closeOther(at: number): void {
|
|
808
|
+
if (this.otherStartedAt === null) return;
|
|
809
|
+
const elapsed = Math.max(0, at - this.otherStartedAt);
|
|
810
|
+
this.wall.otherMs += elapsed;
|
|
811
|
+
if (elapsed > 0) this.wall.approximate = true;
|
|
812
|
+
this.otherStartedAt = null;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
private accountWallEvent(event: { type: string; role?: string; usage?: { input: number; output: number } }, at: number): void {
|
|
816
|
+
const assistant = event.role === "assistant" || (event.role === undefined && event.usage !== undefined);
|
|
817
|
+
if (event.type === "message_start" && assistant) {
|
|
818
|
+
if (this.toolDepth === 0 && this.llmStartedAt === null) this.llmStartedAt = at;
|
|
819
|
+
else this.wall.approximate = true;
|
|
820
|
+
} else if (event.type === "message_end" && assistant) {
|
|
821
|
+
if (this.llmStartedAt === null) this.wall.approximate = true;
|
|
822
|
+
else this.closeLlm(at);
|
|
823
|
+
} else if (event.type === "tool_execution_start") {
|
|
824
|
+
if (this.toolDepth === 0) {
|
|
825
|
+
if (this.llmStartedAt !== null) {
|
|
826
|
+
this.closeLlm(at);
|
|
827
|
+
this.wall.approximate = true;
|
|
828
|
+
}
|
|
829
|
+
this.toolStartedAt = at;
|
|
830
|
+
}
|
|
831
|
+
} else if (event.type === "tool_execution_end") {
|
|
832
|
+
if (this.toolDepth === 0) this.wall.approximate = true;
|
|
833
|
+
else if (this.toolDepth === 1) this.closeTool(at);
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
|
|
837
|
+
private finishWallGeneration(at: number, gen: number): void {
|
|
838
|
+
this.closeQueue(at, gen);
|
|
839
|
+
if (this.llmStartedAt !== null) {
|
|
840
|
+
this.closeLlm(at);
|
|
841
|
+
this.wall.approximate = true;
|
|
842
|
+
}
|
|
843
|
+
if (this.toolStartedAt !== null) {
|
|
844
|
+
this.closeTool(at);
|
|
845
|
+
this.wall.approximate = true;
|
|
846
|
+
}
|
|
847
|
+
if (this.generationAt !== null) {
|
|
848
|
+
const elapsed = Math.max(0, at - this.generationAt);
|
|
849
|
+
const measured = this.generationQueueMs + this.generationLlmMs + this.generationToolMs;
|
|
850
|
+
const other = Math.max(0, elapsed - measured);
|
|
851
|
+
this.wall.otherMs += other;
|
|
852
|
+
if (other > 0) this.wall.approximate = true;
|
|
853
|
+
}
|
|
854
|
+
this.generationAt = null;
|
|
855
|
+
this.generationLlmMs = 0;
|
|
856
|
+
this.generationToolMs = 0;
|
|
857
|
+
this.generationQueueMs = 0;
|
|
858
|
+
this.queueStartedAt = null;
|
|
859
|
+
this.queueGeneration = null;
|
|
860
|
+
this.llmStartedAt = null;
|
|
861
|
+
this.toolStartedAt = null;
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
private handleSessionEvent(event: { type: string; role?: string; toolCallId?: string; usage?: { input: number; output: number } }): void {
|
|
865
|
+
const now = this.now();
|
|
866
|
+
const active = this.status_ === "running";
|
|
867
|
+
let currentToolEvent = false;
|
|
868
|
+
if (event.type === "tool_execution_start" && active) {
|
|
869
|
+
if (event.toolCallId) {
|
|
870
|
+
if (this.toolCallGenerations.has(event.toolCallId)) {
|
|
871
|
+
this.wall.approximate = true;
|
|
872
|
+
} else {
|
|
873
|
+
this.toolCallGenerations.set(event.toolCallId, this.generation);
|
|
874
|
+
currentToolEvent = true;
|
|
875
|
+
}
|
|
876
|
+
} else {
|
|
877
|
+
// Pi emits toolCallId; retain generation ownership for older adapters
|
|
878
|
+
// too, but mark their FIFO pairing as approximate.
|
|
879
|
+
this.anonymousToolGenerations.push(this.generation);
|
|
880
|
+
this.wall.approximate = true;
|
|
881
|
+
currentToolEvent = true;
|
|
882
|
+
}
|
|
883
|
+
} else if (event.type === "tool_execution_end") {
|
|
884
|
+
if (event.toolCallId) {
|
|
885
|
+
const generation = this.toolCallGenerations.get(event.toolCallId);
|
|
886
|
+
this.toolCallGenerations.delete(event.toolCallId);
|
|
887
|
+
currentToolEvent = active && generation === this.generation;
|
|
888
|
+
if (active && generation === undefined) this.wall.approximate = true;
|
|
889
|
+
} else {
|
|
890
|
+
const generation = this.anonymousToolGenerations.shift();
|
|
891
|
+
currentToolEvent = active && generation === this.generation;
|
|
892
|
+
if (active && !currentToolEvent) this.wall.approximate = true;
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
if (active && (!event.type.startsWith("tool_execution_") || currentToolEvent)) {
|
|
896
|
+
this.accountWallEvent(event, now);
|
|
897
|
+
}
|
|
898
|
+
if (
|
|
899
|
+
["message_end", "tool_execution_end", "agent_end"].includes(event.type) ||
|
|
900
|
+
(active && ["message_start", "tool_execution_start"].includes(event.type))
|
|
901
|
+
) {
|
|
902
|
+
this.notifyActivity();
|
|
903
|
+
}
|
|
904
|
+
if (event.type === "tool_execution_start") {
|
|
905
|
+
if (currentToolEvent) {
|
|
906
|
+
this.toolDepth++;
|
|
907
|
+
this.clearStall();
|
|
908
|
+
}
|
|
909
|
+
return;
|
|
910
|
+
}
|
|
911
|
+
if (event.type === "tool_execution_end") {
|
|
912
|
+
if (currentToolEvent) {
|
|
913
|
+
this.toolDepth = Math.max(0, this.toolDepth - 1);
|
|
914
|
+
this.lastEvent = now;
|
|
915
|
+
if (this.toolDepth === 0 && !this.decisionPaused) this.armStall(this.generation);
|
|
916
|
+
}
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
919
|
+
if (!active) return;
|
|
920
|
+
this.lastEvent = now;
|
|
921
|
+
if (this.toolDepth === 0 && !this.decisionPaused) this.armStall(this.generation);
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
private release(terminal = true): void {
|
|
533
925
|
const release = this.releaseSlot;
|
|
534
|
-
this.releaseSlot = null;
|
|
926
|
+
if (terminal) this.releaseSlot = null;
|
|
535
927
|
if (release) {
|
|
536
928
|
try {
|
|
537
|
-
release();
|
|
929
|
+
release(terminal);
|
|
538
930
|
} catch {
|
|
539
931
|
// ignore
|
|
540
932
|
}
|
|
541
933
|
}
|
|
542
934
|
}
|
|
543
935
|
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
936
|
+
/**
|
|
937
|
+
* Soft deadline / reminder timer for the current generation scope, due at
|
|
938
|
+
* nextReminderAt (turn-level, so a stall retry re-arms the same schedule).
|
|
939
|
+
*/
|
|
940
|
+
private armSoftDeadline(gen: number): void {
|
|
941
|
+
if (this.softTimer !== null) {
|
|
942
|
+
this.timerScope?.clearTimeout(this.softTimer);
|
|
943
|
+
this.softTimer = null;
|
|
944
|
+
}
|
|
945
|
+
if (this.nextReminderAt === null) return;
|
|
946
|
+
const delay = Math.max(0, this.nextReminderAt - this.now());
|
|
947
|
+
this.softTimer = this.timerScope?.setTimeout(() => {
|
|
948
|
+
this.softTimer = null;
|
|
949
|
+
// Deliberately NOT isCurrent(): during a stall's retry delay the child
|
|
950
|
+
// is still running from the parent's view, so the reminder is due.
|
|
951
|
+
if (this.settledFlag || this.disposed || gen !== this.generation) return;
|
|
952
|
+
// A pending need_decision already put a supervisor-request wake in
|
|
953
|
+
// front of the parent: hold the reminder (nextReminderAt stays due)
|
|
954
|
+
// until resumeStall() re-arms it when the decision resolves.
|
|
955
|
+
if (this.decisionPaused) return;
|
|
956
|
+
this.fireOverrun(gen);
|
|
957
|
+
}, delay) ?? null;
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
collectDueOverrun(now: number): OverrunTick | undefined {
|
|
961
|
+
if (this.settledFlag || this.disposed || this.status_ !== "running" || this.decisionPaused || this.softDeadlineAt === null) return undefined;
|
|
962
|
+
if (this.softDeadlineAt > now || this.nextReminderAt === null || this.nextReminderAt > now) return undefined;
|
|
963
|
+
if (this.softTimer !== null) this.timerScope?.clearTimeout(this.softTimer);
|
|
964
|
+
this.softTimer = null;
|
|
965
|
+
this.reminders++;
|
|
966
|
+
this.nextReminderAt = now + this.overrunRepeatMs;
|
|
967
|
+
this.armSoftDeadline(this.generation);
|
|
968
|
+
return {
|
|
969
|
+
childId: this.req.childId,
|
|
970
|
+
elapsedMs: now - this.turnBudgetStart,
|
|
971
|
+
budgetMs: this.softDeadlineAt - this.turnBudgetStart,
|
|
972
|
+
reminder: this.reminders,
|
|
973
|
+
nextReminderMs: this.overrunRepeatMs,
|
|
974
|
+
lastEventAt: this.lastEvent,
|
|
975
|
+
hardRemainingMs: this.hardTimeoutMs > 0 ? Math.max(0, this.turnBudgetStart + this.hardTimeoutMs - now) : null,
|
|
976
|
+
};
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
private fireOverrun(gen: number): void {
|
|
980
|
+
const now = this.now();
|
|
981
|
+
this.reminders++;
|
|
982
|
+
this.nextReminderAt = now + this.overrunRepeatMs;
|
|
983
|
+
try {
|
|
984
|
+
this.onOverrun?.({
|
|
985
|
+
childId: this.req.childId,
|
|
986
|
+
elapsedMs: now - this.turnBudgetStart,
|
|
987
|
+
budgetMs: (this.softDeadlineAt ?? now) - this.turnBudgetStart,
|
|
988
|
+
reminder: this.reminders,
|
|
989
|
+
nextReminderMs: this.overrunRepeatMs,
|
|
990
|
+
lastEventAt: this.lastEvent,
|
|
991
|
+
hardRemainingMs:
|
|
992
|
+
this.hardTimeoutMs > 0 ? Math.max(0, this.turnBudgetStart + this.hardTimeoutMs - now) : null,
|
|
993
|
+
});
|
|
994
|
+
} catch {
|
|
995
|
+
// overrun observers must not break the schedule
|
|
996
|
+
}
|
|
997
|
+
this.armSoftDeadline(gen);
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
/**
|
|
1001
|
+
* Opt-in abort ceiling (hardTimeoutMs). The ceiling is per user turn: a
|
|
1002
|
+
* stall retry arms only what the stalled generation did not already
|
|
1003
|
+
* consume, and the budget clock starts after admission (turnBudgetStart).
|
|
1004
|
+
*/
|
|
1005
|
+
private armHardCeiling(gen: number): void {
|
|
1006
|
+
if (this.hardTimer !== null) {
|
|
1007
|
+
this.timerScope?.clearTimeout(this.hardTimer);
|
|
1008
|
+
this.hardTimer = null;
|
|
548
1009
|
}
|
|
549
|
-
if (!(this.
|
|
550
|
-
|
|
551
|
-
// only what the stalled generation did not already consume. The budget
|
|
552
|
-
// clock starts after admission (turnBudgetStart), not at launch.
|
|
553
|
-
const remaining = this.req.timeoutMs - (this.now() - this.turnBudgetStart);
|
|
1010
|
+
if (!(this.hardTimeoutMs > 0)) return;
|
|
1011
|
+
const remaining = this.hardTimeoutMs - (this.now() - this.turnBudgetStart);
|
|
554
1012
|
if (remaining <= 0) {
|
|
555
1013
|
this.settle(gen, {
|
|
556
1014
|
status: "interrupted",
|
|
@@ -560,9 +1018,9 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
560
1018
|
});
|
|
561
1019
|
return;
|
|
562
1020
|
}
|
|
563
|
-
this.
|
|
564
|
-
this.
|
|
565
|
-
// Deliberately NOT isCurrent(): a
|
|
1021
|
+
this.hardTimer = this.timerScope?.setTimeout(() => {
|
|
1022
|
+
this.hardTimer = null;
|
|
1023
|
+
// Deliberately NOT isCurrent(): a ceiling landing during a stall's
|
|
566
1024
|
// retry delay must still fire — the retry has not started a new
|
|
567
1025
|
// generation, and the budget is spent.
|
|
568
1026
|
if (this.settledFlag || this.disposed || gen !== this.generation) return;
|
|
@@ -599,6 +1057,10 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
599
1057
|
*/
|
|
600
1058
|
private handleStall(gen: number): void {
|
|
601
1059
|
this.stallAttempts++;
|
|
1060
|
+
if (this.llmStartedAt !== null) {
|
|
1061
|
+
this.closeLlm(this.now());
|
|
1062
|
+
this.wall.approximate = true;
|
|
1063
|
+
}
|
|
602
1064
|
// Persist whatever the stalled generation produced before it went quiet.
|
|
603
1065
|
this.notifyActivity();
|
|
604
1066
|
try {
|
|
@@ -697,9 +1159,13 @@ class InProcessChildHandle implements DisposableChildHandle {
|
|
|
697
1159
|
}
|
|
698
1160
|
|
|
699
1161
|
private clearTimers(): void {
|
|
700
|
-
if (this.
|
|
701
|
-
this.timerScope?.clearTimeout(this.
|
|
702
|
-
this.
|
|
1162
|
+
if (this.softTimer !== null) {
|
|
1163
|
+
this.timerScope?.clearTimeout(this.softTimer);
|
|
1164
|
+
this.softTimer = null;
|
|
1165
|
+
}
|
|
1166
|
+
if (this.hardTimer !== null) {
|
|
1167
|
+
this.timerScope?.clearTimeout(this.hardTimer);
|
|
1168
|
+
this.hardTimer = null;
|
|
703
1169
|
}
|
|
704
1170
|
this.clearStall();
|
|
705
1171
|
this.clearRetryTimer();
|