pi-famulus 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,13 @@
9
9
  * starts a new generation on the same session):
10
10
  * - a per-generation admission slot is acquired via the optional `acquire`
11
11
  * hook (the registry uses it for the global concurrency cap);
12
- * - `timeoutMs` is a hard timeout: abort -> result {status:"interrupted", error:"timeout"};
12
+ * - `timeoutMs` is a SOFT deadline per user turn: reaching it calls
13
+ * `onOverrun` (the parent gets a subagent-overrun wake) and repeats every
14
+ * `overrunRepeatMs`; the child is not aborted and its shell keeps running.
15
+ * `extend()` re-arms the deadline; steer/followUp after a reminder postpone
16
+ * the next one. See design.md §4.6 "Child lifecycle".
17
+ * - `hardTimeoutMs` (opt-in, default off) is the abort ceiling per turn:
18
+ * abort -> result {status:"interrupted", error:"timeout"};
13
19
  * - a stall watchdog aborts the child after `stallMs` (default 10min) without
14
20
  * any session event. A stall is treated as transient (a silently dropped
15
21
  * provider stream): the child is aborted and auto-resumed on the SAME
@@ -21,9 +27,10 @@
21
27
  * rejects prompt() while a run is still active ("Agent is already
22
28
  * processing"). The wait is bounded by `stallMs` — an abort that never
23
29
  * completes means the stream ignored it and the session is dead.
24
- * - `timeoutMs` is a budget for the whole user turn: stall retries arm only
25
- * the remaining time, and a timeout landing during a retry delay still
26
- * fires (it is not masked by the retired generation).
30
+ * - deadlines belong to the whole user turn: stall retries re-arm the soft
31
+ * deadline / reminder schedule and the hard ceiling from turn-level fields,
32
+ * never from the retry. A reminder landing during a retry delay is sent and
33
+ * the retry still runs; a hard ceiling landing there still settles.
27
34
  * - session/prompt exceptions -> {status:"failed", error}.
28
35
  *
29
36
  * Note on pi semantics: AgentSession.prompt() resolves only after the whole
@@ -32,12 +39,15 @@
32
39
  * steer/followUp processing.
33
40
  */
34
41
  import { realClock, TimerScope, type Clock, type ClockTimer } from "../clock";
42
+ import type { OverrunTick } from "./overrun";
35
43
  import type {
44
+ ChildHandle,
36
45
  ChildResult,
37
46
  ChildRunRequest,
38
47
  ChildRunner,
39
48
  ChildSessionAdapter,
40
49
  ChildStatus,
50
+ ChildWallUsage,
41
51
  CreateSessionFn,
42
52
  DisposableChildHandle,
43
53
  } from "./types";
@@ -48,6 +58,8 @@ export const DEFAULT_STALL_MS = 5 * 60 * 1000;
48
58
  export const DEFAULT_STALL_RETRIES = 1;
49
59
  /** Pause between the stall abort and the retry prompt (ms). */
50
60
  export const DEFAULT_STALL_RETRY_DELAY_MS = 5_000;
61
+ /** Overrun reminder interval while a child stays past its soft budget (ms). */
62
+ export const DEFAULT_OVERRUN_REPEAT_MS = 10 * 60 * 1000;
51
63
 
52
64
  /** Continuation prompt for a stall retry: the transcript holds the context. */
53
65
  export function stallRetryPrompt(stallMs: number): string {
@@ -57,6 +69,14 @@ export function stallRetryPrompt(stallMs: number): string {
57
69
  );
58
70
  }
59
71
 
72
+ /** Identity of one generation's slot request (see InProcessRunnerOptions.acquire). */
73
+ export interface AdmissionTicket {
74
+ /** False once the requesting generation settled or was superseded. */
75
+ current: () => boolean;
76
+ /** Aborted when this generation settles while its admission is queued. */
77
+ signal: AbortSignal;
78
+ }
79
+
60
80
  export interface InProcessRunnerOptions {
61
81
  createSession: CreateSessionFn;
62
82
  /** Stall watchdog timeout (ms). Default 10 minutes. */
@@ -67,14 +87,24 @@ export interface InProcessRunnerOptions {
67
87
  stallRetryDelayMs?: number;
68
88
  /** Called on every stall detection (before any retry), with the 1-based attempt. */
69
89
  onStall?: (childId: string, attempt: number) => void;
90
+ /** Overrun reminder interval (ms). Default 10 minutes. */
91
+ overrunRepeatMs?: number;
92
+ /** Abort ceiling per user turn (ms). 0/absent = off: the soft deadline never aborts. */
93
+ hardTimeoutMs?: number;
94
+ /**
95
+ * The child passed its soft deadline (first reminder) or is still past it
96
+ * (later reminders). The child keeps running; the parent decides.
97
+ */
98
+ onOverrun?: (tick: OverrunTick) => void;
70
99
  /** Shared time source and scheduler. */
71
100
  clock?: Clock;
72
101
  /**
73
102
  * Per-generation admission hook. Awaited before each (re)start; the
74
- * resolved releaser is called when the generation settles. Rejecting
75
- * cancels the generation as {status:"interrupted", error}.
103
+ * releaser gets terminal=false for a resumable interruption so its local
104
+ * slot can be returned while the child-owned machine permit is retained.
105
+ * Rejecting cancels the generation as {status:"interrupted", error}.
76
106
  */
77
- acquire?: (req: ChildRunRequest) => Promise<() => void>;
107
+ acquire?: (req: ChildRunRequest, ticket: AdmissionTicket) => Promise<(terminal?: boolean) => void>;
78
108
  /**
79
109
  * Called after the child's conversation may have changed (a message or
80
110
  * tool finished, or the generation settled). Used to persist transcripts.
@@ -93,8 +123,9 @@ export class InProcessRunner implements ChildRunner {
93
123
  this.opts = opts;
94
124
  }
95
125
 
96
- async start(req: ChildRunRequest): Promise<DisposableChildHandle> {
126
+ async start(req: ChildRunRequest, onHandle?: (handle: ChildHandle) => void): Promise<DisposableChildHandle> {
97
127
  const handle = new InProcessChildHandle(req, this.opts);
128
+ onHandle?.(handle);
98
129
  await handle.launch();
99
130
  return handle;
100
131
  }
@@ -107,25 +138,39 @@ class InProcessChildHandle implements DisposableChildHandle {
107
138
  private readonly stallRetries: number;
108
139
  private readonly stallRetryDelayMs: number;
109
140
  private readonly onStall?: (childId: string, attempt: number) => void;
141
+ private readonly overrunRepeatMs: number;
142
+ private readonly hardTimeoutMs: number;
143
+ private readonly onOverrun?: (tick: OverrunTick) => void;
110
144
  private readonly clock: Clock;
111
- private readonly acquire?: (req: ChildRunRequest) => Promise<() => void>;
145
+ private readonly acquire?: (req: ChildRunRequest, ticket: AdmissionTicket) => Promise<(terminal?: boolean) => void>;
112
146
  private readonly onActivity?: (childId: string) => void;
113
147
 
114
148
  private session: ChildSessionAdapter | null = null;
149
+ private sessionCreation: Promise<ChildSessionAdapter> | null = null;
115
150
  private resolvedModel_: string | undefined;
116
151
  private unsubscribe: (() => void) | null = null;
117
152
  private status_: ChildStatus = "pending";
118
153
  private lastEvent: number;
119
- /** Generation start (reset per generation; durationMs is relative to runStartedAt). */
154
+ /** Session start, set once and preserved across user resumes. */
120
155
  private startedAt: number;
121
156
  /** User-turn start: reset on launch and user resume(), not on stall retries. */
122
157
  private runStartedAt: number;
158
+ /** Timestamp when this generation acquired admission; durationMs is measured from here. */
159
+ private admittedAt: number | null;
123
160
  /**
124
- * When the timeout budget starts counting. Set after admission and session
161
+ * When the turn's deadlines start counting. Set after admission and session
125
162
  * creation (queue wait must not eat the budget), and NOT reset by stall
126
163
  * retries — a stall already consumes budget time by definition.
127
164
  */
128
165
  private turnBudgetStart: number;
166
+ /** This turn's soft budget: spawn timeoutMs, or resume()'s timeoutMs. */
167
+ private turnBudgetMs: number;
168
+ /** Soft deadline (absolute). Moved by extend(). Null = no soft deadline. */
169
+ private softDeadlineAt: number | null = null;
170
+ /** When the soft timer fires next: softDeadlineAt, then each reminder. */
171
+ private nextReminderAt: number | null = null;
172
+ /** Overrun reminders sent this turn. */
173
+ private reminders = 0;
129
174
  /** Abort kicked off by the latest stall detection; awaited (bounded) by the retry. */
130
175
  private abortPromise: Promise<void> | null = null;
131
176
  private generation = 0;
@@ -140,7 +185,8 @@ class InProcessChildHandle implements DisposableChildHandle {
140
185
  * the previous turn's settle by definition).
141
186
  */
142
187
  private settleWaiters: Array<() => void> = [];
143
- private timeoutTimer: ClockTimer | null = null;
188
+ private softTimer: ClockTimer | null = null;
189
+ private hardTimer: ClockTimer | null = null;
144
190
  private stallTimer: ClockTimer | null = null;
145
191
  private retryTimer: { scope: TimerScope; id: ClockTimer } | null = null;
146
192
  private timerScope: TimerScope | null = null;
@@ -148,10 +194,29 @@ class InProcessChildHandle implements DisposableChildHandle {
148
194
  private stallAttempts = 0;
149
195
  /** Generation retired by a stall detection awaiting its retry. */
150
196
  private retiredGen: number | null = null;
151
- private releaseSlot: (() => void) | null = null;
197
+ private releaseSlot: ((terminal?: boolean) => void) | null = null;
198
+ private admissionAbort: AbortController | null = null;
199
+ /** Whether the current generation's settled result is terminal for this child. */
200
+ private terminalSettle = false;
152
201
  private disposed = false;
153
- /** Nested tool_execution_start/end. Stall stays paused while > 0. */
202
+ private disposalPromise: Promise<void> | undefined;
203
+ /** Nested current-generation tool calls. Stall stays paused while > 0. */
154
204
  private toolDepth = 0;
205
+ /** Start generation by Pi toolCallId so late end events cannot close a new interval. */
206
+ private readonly toolCallGenerations = new Map<string, number>();
207
+ /** FIFO fallback for adapters that omit Pi's toolCallId (marked approximate). */
208
+ private readonly anonymousToolGenerations: number[] = [];
209
+ private wall: ChildWallUsage = { llmMs: 0, toolMs: 0, queueMs: 0, otherMs: 0, approximate: false };
210
+ private generationAt: number | null = null;
211
+ private generationLlmMs = 0;
212
+ private generationToolMs = 0;
213
+ private generationQueueMs = 0;
214
+ private queueStartedAt: number | null = null;
215
+ private queueGeneration: number | null = null;
216
+ /** Pending-abort drain before resume is unclassified wall time. */
217
+ private otherStartedAt: number | null = null;
218
+ private llmStartedAt: number | null = null;
219
+ private toolStartedAt: number | null = null;
155
220
  /** contact_supervisor need_decision. Stall stays paused while true. */
156
221
  private decisionPaused = false;
157
222
 
@@ -168,11 +233,21 @@ class InProcessChildHandle implements DisposableChildHandle {
168
233
  ? Math.max(0, Math.floor(rawDelay))
169
234
  : DEFAULT_STALL_RETRY_DELAY_MS;
170
235
  this.onStall = opts.onStall;
236
+ const rawRepeat = opts.overrunRepeatMs ?? DEFAULT_OVERRUN_REPEAT_MS;
237
+ this.overrunRepeatMs =
238
+ Number.isFinite(rawRepeat) && rawRepeat > 0
239
+ ? Math.max(1, Math.floor(rawRepeat))
240
+ : DEFAULT_OVERRUN_REPEAT_MS;
241
+ const rawHard = opts.hardTimeoutMs ?? 0;
242
+ this.hardTimeoutMs = Number.isFinite(rawHard) && rawHard > 0 ? Math.floor(rawHard) : 0;
243
+ this.onOverrun = opts.onOverrun;
244
+ this.turnBudgetMs = req.timeoutMs;
171
245
  this.clock = opts.clock ?? realClock;
172
246
  this.acquire = opts.acquire;
173
247
  this.onActivity = opts.onActivity;
174
248
  this.startedAt = this.clock.now();
175
249
  this.runStartedAt = this.startedAt;
250
+ this.admittedAt = null;
176
251
  this.turnBudgetStart = this.startedAt;
177
252
  this.lastEvent = this.startedAt;
178
253
  this.resultPromise = new Promise((resolve) => {
@@ -201,7 +276,14 @@ class InProcessChildHandle implements DisposableChildHandle {
201
276
  return this.resolvedModel_ ?? this.session?.resolvedModel;
202
277
  }
203
278
 
204
- /** Pause the stall watchdog (need_decision). Nested with tool execution. */
279
+ effectiveThinkingLevel(): string | undefined {
280
+ return this.session?.effectiveThinkingLevel;
281
+ }
282
+
283
+ /**
284
+ * need_decision pending: pause the stall watchdog (nested with tool
285
+ * execution) and hold overrun reminders. The soft budget keeps counting.
286
+ */
205
287
  pauseStall(): void {
206
288
  this.decisionPaused = true;
207
289
  this.clearStall();
@@ -211,12 +293,39 @@ class InProcessChildHandle implements DisposableChildHandle {
211
293
  this.decisionPaused = false;
212
294
  this.lastEvent = this.clock.now();
213
295
  if (this.status_ === "running" && this.toolDepth === 0) this.armStall(this.generation);
296
+ // Deliver a reminder held during the decision (due now, so it fires at
297
+ // once); otherwise this re-arms the unchanged schedule.
298
+ if (this.status_ === "running" && !this.settledFlag) this.armSoftDeadline(this.generation);
214
299
  }
215
300
 
216
301
  conversation() {
217
302
  return this.session?.getConversation() ?? [];
218
303
  }
219
304
 
305
+ tokenUsage() {
306
+ return this.session?.tokenUsage() ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
307
+ }
308
+
309
+ wallUsage(): ChildWallUsage {
310
+ const now = this.clock.now();
311
+ const queueMs = this.queueStartedAt === null ? 0 : Math.max(0, now - this.queueStartedAt);
312
+ const llmMs = this.llmStartedAt === null ? 0 : Math.max(0, now - this.llmStartedAt);
313
+ const toolMs = this.toolStartedAt === null ? 0 : Math.max(0, now - this.toolStartedAt);
314
+ const elapsed = this.generationAt === null ? 0 : Math.max(0, now - this.generationAt);
315
+ const accounted = this.generationQueueMs + queueMs + this.generationLlmMs + llmMs + this.generationToolMs + toolMs;
316
+ const otherMs =
317
+ this.wall.otherMs +
318
+ Math.max(0, elapsed - accounted) +
319
+ (this.otherStartedAt === null ? 0 : Math.max(0, now - this.otherStartedAt));
320
+ return {
321
+ llmMs: this.wall.llmMs + llmMs,
322
+ toolMs: this.wall.toolMs + toolMs,
323
+ queueMs: this.wall.queueMs + queueMs,
324
+ otherMs,
325
+ approximate: this.wall.approximate || otherMs > this.wall.otherMs,
326
+ };
327
+ }
328
+
220
329
  /** Launch generation 1. Resolves once the prompt is issued (not completed). */
221
330
  async launch(): Promise<void> {
222
331
  await this.beginGeneration(this.req.prompt, true);
@@ -230,7 +339,9 @@ class InProcessChildHandle implements DisposableChildHandle {
230
339
  // The previous generation was aborted; delivery would be undefined.
231
340
  throw new Error(`subagent ${this.req.childId} is restarting after a stall; retry shortly`);
232
341
  }
342
+ const gen = this.generation;
233
343
  await this.session.steer(message);
344
+ this.parentActed(gen);
234
345
  }
235
346
 
236
347
  async followUp(message: string): Promise<void> {
@@ -240,10 +351,57 @@ class InProcessChildHandle implements DisposableChildHandle {
240
351
  if (this.retiredGen !== null) {
241
352
  throw new Error(`subagent ${this.req.childId} is restarting after a stall; retry shortly`);
242
353
  }
354
+ const gen = this.generation;
243
355
  await this.session.followUp(message);
356
+ this.parentActed(gen);
357
+ }
358
+
359
+ /**
360
+ * Parent action on an overrun: move the soft deadline to now + timeoutMs
361
+ * (default: the spawn budget). The reminder count continues; the hard
362
+ * ceiling (if any) does not move.
363
+ */
364
+ extend(timeoutMs?: number): { deadlineAt: number; hardDeadlineAt: number | null } {
365
+ if (this.status_ !== "running") {
366
+ throw new Error(`subagent ${this.req.childId} is not running (status: ${this.status_})`);
367
+ }
368
+ const ms = timeoutMs ?? this.req.timeoutMs;
369
+ if (!(ms > 0)) throw new Error("extend needs a positive timeout_ms");
370
+ this.softDeadlineAt = this.now() + ms;
371
+ this.nextReminderAt = this.softDeadlineAt;
372
+ this.armSoftDeadline(this.generation);
373
+ return {
374
+ deadlineAt: this.softDeadlineAt,
375
+ hardDeadlineAt: this.hardTimeoutMs > 0 ? this.turnBudgetStart + this.hardTimeoutMs : null,
376
+ };
377
+ }
378
+
379
+ /**
380
+ * steer/followUp after a reminder: the parent is handling the overrun, so
381
+ * the next reminder waits a full interval. Before the first reminder the
382
+ * deadline is untouched, and so is a deadline that extend() moved into the
383
+ * future: the child is inside its budget again, and the next wake belongs
384
+ * at that deadline, not one repeat from now.
385
+ */
386
+ private parentActed(gen: number): void {
387
+ // The delivery was awaited: if its turn settled (and maybe a resume
388
+ // started a new one) meanwhile, it must not move the new schedule.
389
+ if (gen !== this.generation || this.settledFlag) return;
390
+ if (this.reminders === 0 || this.status_ !== "running") return;
391
+ const now = this.now();
392
+ if (this.softDeadlineAt !== null && now < this.softDeadlineAt) return;
393
+ this.nextReminderAt = now + this.overrunRepeatMs;
394
+ this.armSoftDeadline(this.generation);
244
395
  }
245
396
 
246
- async resume(message: string): Promise<void> {
397
+ /**
398
+ * Start a new user turn. Resolves once the request is accepted, not when
399
+ * the turn starts: draining a still-unwinding abort and waiting for an
400
+ * admission slot run in the background, so the parent's tool call never
401
+ * waits for a slot. Until admission the child is "pending" (queued).
402
+ * Failures after acceptance settle the new turn instead of rejecting.
403
+ */
404
+ async resume(message: string, opts: { timeoutMs?: number } = {}): Promise<void> {
247
405
  if (this.status_ === "running" || this.status_ === "pending") {
248
406
  throw new Error(
249
407
  `subagent ${this.req.childId} is still ${this.status_}; use steer for a running subagent`,
@@ -255,35 +413,62 @@ class InProcessChildHandle implements DisposableChildHandle {
255
413
  if (!this.session) {
256
414
  throw new Error(`subagent ${this.req.childId} has no session to resume`);
257
415
  }
258
- // Drain a stall/timeout abort that is still unwinding (real pi rejects
259
- // prompt() while the aborted run is active). Bounded: a hung abort means
260
- // the session is dead and cannot be resumed.
261
- const pendingAbort = this.abortPromise;
262
- this.abortPromise = null;
263
- if (pendingAbort && !(await this.awaitAbortBounded(pendingAbort))) {
264
- throw new Error(
265
- `subagent ${this.req.childId} session did not go idle after the abort; cannot resume`,
266
- );
267
- }
268
- if (this.disposed) {
269
- throw new Error(`subagent ${this.req.childId} has been disposed`);
270
- }
271
- // A new user turn gets a fresh stall budget and a fresh timeout budget.
416
+ // A new user turn gets a fresh stall budget and a fresh soft budget:
417
+ // the one passed with resume, else the spawn budget.
272
418
  this.stallAttempts = 0;
419
+ this.turnBudgetMs =
420
+ opts.timeoutMs !== undefined && opts.timeoutMs > 0 ? opts.timeoutMs : this.req.timeoutMs;
421
+ this.reminders = 0;
273
422
  this.runStartedAt = this.clock.now();
274
- // Swap in the new generation's result promise synchronously so that
275
- // registry.getResult() observes it before/while admission runs.
423
+ this.admittedAt = null;
424
+ // The turn's generation exists from here: interrupt()/dispose() while it
425
+ // is queued settle it, and its result promise is observable through
426
+ // registry.getResult() before admission.
427
+ const gen = ++this.generation;
428
+ this.settledFlag = false;
429
+ this.retiredGen = null;
430
+ this.status_ = "pending";
431
+ this.lastEvent = this.runStartedAt;
276
432
  this.resultPromise = new Promise((resolve) => {
277
433
  this.resolveResult = resolve;
278
434
  });
279
- await this.beginGeneration(message, false);
435
+ const pendingAbort = this.abortPromise;
436
+ this.abortPromise = null;
437
+ void this.startResumedTurn(gen, message, pendingAbort);
438
+ }
439
+
440
+ /**
441
+ * Background half of resume(): drain a stall/timeout abort that is still
442
+ * unwinding (real pi rejects prompt() while the aborted run is active),
443
+ * then admission and prompt. Bounded: a hung abort means the session is
444
+ * dead, and the turn settles failed.
445
+ */
446
+ private async startResumedTurn(gen: number, message: string, pendingAbort: Promise<void> | null): Promise<void> {
447
+ if (pendingAbort) {
448
+ this.otherStartedAt = this.now();
449
+ const idle = await this.awaitAbortBounded(pendingAbort);
450
+ this.closeOther(this.now());
451
+ if (!idle) {
452
+ this.settle(gen, {
453
+ status: "failed",
454
+ text: "",
455
+ error: `subagent ${this.req.childId} session did not go idle after the abort; cannot resume`,
456
+ durationMs: this.now() - this.runStartedAt,
457
+ });
458
+ return;
459
+ }
460
+ }
461
+ if (this.disposed || this.isSettled(gen)) return;
462
+ await this.beginGeneration(message, false, false, gen);
280
463
  }
281
464
 
282
465
  async interrupt(): Promise<void> {
283
466
  if (this.status_ !== "running" && this.status_ !== "pending") return;
284
467
  this.settle(this.generation, {
285
468
  status: "interrupted",
286
- text: this.partialText(),
469
+ // A pending turn has produced nothing yet; the last assistant text
470
+ // belongs to the previous turn.
471
+ text: this.status_ === "pending" ? "" : this.partialText(),
287
472
  durationMs: this.now() - this.startedAt,
288
473
  });
289
474
  try {
@@ -293,7 +478,8 @@ class InProcessChildHandle implements DisposableChildHandle {
293
478
  }
294
479
  }
295
480
 
296
- dispose(): void {
481
+ dispose(): void | Promise<void> {
482
+ if (this.disposed) return this.disposalPromise;
297
483
  this.disposed = true;
298
484
  this.clearTimers();
299
485
  if (this.unsubscribe) {
@@ -306,15 +492,18 @@ class InProcessChildHandle implements DisposableChildHandle {
306
492
  }
307
493
  const session = this.session;
308
494
  this.session = null;
309
- if (session) {
310
- try {
311
- session.dispose();
312
- } catch {
313
- // ignore
314
- }
495
+ try {
496
+ this.disposalPromise = session
497
+ ? session.dispose() || undefined
498
+ : this.sessionCreation?.then((created) => created.dispose(), () => {});
499
+ return this.disposalPromise;
500
+ } finally {
501
+ // Never leave result waiters hanging, even when cleanup throws or awaits.
502
+ this.settle(this.generation, { status: "interrupted", text: "", error: "disposed", durationMs: 0 });
503
+ // settle() can be a no-op if the child was already interrupted; disposal
504
+ // is terminal and must still return its retained machine permit.
505
+ this.release(true);
315
506
  }
316
- // Never leave result waiters hanging.
317
- this.settle(this.generation, { status: "interrupted", text: "", error: "disposed", durationMs: 0 });
318
507
  }
319
508
 
320
509
  // -------------------------------------------------------------------------
@@ -326,26 +515,62 @@ class InProcessChildHandle implements DisposableChildHandle {
326
515
  * slot acquired by the stalled generation is still held (no settle happened),
327
516
  * so admission is not re-run and the slot is not double-counted.
328
517
  */
329
- private async beginGeneration(prompt: string, first: boolean, reuseSlot = false): Promise<void> {
330
- const gen = ++this.generation;
518
+ private async beginGeneration(
519
+ prompt: string,
520
+ first: boolean,
521
+ reuseSlot = false,
522
+ resumedGen?: number,
523
+ ): Promise<void> {
524
+ if (this.disposed) return;
525
+ // Stall retries replace a generation without settling the user turn.
526
+ if (reuseSlot) this.pruneToolCalls(this.generation);
527
+ // A resumed turn allocated its generation when it was requested.
528
+ const gen = resumedGen ?? ++this.generation;
331
529
  this.retiredGen = null;
332
530
  this.clearRetryTimer();
333
531
  this.timerScope?.dispose();
334
532
  this.timerScope = new TimerScope(this.clock);
335
533
  this.settledFlag = false;
534
+ this.terminalSettle = false;
336
535
  this.status_ = "pending";
337
536
  this.startedAt = this.clock.now();
338
537
  this.lastEvent = this.startedAt;
538
+ if (!reuseSlot) {
539
+ this.generationAt = this.startedAt;
540
+ this.generationLlmMs = 0;
541
+ this.generationToolMs = 0;
542
+ this.generationQueueMs = 0;
543
+ this.queueStartedAt = this.acquire ? this.startedAt : null;
544
+ this.queueGeneration = this.queueStartedAt === null ? null : gen;
545
+ this.llmStartedAt = null;
546
+ this.toolStartedAt = null;
547
+ }
339
548
  // A tool_execution_end from the previous generation may have been dropped
340
549
  // after settle. Don't carry that depth (or a pending decision) into this one.
341
550
  this.toolDepth = 0;
342
551
  this.decisionPaused = false;
343
552
 
553
+ if (!reuseSlot) this.generationQueueMs = 0;
344
554
  if (this.acquire && !reuseSlot) {
555
+ const admissionAbort = new AbortController();
556
+ this.admissionAbort = admissionAbort;
345
557
  try {
346
- this.releaseSlot = await this.acquire(this.req);
558
+ // The ticket ties the slot request to THIS generation: a request
559
+ // whose generation settled while queued (interrupt, then a new
560
+ // resume) must not admit the child's next generation.
561
+ const release = await this.acquire(this.req, {
562
+ current: () => !this.disposed && !this.isSettled(gen),
563
+ signal: admissionAbort.signal,
564
+ });
565
+ if (this.disposed || this.isSettled(gen)) {
566
+ // A queued interruption is resumable; only disposal is terminal.
567
+ release(this.disposed);
568
+ return;
569
+ }
570
+ this.releaseSlot = release;
347
571
  } catch (err) {
348
572
  // Admission denied (e.g. fail_fast cancellation while queued).
573
+ this.closeQueue(this.now(), gen);
349
574
  this.settle(gen, {
350
575
  status: "interrupted",
351
576
  text: "",
@@ -353,18 +578,30 @@ class InProcessChildHandle implements DisposableChildHandle {
353
578
  durationMs: this.now() - this.startedAt,
354
579
  });
355
580
  return;
581
+ } finally {
582
+ if (gen === this.generation) {
583
+ // closeQueue()/settle() accounts for this interval exactly once;
584
+ // the finally block only clears state owned by the current generation.
585
+ if (this.admissionAbort === admissionAbort) this.admissionAbort = null;
586
+ }
356
587
  }
588
+ this.closeQueue(this.now(), gen);
357
589
  }
358
590
  if (this.disposed || this.isSettled(gen)) {
359
- this.release();
591
+ this.release(this.terminalSettle);
360
592
  return;
361
593
  }
594
+ if (!reuseSlot) this.admittedAt = this.now();
362
595
  this.status_ = "running";
363
596
 
364
597
  if (first) {
365
598
  try {
366
- this.session = await this.createSession(this.req);
367
- this.resolvedModel_ = this.session.resolvedModel;
599
+ this.sessionCreation = this.createSession(this.req);
600
+ const created = await this.sessionCreation;
601
+ // dispose() owns cleanup of an adapter that resolves after teardown.
602
+ if (this.disposed) return;
603
+ this.session = created;
604
+ this.resolvedModel_ = created.resolvedModel;
368
605
  } catch (err) {
369
606
  this.settle(gen, {
370
607
  status: "failed",
@@ -373,34 +610,14 @@ class InProcessChildHandle implements DisposableChildHandle {
373
610
  durationMs: this.now() - this.startedAt,
374
611
  });
375
612
  return;
613
+ } finally {
614
+ this.sessionCreation = null;
376
615
  }
377
616
  if (this.disposed || this.isSettled(gen)) {
378
617
  this.release();
379
618
  return;
380
619
  }
381
- this.unsubscribe = this.session.subscribe((event) => {
382
- if (event.type === "message_end" || event.type === "tool_execution_end" || event.type === "agent_end") {
383
- this.notifyActivity();
384
- }
385
- // Track depth even after settle. A late tool_execution_end must not
386
- // leak into the next resume, and must not rearm a stale generation.
387
- if (event.type === "tool_execution_start") {
388
- this.toolDepth++;
389
- if (this.status_ === "running") this.clearStall();
390
- return;
391
- }
392
- if (event.type === "tool_execution_end") {
393
- this.toolDepth = Math.max(0, this.toolDepth - 1);
394
- this.lastEvent = this.now();
395
- if (this.status_ === "running" && this.toolDepth === 0 && !this.decisionPaused) {
396
- this.armStall(this.generation);
397
- }
398
- return;
399
- }
400
- if (this.status_ !== "running") return;
401
- this.lastEvent = this.now();
402
- if (this.toolDepth === 0 && !this.decisionPaused) this.armStall(this.generation);
403
- });
620
+ this.unsubscribe = this.session.subscribe((event) => this.handleSessionEvent(event));
404
621
  }
405
622
 
406
623
  const session = this.session;
@@ -414,13 +631,19 @@ class InProcessChildHandle implements DisposableChildHandle {
414
631
  return;
415
632
  }
416
633
 
417
- // The timeout budget starts only after admission and session creation:
634
+ // The turn's deadlines start only after admission and session creation:
418
635
  // time spent queued for an admission slot is not the child's budget.
419
- if (!reuseSlot) this.turnBudgetStart = this.now();
420
- this.armTimeout(gen);
421
- // A stall can spend the whole timeout budget before this generation
422
- // starts; armTimeout settles in that case and prompting must not proceed.
636
+ // Stall retries (reuseSlot) keep the turn's deadlines and schedule.
637
+ if (!reuseSlot) {
638
+ this.turnBudgetStart = this.now();
639
+ this.softDeadlineAt = this.turnBudgetMs > 0 ? this.turnBudgetStart + this.turnBudgetMs : null;
640
+ this.nextReminderAt = this.softDeadlineAt;
641
+ }
642
+ this.armHardCeiling(gen);
643
+ // A stall can spend the whole hard budget before this generation starts;
644
+ // armHardCeiling settles in that case and prompting must not proceed.
423
645
  if (this.isSettled(gen)) return;
646
+ this.armSoftDeadline(gen);
424
647
  this.armStall(gen);
425
648
  // Tell the child which model it is — otherwise only the parent/fleet knows.
426
649
  const prompted =
@@ -495,7 +718,19 @@ class InProcessChildHandle implements DisposableChildHandle {
495
718
 
496
719
  private settle(gen: number, result: ChildResult): void {
497
720
  if (gen !== this.generation || this.settledFlag) return;
721
+ const at = this.now();
722
+ this.pruneToolCalls(gen);
723
+ const queueMs = this.generationQueueMs + (
724
+ this.queueStartedAt !== null && this.queueGeneration === gen
725
+ ? Math.max(0, at - this.queueStartedAt)
726
+ : 0
727
+ );
728
+ if (result.queueMs === undefined && queueMs > 0) result.queueMs = queueMs;
729
+ this.closeOther(at);
730
+ this.finishWallGeneration(at, gen);
498
731
  this.settledFlag = true;
732
+ this.admissionAbort?.abort();
733
+ this.admissionAbort = null;
499
734
  this.clearTimers();
500
735
  // Generations run: 1 + resumes + stall retries. Forensics for the
501
736
  // incident class this exists for (a stalled child that needed retries).
@@ -503,9 +738,14 @@ class InProcessChildHandle implements DisposableChildHandle {
503
738
  if (result.attempts === undefined && this.generation > 1) {
504
739
  result.attempts = this.generation;
505
740
  }
506
- // durationMs covers the whole user turn (launch/resume -> settle),
507
- // including time lost to stalls and retry delays. dispose keeps its 0.
508
- if (result.error !== "disposed") result.durationMs = this.now() - this.runStartedAt;
741
+ // durationMs covers admitted execution (including stalls and retry delays),
742
+ // but excludes time waiting for an admission slot. dispose keeps its 0.
743
+ if (result.error !== "disposed") {
744
+ result.durationMs = this.admittedAt === null ? 0 : Math.max(0, this.now() - this.admittedAt);
745
+ }
746
+ result.queueMs = this.admittedAt === null
747
+ ? Math.max(0, this.now() - this.runStartedAt)
748
+ : Math.max(0, this.admittedAt - this.runStartedAt);
509
749
  if (result.stalls === undefined && this.stallAttempts > 0) {
510
750
  result.stalls = this.stallAttempts;
511
751
  }
@@ -516,7 +756,8 @@ class InProcessChildHandle implements DisposableChildHandle {
516
756
  result.warning = this.session.warning;
517
757
  }
518
758
  this.status_ = result.status;
519
- this.release();
759
+ this.terminalSettle = result.status !== "interrupted" || result.error === "disposed";
760
+ this.release(this.terminalSettle);
520
761
  this.notifyActivity();
521
762
  this.resolveResult(result);
522
763
  }
@@ -529,28 +770,245 @@ class InProcessChildHandle implements DisposableChildHandle {
529
770
  }
530
771
  }
531
772
 
532
- private release(): void {
773
+ private pruneToolCalls(gen: number): void {
774
+ for (const [toolCallId, generation] of this.toolCallGenerations) {
775
+ if (generation === gen) this.toolCallGenerations.delete(toolCallId);
776
+ }
777
+ for (let i = this.anonymousToolGenerations.length - 1; i >= 0; i--) {
778
+ if (this.anonymousToolGenerations[i] === gen) this.anonymousToolGenerations.splice(i, 1);
779
+ }
780
+ }
781
+
782
+ private closeQueue(at: number, gen: number): void {
783
+ if (this.queueStartedAt === null || this.queueGeneration !== gen) return;
784
+ const elapsed = Math.max(0, at - this.queueStartedAt);
785
+ this.wall.queueMs += elapsed;
786
+ this.generationQueueMs += elapsed;
787
+ this.queueStartedAt = null;
788
+ this.queueGeneration = null;
789
+ }
790
+
791
+ private closeLlm(at: number): void {
792
+ if (this.llmStartedAt === null) return;
793
+ const elapsed = Math.max(0, at - this.llmStartedAt);
794
+ this.wall.llmMs += elapsed;
795
+ this.generationLlmMs += elapsed;
796
+ this.llmStartedAt = null;
797
+ }
798
+
799
+ private closeTool(at: number): void {
800
+ if (this.toolStartedAt === null) return;
801
+ const elapsed = Math.max(0, at - this.toolStartedAt);
802
+ this.wall.toolMs += elapsed;
803
+ this.generationToolMs += elapsed;
804
+ this.toolStartedAt = null;
805
+ }
806
+
807
+ private closeOther(at: number): void {
808
+ if (this.otherStartedAt === null) return;
809
+ const elapsed = Math.max(0, at - this.otherStartedAt);
810
+ this.wall.otherMs += elapsed;
811
+ if (elapsed > 0) this.wall.approximate = true;
812
+ this.otherStartedAt = null;
813
+ }
814
+
815
+ private accountWallEvent(event: { type: string; role?: string; usage?: { input: number; output: number } }, at: number): void {
816
+ const assistant = event.role === "assistant" || (event.role === undefined && event.usage !== undefined);
817
+ if (event.type === "message_start" && assistant) {
818
+ if (this.toolDepth === 0 && this.llmStartedAt === null) this.llmStartedAt = at;
819
+ else this.wall.approximate = true;
820
+ } else if (event.type === "message_end" && assistant) {
821
+ if (this.llmStartedAt === null) this.wall.approximate = true;
822
+ else this.closeLlm(at);
823
+ } else if (event.type === "tool_execution_start") {
824
+ if (this.toolDepth === 0) {
825
+ if (this.llmStartedAt !== null) {
826
+ this.closeLlm(at);
827
+ this.wall.approximate = true;
828
+ }
829
+ this.toolStartedAt = at;
830
+ }
831
+ } else if (event.type === "tool_execution_end") {
832
+ if (this.toolDepth === 0) this.wall.approximate = true;
833
+ else if (this.toolDepth === 1) this.closeTool(at);
834
+ }
835
+ }
836
+
837
+ private finishWallGeneration(at: number, gen: number): void {
838
+ this.closeQueue(at, gen);
839
+ if (this.llmStartedAt !== null) {
840
+ this.closeLlm(at);
841
+ this.wall.approximate = true;
842
+ }
843
+ if (this.toolStartedAt !== null) {
844
+ this.closeTool(at);
845
+ this.wall.approximate = true;
846
+ }
847
+ if (this.generationAt !== null) {
848
+ const elapsed = Math.max(0, at - this.generationAt);
849
+ const measured = this.generationQueueMs + this.generationLlmMs + this.generationToolMs;
850
+ const other = Math.max(0, elapsed - measured);
851
+ this.wall.otherMs += other;
852
+ if (other > 0) this.wall.approximate = true;
853
+ }
854
+ this.generationAt = null;
855
+ this.generationLlmMs = 0;
856
+ this.generationToolMs = 0;
857
+ this.generationQueueMs = 0;
858
+ this.queueStartedAt = null;
859
+ this.queueGeneration = null;
860
+ this.llmStartedAt = null;
861
+ this.toolStartedAt = null;
862
+ }
863
+
864
+ private handleSessionEvent(event: { type: string; role?: string; toolCallId?: string; usage?: { input: number; output: number } }): void {
865
+ const now = this.now();
866
+ const active = this.status_ === "running";
867
+ let currentToolEvent = false;
868
+ if (event.type === "tool_execution_start" && active) {
869
+ if (event.toolCallId) {
870
+ if (this.toolCallGenerations.has(event.toolCallId)) {
871
+ this.wall.approximate = true;
872
+ } else {
873
+ this.toolCallGenerations.set(event.toolCallId, this.generation);
874
+ currentToolEvent = true;
875
+ }
876
+ } else {
877
+ // Pi emits toolCallId; retain generation ownership for older adapters
878
+ // too, but mark their FIFO pairing as approximate.
879
+ this.anonymousToolGenerations.push(this.generation);
880
+ this.wall.approximate = true;
881
+ currentToolEvent = true;
882
+ }
883
+ } else if (event.type === "tool_execution_end") {
884
+ if (event.toolCallId) {
885
+ const generation = this.toolCallGenerations.get(event.toolCallId);
886
+ this.toolCallGenerations.delete(event.toolCallId);
887
+ currentToolEvent = active && generation === this.generation;
888
+ if (active && generation === undefined) this.wall.approximate = true;
889
+ } else {
890
+ const generation = this.anonymousToolGenerations.shift();
891
+ currentToolEvent = active && generation === this.generation;
892
+ if (active && !currentToolEvent) this.wall.approximate = true;
893
+ }
894
+ }
895
+ if (active && (!event.type.startsWith("tool_execution_") || currentToolEvent)) {
896
+ this.accountWallEvent(event, now);
897
+ }
898
+ if (
899
+ ["message_end", "tool_execution_end", "agent_end"].includes(event.type) ||
900
+ (active && ["message_start", "tool_execution_start"].includes(event.type))
901
+ ) {
902
+ this.notifyActivity();
903
+ }
904
+ if (event.type === "tool_execution_start") {
905
+ if (currentToolEvent) {
906
+ this.toolDepth++;
907
+ this.clearStall();
908
+ }
909
+ return;
910
+ }
911
+ if (event.type === "tool_execution_end") {
912
+ if (currentToolEvent) {
913
+ this.toolDepth = Math.max(0, this.toolDepth - 1);
914
+ this.lastEvent = now;
915
+ if (this.toolDepth === 0 && !this.decisionPaused) this.armStall(this.generation);
916
+ }
917
+ return;
918
+ }
919
+ if (!active) return;
920
+ this.lastEvent = now;
921
+ if (this.toolDepth === 0 && !this.decisionPaused) this.armStall(this.generation);
922
+ }
923
+
924
+ private release(terminal = true): void {
533
925
  const release = this.releaseSlot;
534
- this.releaseSlot = null;
926
+ if (terminal) this.releaseSlot = null;
535
927
  if (release) {
536
928
  try {
537
- release();
929
+ release(terminal);
538
930
  } catch {
539
931
  // ignore
540
932
  }
541
933
  }
542
934
  }
543
935
 
544
- private armTimeout(gen: number): void {
545
- if (this.timeoutTimer !== null) {
546
- this.timerScope?.clearTimeout(this.timeoutTimer);
547
- this.timeoutTimer = null;
936
+ /**
937
+ * Soft deadline / reminder timer for the current generation scope, due at
938
+ * nextReminderAt (turn-level, so a stall retry re-arms the same schedule).
939
+ */
940
+ private armSoftDeadline(gen: number): void {
941
+ if (this.softTimer !== null) {
942
+ this.timerScope?.clearTimeout(this.softTimer);
943
+ this.softTimer = null;
944
+ }
945
+ if (this.nextReminderAt === null) return;
946
+ const delay = Math.max(0, this.nextReminderAt - this.now());
947
+ this.softTimer = this.timerScope?.setTimeout(() => {
948
+ this.softTimer = null;
949
+ // Deliberately NOT isCurrent(): during a stall's retry delay the child
950
+ // is still running from the parent's view, so the reminder is due.
951
+ if (this.settledFlag || this.disposed || gen !== this.generation) return;
952
+ // A pending need_decision already put a supervisor-request wake in
953
+ // front of the parent: hold the reminder (nextReminderAt stays due)
954
+ // until resumeStall() re-arms it when the decision resolves.
955
+ if (this.decisionPaused) return;
956
+ this.fireOverrun(gen);
957
+ }, delay) ?? null;
958
+ }
959
+
960
+ collectDueOverrun(now: number): OverrunTick | undefined {
961
+ if (this.settledFlag || this.disposed || this.status_ !== "running" || this.decisionPaused || this.softDeadlineAt === null) return undefined;
962
+ if (this.softDeadlineAt > now || this.nextReminderAt === null || this.nextReminderAt > now) return undefined;
963
+ if (this.softTimer !== null) this.timerScope?.clearTimeout(this.softTimer);
964
+ this.softTimer = null;
965
+ this.reminders++;
966
+ this.nextReminderAt = now + this.overrunRepeatMs;
967
+ this.armSoftDeadline(this.generation);
968
+ return {
969
+ childId: this.req.childId,
970
+ elapsedMs: now - this.turnBudgetStart,
971
+ budgetMs: this.softDeadlineAt - this.turnBudgetStart,
972
+ reminder: this.reminders,
973
+ nextReminderMs: this.overrunRepeatMs,
974
+ lastEventAt: this.lastEvent,
975
+ hardRemainingMs: this.hardTimeoutMs > 0 ? Math.max(0, this.turnBudgetStart + this.hardTimeoutMs - now) : null,
976
+ };
977
+ }
978
+
979
+ private fireOverrun(gen: number): void {
980
+ const now = this.now();
981
+ this.reminders++;
982
+ this.nextReminderAt = now + this.overrunRepeatMs;
983
+ try {
984
+ this.onOverrun?.({
985
+ childId: this.req.childId,
986
+ elapsedMs: now - this.turnBudgetStart,
987
+ budgetMs: (this.softDeadlineAt ?? now) - this.turnBudgetStart,
988
+ reminder: this.reminders,
989
+ nextReminderMs: this.overrunRepeatMs,
990
+ lastEventAt: this.lastEvent,
991
+ hardRemainingMs:
992
+ this.hardTimeoutMs > 0 ? Math.max(0, this.turnBudgetStart + this.hardTimeoutMs - now) : null,
993
+ });
994
+ } catch {
995
+ // overrun observers must not break the schedule
996
+ }
997
+ this.armSoftDeadline(gen);
998
+ }
999
+
1000
+ /**
1001
+ * Opt-in abort ceiling (hardTimeoutMs). The ceiling is per user turn: a
1002
+ * stall retry arms only what the stalled generation did not already
1003
+ * consume, and the budget clock starts after admission (turnBudgetStart).
1004
+ */
1005
+ private armHardCeiling(gen: number): void {
1006
+ if (this.hardTimer !== null) {
1007
+ this.timerScope?.clearTimeout(this.hardTimer);
1008
+ this.hardTimer = null;
548
1009
  }
549
- if (!(this.req.timeoutMs > 0)) return;
550
- // The timeout is a budget for the whole user turn: a stall retry arms
551
- // only what the stalled generation did not already consume. The budget
552
- // clock starts after admission (turnBudgetStart), not at launch.
553
- const remaining = this.req.timeoutMs - (this.now() - this.turnBudgetStart);
1010
+ if (!(this.hardTimeoutMs > 0)) return;
1011
+ const remaining = this.hardTimeoutMs - (this.now() - this.turnBudgetStart);
554
1012
  if (remaining <= 0) {
555
1013
  this.settle(gen, {
556
1014
  status: "interrupted",
@@ -560,9 +1018,9 @@ class InProcessChildHandle implements DisposableChildHandle {
560
1018
  });
561
1019
  return;
562
1020
  }
563
- this.timeoutTimer = this.timerScope?.setTimeout(() => {
564
- this.timeoutTimer = null;
565
- // Deliberately NOT isCurrent(): a timeout landing during a stall's
1021
+ this.hardTimer = this.timerScope?.setTimeout(() => {
1022
+ this.hardTimer = null;
1023
+ // Deliberately NOT isCurrent(): a ceiling landing during a stall's
566
1024
  // retry delay must still fire — the retry has not started a new
567
1025
  // generation, and the budget is spent.
568
1026
  if (this.settledFlag || this.disposed || gen !== this.generation) return;
@@ -599,6 +1057,10 @@ class InProcessChildHandle implements DisposableChildHandle {
599
1057
  */
600
1058
  private handleStall(gen: number): void {
601
1059
  this.stallAttempts++;
1060
+ if (this.llmStartedAt !== null) {
1061
+ this.closeLlm(this.now());
1062
+ this.wall.approximate = true;
1063
+ }
602
1064
  // Persist whatever the stalled generation produced before it went quiet.
603
1065
  this.notifyActivity();
604
1066
  try {
@@ -697,9 +1159,13 @@ class InProcessChildHandle implements DisposableChildHandle {
697
1159
  }
698
1160
 
699
1161
  private clearTimers(): void {
700
- if (this.timeoutTimer !== null) {
701
- this.timerScope?.clearTimeout(this.timeoutTimer);
702
- this.timeoutTimer = null;
1162
+ if (this.softTimer !== null) {
1163
+ this.timerScope?.clearTimeout(this.softTimer);
1164
+ this.softTimer = null;
1165
+ }
1166
+ if (this.hardTimer !== null) {
1167
+ this.timerScope?.clearTimeout(this.hardTimer);
1168
+ this.hardTimer = null;
703
1169
  }
704
1170
  this.clearStall();
705
1171
  this.clearRetryTimer();