pi-long-task 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,7 +10,19 @@ import type {
10
10
  } from "./types.ts";
11
11
  import { commitAfterSession, gitDirtyPaths, shouldCommitOutcome, type CommitAfterSessionResult } from "./git.ts";
12
12
  import { formatCoordinatorResultMessage } from "./render.ts";
13
- import { extractResultSummary } from "./result_writer.ts";
13
+ import { classifyNetworkFailure } from "./network_failure.ts";
14
+ import {
15
+ formatNetworkRecoveryStatus,
16
+ recoverNetworkOperation,
17
+ type NetworkRecoveryEvent,
18
+ type NetworkRecoveryEventType,
19
+ } from "./network_recovery.ts";
20
+ import {
21
+ DEFAULT_NETWORK_RECOVERY_CONFIG,
22
+ resolveNetworkRecoveryConfig,
23
+ type NetworkRecoveryConfig,
24
+ } from "./network_recovery_config.ts";
25
+ import { extractResultSummary, hasCompleteTaskResult } from "./result_writer.ts";
14
26
  import { runGuardedSessionPrompt } from "./session_guard.ts";
15
27
  import {
16
28
  generatePlanRevision,
@@ -28,6 +40,16 @@ import {
28
40
  } from "./plan_store.ts";
29
41
  import type { SerializedSteeringQueue, SteeringMessage } from "./steering.ts";
30
42
  import { parseWorkerRuntimeConfig } from "./worker_config.ts";
43
+ import {
44
+ classifyWorkerSessionRetry,
45
+ createWorkerSessionCompatibilityFingerprint,
46
+ decideWorkerSessionReuse,
47
+ DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
48
+ DEFAULT_WORKER_SESSION_REUSE_ENABLED,
49
+ resolveWorkerSessionReuseConfig,
50
+ type WorkerSessionCompatibilityFingerprint,
51
+ type WorkerSessionHealth,
52
+ } from "./worker_reuse_policy.ts";
31
53
  import { buildTaskProgressModel, type TaskProgressModel, type TaskProgressStatus } from "./task_progress.ts";
32
54
  import {
33
55
  applyGoalInstructionsToTodoMarkdown,
@@ -40,12 +62,21 @@ import {
40
62
  } from "./todo_generator.ts";
41
63
  import { parseTasks, todoGlobalInstructions, type Task } from "./todo_parser.ts";
42
64
  import {
65
+ buildWorkerSessionCreationFailureOutcome,
43
66
  createIsolatedWorkerSession,
67
+ createWorkerSessionResource,
68
+ DEFAULT_WORKER_TOOLS,
69
+ disposeWorkerSessionResource,
44
70
  runWorkerTask,
71
+ runWorkerTaskAssignment,
72
+ workerSessionContextUsagePercent,
45
73
  type RunWorkerTaskOptions,
46
74
  type SessionOutcome,
75
+ type WorkerSessionDiagnostic,
47
76
  type WorkerSessionFactory,
48
77
  type WorkerSessionLike,
78
+ type WorkerSessionResource,
79
+ type WorkerUsageTotals,
49
80
  } from "./worker_session.ts";
50
81
 
51
82
  export type { CoordinatorStatus } from "./types.ts";
@@ -58,6 +89,9 @@ export const DEFAULT_COORDINATOR_OPTIONS = {
58
89
  maxBashTimeoutMs: 300_000,
59
90
  taskThinking: "high",
60
91
  todoThinking: "xhigh",
92
+ workerSessionReuse: DEFAULT_WORKER_SESSION_REUSE_ENABLED,
93
+ workerSessionReuseContextThresholdPercent: DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
94
+ networkRecovery: DEFAULT_NETWORK_RECOVERY_CONFIG,
61
95
  } as const;
62
96
 
63
97
  export type WorkerRunner = (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
@@ -65,7 +99,9 @@ export type CoordinatorProgressPhase =
65
99
  | "planning"
66
100
  | "planned"
67
101
  | "task_start"
102
+ | "worker_session"
68
103
  | "worker_tool"
104
+ | "network_wait"
69
105
  | "task_done"
70
106
  | "task_blocked"
71
107
  | "task_failed"
@@ -124,6 +160,16 @@ export interface CoordinatorProgressUpdate {
124
160
  plannerDiagnostics?: string[];
125
161
  plannerSessionFile?: string;
126
162
  plannerSessionId?: string;
163
+ workerSessionEvent?: WorkerSessionDiagnostic["event"];
164
+ workerSessionReason?: string;
165
+ workerSessionContextUsagePercent?: number;
166
+ workerSessionContextThresholdPercent?: number;
167
+ networkRecoveryEvent?: NetworkRecoveryEventType;
168
+ networkRetryCount?: number;
169
+ networkOutageElapsedMs?: number;
170
+ networkNextRetryAtMs?: number;
171
+ networkNextRetryInMs?: number;
172
+ networkFailureReason?: string;
127
173
  }
128
174
 
129
175
  export type CoordinatorProgressHandler = (update: CoordinatorProgressUpdate) => void;
@@ -146,12 +192,18 @@ export interface RunCoordinatorOptions extends PiLongTaskInput {
146
192
  maxBashTimeoutMs?: number;
147
193
  taskThinking?: string;
148
194
  todoThinking?: string;
195
+ /** Set false to retain the legacy one-session-per-task lifecycle. */
196
+ workerSessionReuse?: boolean;
197
+ /** Rotate before another assignment when context usage reaches this percentage. */
198
+ workerSessionReuseContextThresholdPercent?: number;
149
199
  now?: () => Date;
150
200
  onProgress?: CoordinatorProgressHandler;
151
201
  /** Run-scoped FIFO populated by the extension input handler during active execution. */
152
202
  steeringQueue?: SerializedSteeringQueue;
153
203
  /** Runs after rebase/validation and immediately before the revision is atomically persisted. */
154
204
  onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
205
+ /** Receives coordinator-level outage lifecycle events for parent orchestrators and status integrations. */
206
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
155
207
  }
156
208
 
157
209
  export interface TodoPlannerOptions {
@@ -170,6 +222,8 @@ export interface TodoPlannerOptions {
170
222
  plannerPrompt?: string;
171
223
  /** Structured revision context supplied alongside plannerPrompt. */
172
224
  planRevision?: Readonly<PlanRevisionRequest>;
225
+ /** Normalized coordinator recovery policy; network wait is excluded from operation timeouts. */
226
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
173
227
  }
174
228
 
175
229
  export interface TaskAttemptSummary {
@@ -190,6 +244,14 @@ export interface TaskAttemptSummary {
190
244
  resultText?: string;
191
245
  }
192
246
 
247
+ export interface WorkerSessionMetrics {
248
+ starts: number;
249
+ reuses: number;
250
+ rotations: number;
251
+ retained: number;
252
+ rotationReasons: Record<string, number>;
253
+ }
254
+
193
255
  export interface CoordinatorResult {
194
256
  status: CoordinatorStatus;
195
257
  summary: string;
@@ -210,6 +272,10 @@ export interface CoordinatorResult {
210
272
  attempts: TaskAttemptSummary[];
211
273
  taskProgress: TaskProgressModel;
212
274
  workerCostTotal: number;
275
+ /** Sum of task/attempt token deltas; omitted when statistics are unavailable. */
276
+ workerUsageTotal?: WorkerUsageTotals;
277
+ /** Additive lifecycle counters for adaptive worker-session reuse. */
278
+ workerSessionMetrics?: WorkerSessionMetrics;
213
279
  commit: boolean;
214
280
  goal?: string;
215
281
  error?: string;
@@ -236,9 +302,13 @@ interface RuntimeOptions {
236
302
  goal?: string;
237
303
  taskThinking: string;
238
304
  todoThinking: string;
305
+ workerSessionReuse: boolean;
306
+ workerSessionReuseContextThresholdPercent: number;
307
+ networkRecovery: NetworkRecoveryConfig;
239
308
  todoTimeoutMs: number;
240
309
  todoGracefulShutdownMs: number;
241
310
  workerRunner: WorkerRunner;
311
+ useRetainedWorkerLifecycle: boolean;
242
312
  todoPlanner: TodoPlanner;
243
313
  abortSignal?: AbortSignal;
244
314
  workerSessionFactory?: WorkerSessionFactory;
@@ -250,12 +320,636 @@ interface RuntimeOptions {
250
320
  workerTextByWorker: Map<string, string>;
251
321
  workerTextPublishedLengthByWorker: Map<string, number>;
252
322
  plannerDiagnostics: PlannerDiagnostic[];
323
+ workerSessionMetrics: WorkerSessionMetrics;
253
324
  steeringQueue?: SerializedSteeringQueue;
254
325
  onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
326
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
327
+ lastProgress?: CoordinatorProgressUpdate;
328
+ progressClosed: boolean;
329
+ networkRecoverySequence: number;
330
+ activeNetworkRecoveries: Map<number, NetworkRecoveryEvent>;
331
+ }
332
+
333
+ type RetainedWorkerReuseScope = "sequential_task" | "partial_continuation";
334
+
335
+ export interface WorkerAssignmentIdentity {
336
+ /** Unique invocation identity, even when a replacement reuses a task ID and attempt number. */
337
+ assignmentId: string;
338
+ /** Stable task identity or semantic fingerprint from the plan that launched this assignment. */
339
+ taskIdentity: string;
340
+ /** Accepted-steering generation at the assignment boundary. */
341
+ steeringGeneration: number;
342
+ /** Structural plan generation from which the assignment was selected. */
343
+ planAuthorityToken: string;
344
+ }
345
+
346
+ interface RetainedWorkerState {
347
+ resource: WorkerSessionResource;
348
+ compatibility: WorkerSessionCompatibilityFingerprint;
349
+ health: WorkerSessionHealth;
350
+ contextUsagePercent?: number;
351
+ previousTask: Pick<Task, "taskId" | "title">;
352
+ previousAttempt: number;
353
+ previousAssignmentIdentity: WorkerAssignmentIdentity;
354
+ reportDiagnostic: (diagnostic: WorkerSessionDiagnostic) => void;
355
+ reuseScope: RetainedWorkerReuseScope;
356
+ }
357
+
358
+ interface ActiveWorkerSessionAssignment {
359
+ identity: WorkerAssignmentIdentity;
360
+ controller: AbortController;
361
+ tainted: boolean;
362
+ rotationReported: boolean;
363
+ resource?: WorkerSessionResource;
364
+ reportDiagnostic?: (diagnostic: WorkerSessionDiagnostic) => void;
365
+ resolveCompletion: () => void;
366
+ completion: Promise<void>;
367
+ }
368
+
369
+ export interface CoordinatorWorkerSessionOwnerOptions {
370
+ runId: string;
371
+ cwd: string;
372
+ workerSessionReuse: boolean;
373
+ workerSessionReuseContextThresholdPercent: number;
374
+ }
375
+
376
+ /** Run-scoped owner for the single retained worker session and its assignment lock. */
377
+ export class CoordinatorWorkerSessionOwner {
378
+ private retained: RetainedWorkerState | undefined;
379
+ private active: ActiveWorkerSessionAssignment | undefined;
380
+ private closed = false;
381
+ private disposePromise: Promise<void> | undefined;
382
+ private assignmentSequence = 0;
383
+ private readonly runtime: CoordinatorWorkerSessionOwnerOptions;
384
+
385
+ constructor(runtime: CoordinatorWorkerSessionOwnerOptions) {
386
+ this.runtime = runtime;
387
+ }
388
+
389
+ async run(options: RunWorkerTaskOptions, identity?: WorkerAssignmentIdentity): Promise<SessionOutcome> {
390
+ if (this.closed) {
391
+ throw new Error("retained worker session owner is closed");
392
+ }
393
+ const assignmentIdentity = identity ?? this.defaultIdentity(options);
394
+ if (this.active) {
395
+ throw new Error("retained worker session already has an active assignment");
396
+ }
397
+
398
+ let resolveCompletion: (() => void) | undefined;
399
+ const completion = new Promise<void>((resolve) => {
400
+ resolveCompletion = resolve;
401
+ });
402
+ const active: ActiveWorkerSessionAssignment = {
403
+ identity: assignmentIdentity,
404
+ controller: new AbortController(),
405
+ tainted: false,
406
+ rotationReported: false,
407
+ resolveCompletion: resolveCompletion as () => void,
408
+ completion,
409
+ };
410
+ this.active = active;
411
+ const assignmentOptions = {
412
+ ...options,
413
+ abortSignal: combineAbortSignals(options.abortSignal, active.controller.signal),
414
+ };
415
+ try {
416
+ return await this.runExclusive(assignmentOptions, assignmentIdentity, active);
417
+ } finally {
418
+ const activeResource = active.resource;
419
+ const retained = this.retained;
420
+ if (active.tainted && activeResource && retained?.resource === activeResource) {
421
+ retained.health = "cancelled";
422
+ await this.disposeRetainedResource(activeResource);
423
+ }
424
+ if (this.active === active) {
425
+ this.active = undefined;
426
+ }
427
+ active.resolveCompletion();
428
+ }
429
+ }
430
+
431
+ private async runExclusive(
432
+ options: RunWorkerTaskOptions,
433
+ identity: WorkerAssignmentIdentity,
434
+ active: ActiveWorkerSessionAssignment,
435
+ ): Promise<SessionOutcome> {
436
+ const compatibility = this.compatibilityFor(options);
437
+ const diagnostics: WorkerSessionDiagnostic[] = [];
438
+ const report = (diagnostic: WorkerSessionDiagnostic) => {
439
+ diagnostics.push(diagnostic);
440
+ options.onSessionDiagnostic?.(diagnostic);
441
+ };
442
+ active.reportDiagnostic = report;
443
+ const diagnosticContext = (retained: RetainedWorkerState) => ({
444
+ ...(retained.contextUsagePercent !== undefined ? { contextUsagePercent: retained.contextUsagePercent } : {}),
445
+ contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
446
+ previousTaskId: retained.previousTask.taskId,
447
+ });
448
+
449
+ let reusedFrom: Pick<Task, "taskId" | "title"> | undefined;
450
+ if (this.retained && !this.assignmentMatchesRetainedScope(options, this.retained)) {
451
+ // A partial-work session is continuity for exactly the next attempt of
452
+ // that task. It must never spill into unrelated work or a later retry.
453
+ report({
454
+ event: "session_rotated",
455
+ reasonCode: "partial_continuation_scope_mismatch",
456
+ ...diagnosticContext(this.retained),
457
+ });
458
+ await this.disposeRetained();
459
+ }
460
+ if (this.retained) {
461
+ const decision = decideWorkerSessionReuse({
462
+ config: {
463
+ enabled: this.runtime.workerSessionReuse,
464
+ contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
465
+ },
466
+ candidate: {
467
+ health: this.retained.health,
468
+ compatibility: this.retained.compatibility,
469
+ contextUsagePercent: this.retained.contextUsagePercent,
470
+ assignmentState: "idle",
471
+ disposed: this.retained.resource.disposed,
472
+ },
473
+ requestedCompatibility: compatibility,
474
+ });
475
+ if (decision.reusable) {
476
+ reusedFrom = this.retained.previousTask;
477
+ report({
478
+ event: "session_reused",
479
+ reasonCode: decision.reasonCode,
480
+ ...diagnosticContext(this.retained),
481
+ });
482
+ } else {
483
+ report({
484
+ event: "session_rotated",
485
+ reasonCode: decision.reasonCode,
486
+ ...diagnosticContext(this.retained),
487
+ });
488
+ await this.disposeRetained();
489
+ }
490
+ }
491
+
492
+ if (!this.retained) {
493
+ try {
494
+ this.retained = {
495
+ resource: await createWorkerSessionResource(options, options.sessionFactory ?? createIsolatedWorkerSession),
496
+ compatibility,
497
+ health: "healthy",
498
+ previousTask: options.task,
499
+ previousAttempt: options.attempt,
500
+ previousAssignmentIdentity: identity,
501
+ reportDiagnostic: report,
502
+ reuseScope: "sequential_task",
503
+ };
504
+ report({
505
+ event: "session_started",
506
+ reasonCode: diagnostics.some((item) => item.event === "session_rotated")
507
+ ? "rotation_completed"
508
+ : "fresh_session",
509
+ contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
510
+ });
511
+ } catch (error) {
512
+ const failed = buildWorkerSessionCreationFailureOutcome(options, error);
513
+ failed.sessionDiagnostics = diagnostics;
514
+ return failed;
515
+ }
516
+ }
517
+
518
+ active.resource = this.retained.resource;
519
+ let outcome: SessionOutcome;
520
+ try {
521
+ outcome = await runWorkerTaskAssignment(
522
+ options,
523
+ this.retained.resource,
524
+ reusedFrom ? { previousTask: reusedFrom } : undefined,
525
+ );
526
+ } catch (error) {
527
+ this.retained.health = active.tainted ? "cancelled" : "unrecoverable_error";
528
+ if (!active.rotationReported) {
529
+ active.rotationReported = true;
530
+ report({
531
+ event: "session_rotated",
532
+ reasonCode: "health_unrecoverable_error",
533
+ ...diagnosticContext(this.retained),
534
+ });
535
+ }
536
+ await this.disposeRetained();
537
+ throw error;
538
+ }
539
+
540
+ if (this.retained) {
541
+ const cancelled = Boolean(options.abortSignal?.aborted);
542
+ this.retained.health = workerSessionHealthForOutcome(outcome, cancelled);
543
+ this.retained.contextUsagePercent = await workerSessionContextUsagePercent(this.retained.resource.session);
544
+ this.retained.previousTask = options.task;
545
+ this.retained.previousAttempt = options.attempt;
546
+ this.retained.previousAssignmentIdentity = identity;
547
+ this.retained.reportDiagnostic = report;
548
+
549
+ if (active.tainted) {
550
+ if (!active.rotationReported) {
551
+ this.reportTaintedRotation(active, "assignment_cancelled");
552
+ }
553
+ await this.disposeRetainedResource(this.retained.resource);
554
+ outcome.sessionDiagnostics = [...(outcome.sessionDiagnostics ?? []), ...diagnostics];
555
+ return outcome;
556
+ }
557
+
558
+ const retry = classifyWorkerSessionRetry({
559
+ done: outcome.done,
560
+ reportedStatus: outcome.reportedStatus,
561
+ completeTaskResult: hasCompleteTaskResult(outcome.assistantText),
562
+ timedOut: outcome.timedOut,
563
+ aborted: outcome.aborted,
564
+ cancelled,
565
+ error: outcome.error,
566
+ });
567
+ this.retained.reuseScope = retry.mayContinueInSession ? "partial_continuation" : "sequential_task";
568
+
569
+ const postAssignmentDecision = decideWorkerSessionReuse({
570
+ config: {
571
+ enabled: this.runtime.workerSessionReuse,
572
+ contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
573
+ },
574
+ candidate: {
575
+ health: this.retained.health,
576
+ compatibility: this.retained.compatibility,
577
+ contextUsagePercent: this.retained.contextUsagePercent,
578
+ assignmentState: "idle",
579
+ disposed: this.retained.resource.disposed,
580
+ },
581
+ requestedCompatibility: compatibility,
582
+ });
583
+ // Completed tasks may flow into the next sequential TODO. A retry may
584
+ // remain only when it is an explicitly safe partial continuation and all
585
+ // normal health/compatibility/context checks still pass.
586
+ const rotateForRetry = !outcome.done && !retry.mayContinueInSession;
587
+ if (!postAssignmentDecision.reusable || rotateForRetry) {
588
+ if (!active.rotationReported) {
589
+ active.rotationReported = true;
590
+ report({
591
+ event: "session_rotated",
592
+ reasonCode: postAssignmentDecision.reusable ? retry.reasonCode : postAssignmentDecision.reasonCode,
593
+ ...diagnosticContext(this.retained),
594
+ });
595
+ }
596
+ await this.disposeRetained();
597
+ } else {
598
+ report({
599
+ event: "session_retained",
600
+ reasonCode: postAssignmentDecision.reasonCode,
601
+ ...diagnosticContext(this.retained),
602
+ });
603
+ }
604
+ }
605
+ outcome.sessionDiagnostics = [...(outcome.sessionDiagnostics ?? []), ...diagnostics];
606
+ return outcome;
607
+ }
608
+
609
+ /**
610
+ * Taint and abort only the matching obsolete assignment. Late cancellation
611
+ * from an older steering generation cannot affect a replacement assignment.
612
+ */
613
+ async invalidateAssignment(identity: WorkerAssignmentIdentity): Promise<boolean> {
614
+ const active = this.active;
615
+ if (active && sameWorkerAssignment(active.identity, identity)) {
616
+ this.taintActiveAssignment(active, "steering_revision_obsolete");
617
+ if (!active.controller.signal.aborted) {
618
+ active.controller.abort(new Error(`worker assignment ${identity.assignmentId} became obsolete`));
619
+ }
620
+ return true;
621
+ }
622
+
623
+ if (this.retained && sameWorkerAssignment(this.retained.previousAssignmentIdentity, identity)) {
624
+ const retained = this.retained;
625
+ retained.health = "cancelled";
626
+ retained.reportDiagnostic({
627
+ event: "session_rotated",
628
+ reasonCode: "steering_revision_obsolete",
629
+ ...this.diagnosticContext(retained),
630
+ });
631
+ await this.disposeRetained();
632
+ return true;
633
+ }
634
+ return false;
635
+ }
636
+
637
+ /** Abort active work, wait for its ownership path, then dispose retained state once. */
638
+ dispose(): Promise<void> {
639
+ if (!this.disposePromise) {
640
+ this.closed = true;
641
+ this.disposePromise = this.disposeAfterActiveAssignment();
642
+ }
643
+ return this.disposePromise;
644
+ }
645
+
646
+ private async disposeAfterActiveAssignment(): Promise<void> {
647
+ const active = this.active;
648
+ if (active) {
649
+ this.taintActiveAssignment(active, "coordinator_shutdown");
650
+ if (!active.controller.signal.aborted) {
651
+ active.controller.abort(new Error("worker session owner disposed"));
652
+ }
653
+ await active.completion;
654
+ }
655
+ await this.disposeRetained();
656
+ }
657
+
658
+ private assignmentMatchesRetainedScope(options: RunWorkerTaskOptions, retained: RetainedWorkerState): boolean {
659
+ if (retained.reuseScope === "sequential_task") {
660
+ return true;
661
+ }
662
+ return (
663
+ options.task.taskId === retained.previousTask.taskId &&
664
+ options.task.title === retained.previousTask.title &&
665
+ options.attempt === retained.previousAttempt + 1
666
+ );
667
+ }
668
+
669
+ private taintActiveAssignment(active: ActiveWorkerSessionAssignment, reasonCode: string): void {
670
+ active.tainted = true;
671
+ const retained = this.retained;
672
+ if (retained && retained.resource === active.resource) {
673
+ retained.health = "cancelled";
674
+ }
675
+ this.reportTaintedRotation(active, reasonCode);
676
+ }
677
+
678
+ private reportTaintedRotation(active: ActiveWorkerSessionAssignment, reasonCode: string): void {
679
+ if (active.rotationReported) return;
680
+ active.rotationReported = true;
681
+ const retained = this.retained;
682
+ active.reportDiagnostic?.({
683
+ event: "session_rotated",
684
+ reasonCode,
685
+ ...(retained ? this.diagnosticContext(retained) : {}),
686
+ });
687
+ }
688
+
689
+ private diagnosticContext(retained: RetainedWorkerState): {
690
+ contextUsagePercent?: number;
691
+ contextThresholdPercent: number;
692
+ previousTaskId: string;
693
+ } {
694
+ return {
695
+ ...(retained.contextUsagePercent !== undefined ? { contextUsagePercent: retained.contextUsagePercent } : {}),
696
+ contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
697
+ previousTaskId: retained.previousTask.taskId,
698
+ };
699
+ }
700
+
701
+ private defaultIdentity(options: RunWorkerTaskOptions): WorkerAssignmentIdentity {
702
+ const sequence = ++this.assignmentSequence;
703
+ return {
704
+ assignmentId: `${options.task.taskId}:${options.attempt}:${sequence}`,
705
+ taskIdentity: `${options.task.taskId}:${options.task.title}`,
706
+ steeringGeneration: 0,
707
+ planAuthorityToken: "direct-owner",
708
+ };
709
+ }
710
+
711
+ private compatibilityFor(options: RunWorkerTaskOptions): WorkerSessionCompatibilityFingerprint {
712
+ return createWorkerSessionCompatibilityFingerprint({
713
+ coordinatorRunId: this.runtime.runId,
714
+ repositoryRoot: this.runtime.cwd,
715
+ worktreeRoot: options.cwd,
716
+ modelName: options.modelName,
717
+ model: options.model,
718
+ tools: options.tools ?? DEFAULT_WORKER_TOOLS,
719
+ thinkingLevel: options.thinkingLevel,
720
+ agentDir: options.agentDir,
721
+ modelRuntime: options.modelRuntime,
722
+ authStorage: options.authStorage,
723
+ modelRegistry: options.modelRegistry,
724
+ settingsManager: options.settingsManager,
725
+ resourceLoader: options.resourceLoader,
726
+ sessionFactory: options.sessionFactory ?? createIsolatedWorkerSession,
727
+ });
728
+ }
729
+
730
+ private async disposeRetained(): Promise<void> {
731
+ const retained = this.retained;
732
+ if (!retained) {
733
+ return;
734
+ }
735
+ await this.disposeRetainedResource(retained.resource);
736
+ }
737
+
738
+ private async disposeRetainedResource(resource: WorkerSessionResource): Promise<void> {
739
+ if (this.retained?.resource === resource) {
740
+ this.retained = undefined;
741
+ }
742
+ try {
743
+ await disposeWorkerSessionResource(resource);
744
+ } catch {
745
+ // Session disposal is best effort; resource ownership is still closed exactly once.
746
+ }
747
+ }
748
+ }
749
+
750
+ function sameWorkerAssignment(left: WorkerAssignmentIdentity, right: WorkerAssignmentIdentity): boolean {
751
+ return (
752
+ left.assignmentId === right.assignmentId &&
753
+ left.taskIdentity === right.taskIdentity &&
754
+ left.steeringGeneration === right.steeringGeneration &&
755
+ left.planAuthorityToken === right.planAuthorityToken
756
+ );
757
+ }
758
+
759
+ function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined {
760
+ const available = signals.filter((signal): signal is AbortSignal => Boolean(signal));
761
+ if (available.length === 0) return undefined;
762
+ if (available.length === 1) return available[0];
763
+ return AbortSignal.any(available);
764
+ }
765
+
766
+ /** Retains the full worker result while exposing its provider error to the shared classifier. */
767
+ class WorkerNetworkFailure extends Error {
768
+ readonly outcome: SessionOutcome;
769
+
770
+ constructor(outcome: SessionOutcome) {
771
+ const message = outcome.error ?? "worker network operation failed";
772
+ super(message, { cause: workerFailureValue(outcome) });
773
+ this.name = "WorkerNetworkFailure";
774
+ this.outcome = outcome;
775
+ }
776
+ }
777
+
778
+ interface WorkerRecoveryExecutionOptions {
779
+ workerOptions: RunWorkerTaskOptions;
780
+ run: (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
781
+ taskResultPath: string;
782
+ networkRecovery: Readonly<NetworkRecoveryConfig>;
783
+ signal?: AbortSignal;
784
+ onInterruption?: (outcome: SessionOutcome) => void;
785
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
786
+ }
787
+
788
+ /**
789
+ * Run one ordinary worker attempt, replacing only transport-failed sessions.
790
+ * Pi owns bounded request retries inside session.prompt(); therefore an outcome
791
+ * reaches this boundary only after those retries have settled. Coordinator
792
+ * probes retain the same task/attempt and always receive a recovery prompt.
793
+ */
794
+ async function runWorkerAttemptWithNetworkRecovery(options: WorkerRecoveryExecutionOptions): Promise<SessionOutcome> {
795
+ const interrupted: SessionOutcome[] = [];
796
+ const thrownErrors = new Map<SessionOutcome, unknown>();
797
+ const execute = async (workerOptions: RunWorkerTaskOptions): Promise<SessionOutcome> => {
798
+ try {
799
+ return await options.run(workerOptions);
800
+ } catch (error) {
801
+ const outcome = buildWorkerSessionCreationFailureOutcome(workerOptions, error);
802
+ thrownErrors.set(outcome, error);
803
+ return outcome;
804
+ }
805
+ };
806
+ const recordRecoverableInterruption = async (outcome: SessionOutcome): Promise<void> => {
807
+ await appendNetworkInterruptionEvidence(options.taskResultPath, outcome, interrupted.length + 1);
808
+ interrupted.push(outcome);
809
+ options.onInterruption?.(outcome);
810
+ };
811
+
812
+ const initial = await execute(options.workerOptions);
813
+ const initialFailure = workerFailureValue(initial);
814
+ const initialClassification = initialFailure === undefined ? undefined : classifyNetworkFailure(initialFailure);
815
+ if (initialClassification && isFailFastWorkerFailure(initialClassification.reason)) {
816
+ throw new WorkerNetworkFailure(initial);
817
+ }
818
+ if (!options.networkRecovery.enabled || !initialClassification?.recoverable) {
819
+ if (thrownErrors.has(initial)) throw thrownErrors.get(initial);
820
+ return initial;
821
+ }
822
+ await recordRecoverableInterruption(initial);
823
+
824
+ try {
825
+ const recovered = await recoverNetworkOperation({
826
+ initialFailure: new WorkerNetworkFailure(initial),
827
+ config: options.networkRecovery,
828
+ signal: options.signal,
829
+ onEvent: options.onNetworkRecovery,
830
+ retry: async ({ retryCount, signal }) => {
831
+ const previous = interrupted.at(-1)!;
832
+ const resumed = await execute({
833
+ ...options.workerOptions,
834
+ // The recovery signal includes both run cancellation and the outage
835
+ // deadline without replacing the assignment/steering cancellation.
836
+ abortSignal: combineAbortSignals(options.workerOptions.abortSignal, signal),
837
+ networkRecoveryContext: {
838
+ retryCount,
839
+ durableEvidencePath: options.taskResultPath,
840
+ priorSessionId: previous.sessionId,
841
+ failure: previous.error ?? "transient provider or transport failure",
842
+ },
843
+ });
844
+ const resumedFailure = workerFailureValue(resumed);
845
+ const classification = resumedFailure === undefined ? undefined : classifyNetworkFailure(resumedFailure);
846
+ if (classification?.recoverable) {
847
+ await recordRecoverableInterruption(resumed);
848
+ }
849
+ if (resumed.error) {
850
+ if (
851
+ thrownErrors.has(resumed) &&
852
+ !classification?.recoverable &&
853
+ !isFailFastWorkerFailure(classification!.reason)
854
+ ) {
855
+ throw thrownErrors.get(resumed);
856
+ }
857
+ throw new WorkerNetworkFailure(resumed);
858
+ }
859
+ return resumed;
860
+ },
861
+ });
862
+ return mergeWorkerRecoveryOutcomes(interrupted, recovered.value);
863
+ } catch (error) {
864
+ // If connectivity recovered but the fresh session failed deterministically,
865
+ // hand its outcome back to the ordinary worker failure path immediately.
866
+ if (error instanceof WorkerNetworkFailure) {
867
+ const merged = mergeWorkerRecoveryOutcomes(interrupted, error.outcome);
868
+ const mergedFailure = workerFailureValue(merged);
869
+ const classification = mergedFailure === undefined ? undefined : classifyNetworkFailure(mergedFailure);
870
+ if (classification && isFailFastWorkerFailure(classification.reason)) {
871
+ throw new WorkerNetworkFailure(merged);
872
+ }
873
+ return merged;
874
+ }
875
+ throw error;
876
+ }
877
+ }
878
+
879
+ function workerFailureValue(outcome: SessionOutcome): unknown {
880
+ return outcome.failure ?? outcome.error;
881
+ }
882
+
883
+ function isFailFastWorkerFailure(reason: ReturnType<typeof classifyNetworkFailure>["reason"]): boolean {
884
+ return [
885
+ "authentication",
886
+ "authorization",
887
+ "billing",
888
+ "quota_exhausted",
889
+ "invalid_model",
890
+ "invalid_request",
891
+ "http_client_error",
892
+ "non_retryable_server_error",
893
+ ].includes(reason);
894
+ }
895
+
896
+ function mergeWorkerRecoveryOutcomes(interrupted: readonly SessionOutcome[], final: SessionOutcome): SessionOutcome {
897
+ if (interrupted.length === 0) return final;
898
+ const usage = addWorkerUsage([...interrupted.map((item) => item.workerUsage), final.workerUsage]);
899
+ return {
900
+ ...final,
901
+ startedAt: interrupted[0].startedAt,
902
+ contextObservations: [
903
+ ...interrupted.flatMap((item, index) => [
904
+ `network interruption ${index + 1}: ${item.error ?? "transient provider or transport failure"}`,
905
+ ...item.contextObservations,
906
+ ]),
907
+ ...final.contextObservations,
908
+ ],
909
+ compactionEvents: [...interrupted.flatMap((item) => item.compactionEvents), ...final.compactionEvents],
910
+ events: [...interrupted.flatMap((item) => item.events), ...final.events],
911
+ workerCostTotal: [...interrupted, final].reduce((total, item) => total + item.workerCostTotal, 0),
912
+ workerCostSource: "network_recovery_aggregate",
913
+ workerUsage: usage,
914
+ sessionDiagnostics: [
915
+ ...interrupted.flatMap((item) => item.sessionDiagnostics ?? []),
916
+ ...(final.sessionDiagnostics ?? []),
917
+ ],
918
+ };
919
+ }
920
+
921
+ function addWorkerUsage(values: Array<WorkerUsageTotals | undefined>): WorkerUsageTotals | undefined {
922
+ const available = values.filter((value): value is WorkerUsageTotals => Boolean(value));
923
+ if (available.length === 0) return undefined;
924
+ return available.reduce<WorkerUsageTotals>(
925
+ (total, value) => ({
926
+ input: total.input + value.input,
927
+ output: total.output + value.output,
928
+ cacheRead: total.cacheRead + value.cacheRead,
929
+ cacheWrite: total.cacheWrite + value.cacheWrite,
930
+ total: total.total + value.total,
931
+ }),
932
+ { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
933
+ );
934
+ }
935
+
936
+ export function workerSessionHealthForOutcome(
937
+ outcome: Pick<SessionOutcome, "timedOut" | "aborted" | "error" | "assistantText">,
938
+ cancelled = false,
939
+ ): WorkerSessionHealth {
940
+ if (outcome.timedOut) return "timed_out";
941
+ if (cancelled) return "cancelled";
942
+ if (outcome.aborted) return "aborted";
943
+ if (outcome.error) return "unrecoverable_error";
944
+ if (!hasCompleteTaskResult(outcome.assistantText)) return "invalid_state";
945
+ return "healthy";
255
946
  }
256
947
 
257
948
  export async function runCoordinator(options: RunCoordinatorOptions): Promise<CoordinatorResult> {
258
949
  const runtime = buildRuntimeOptions(options);
950
+ const workerSessionOwner = runtime.useRetainedWorkerLifecycle
951
+ ? new CoordinatorWorkerSessionOwner(runtime)
952
+ : undefined;
259
953
  const inputText = coordinatorInputText(options);
260
954
  const attempts: TaskAttemptSummary[] = [];
261
955
  const outcomes: SessionOutcome[] = [];
@@ -269,6 +963,12 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
269
963
  let activeTask: Task | undefined;
270
964
  let activeTaskReference: PlanTaskReference | undefined;
271
965
  let activeAttempt: number | undefined;
966
+ let activeWorkerAssignment:
967
+ | { identity: WorkerAssignmentIdentity; controller: AbortController; obsolete: boolean }
968
+ | undefined;
969
+ let steeringGeneration = 0;
970
+ let workerExecutionSequence = 0;
971
+ let removeSteeringProcessor: (() => void) | undefined;
272
972
  const protectedDirtyPathsByTask = new Map<string, Set<string>>();
273
973
 
274
974
  try {
@@ -287,7 +987,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
287
987
  });
288
988
 
289
989
  let failure: string | undefined;
290
- runtime.steeringQueue?.setProcessor(async (message) => {
990
+ removeSteeringProcessor = runtime.steeringQueue?.setProcessor(async (message) => {
291
991
  const baseAtRequest = planStore.snapshot();
292
992
  const activeTaskAtRequest = activeTaskReference
293
993
  ? resolvePlanTaskReference(
@@ -320,6 +1020,25 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
320
1020
  // boundary, so this accepted revision continues the same run.
321
1021
  latestTodoMarkdown = appliedRevision.todoMarkdown;
322
1022
  latestTasks = appliedRevision.reconciliation.activeTasks.map((item) => item.task);
1023
+ steeringGeneration += 1;
1024
+
1025
+ // Once replacement/removal is authoritative, make the exact old
1026
+ // invocation obsolete before aborting it. Event callbacks consult this
1027
+ // identity, so a late old result cannot repaint replacement progress.
1028
+ const assignmentAtAcceptance = activeWorkerAssignment;
1029
+ const activeStillValid = activeTaskReference
1030
+ ? Boolean(resolvePlanTaskReference(latestTasks, activeTaskReference, planStore.snapshot().authorityToken))
1031
+ : true;
1032
+ if (assignmentAtAcceptance && !activeStillValid) {
1033
+ assignmentAtAcceptance.obsolete = true;
1034
+ if (!assignmentAtAcceptance.controller.signal.aborted) {
1035
+ assignmentAtAcceptance.controller.abort(
1036
+ new Error(`steering revision ${message.sequence} replaced the active assignment`),
1037
+ );
1038
+ }
1039
+ await workerSessionOwner?.invalidateAssignment(assignmentAtAcceptance.identity);
1040
+ }
1041
+
323
1042
  emitProgress(
324
1043
  runtime,
325
1044
  `Accepted steering revision ${message.sequence} with ${appliedRevision.reconciliation.activeTasks.length} task(s).`,
@@ -365,6 +1084,23 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
365
1084
  const initialActivity =
366
1085
  nextTask.statusItems.find((item) => !item.done)?.text ?? `Starting TODO ${nextTask.taskId}`;
367
1086
  const worker = workerKey(nextTask.taskId, attempt);
1087
+ // Task IDs and attempt numbers may be reused after an in-flight task is
1088
+ // replaced by steering. Accounting needs an invocation identity so the
1089
+ // obsolete attempt's finalized spend cannot be overwritten.
1090
+ const accountingWorker = `${worker}#${++workerExecutionSequence}`;
1091
+ const assignmentIdentity: WorkerAssignmentIdentity = {
1092
+ assignmentId: accountingWorker,
1093
+ taskIdentity: nextTask.stableId ?? taskSemanticFingerprint(nextTask),
1094
+ steeringGeneration,
1095
+ planAuthorityToken: schedulingSnapshot.authorityToken,
1096
+ };
1097
+ const assignmentController = new AbortController();
1098
+ const assignmentState = { identity: assignmentIdentity, controller: assignmentController, obsolete: false };
1099
+ const taskPlanReference = planTaskReference(nextTask, schedulingSnapshot.authorityToken);
1100
+ activeWorkerAssignment = assignmentState;
1101
+ activeTask = nextTask;
1102
+ activeAttempt = attempt;
1103
+ activeTaskReference = taskPlanReference;
368
1104
  runtime.workerActivityByWorker.set(worker, initialActivity);
369
1105
  runtime.workerTextByWorker.delete(worker);
370
1106
  runtime.workerTextPublishedLengthByWorker.delete(worker);
@@ -393,11 +1129,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
393
1129
  : new Set<string>();
394
1130
  protectedDirtyPathsByTask.set(executionIdentity, preExistingDirtyPaths);
395
1131
  }
396
- activeTask = nextTask;
397
- activeAttempt = attempt;
398
- const taskPlanReference = planTaskReference(nextTask, schedulingSnapshot.authorityToken);
399
- activeTaskReference = taskPlanReference;
400
- const outcome = await runtime.workerRunner({
1132
+ const workerOptions: RunWorkerTaskOptions = {
401
1133
  cwd: runtime.cwd,
402
1134
  todoPath: runtime.todoPath,
403
1135
  task: nextTask,
@@ -415,12 +1147,62 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
415
1147
  model: runtime.workerModel,
416
1148
  modelName: runtime.workerModelName,
417
1149
  thinkingLevel: runtime.taskThinking,
418
- abortSignal: runtime.abortSignal,
1150
+ abortSignal: combineAbortSignals(runtime.abortSignal, assignmentController.signal),
419
1151
  sessionFactory: runtime.workerSessionFactory,
1152
+ networkRecovery: runtime.networkRecovery,
420
1153
  now: runtime.now,
421
- onEvent: (event) => emitWorkerEventProgress(runtime, tasksBeforeAttempt, nextTask, attempts, attempt, event),
422
- });
423
- finalizeWorkerCost(runtime.workerCostState, outcome);
1154
+ onEvent: (event) => {
1155
+ if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
1156
+ emitWorkerEventProgress(runtime, tasksBeforeAttempt, nextTask, attempts, attempt, event, accountingWorker);
1157
+ }
1158
+ },
1159
+ onSessionDiagnostic: (diagnostic) => {
1160
+ if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
1161
+ emitWorkerSessionProgress(runtime, tasksBeforeAttempt, nextTask, attempts, attempt, diagnostic);
1162
+ } else {
1163
+ // Lifecycle accounting remains accurate, but obsolete diagnostics
1164
+ // must not mutate the replacement task's visible progress.
1165
+ recordWorkerSessionMetric(runtime.workerSessionMetrics, diagnostic);
1166
+ }
1167
+ },
1168
+ };
1169
+ let outcome: SessionOutcome;
1170
+ const networkInterruptedOutcomes: SessionOutcome[] = [];
1171
+ try {
1172
+ outcome = await runWorkerAttemptWithNetworkRecovery({
1173
+ workerOptions,
1174
+ run: (resumedOptions) =>
1175
+ workerSessionOwner
1176
+ ? workerSessionOwner.run(resumedOptions, assignmentIdentity)
1177
+ : runtime.workerRunner(resumedOptions),
1178
+ taskResultPath: runtime.taskResultPath,
1179
+ networkRecovery: runtime.networkRecovery,
1180
+ signal: workerOptions.abortSignal,
1181
+ onInterruption: (interrupted) => networkInterruptedOutcomes.push(interrupted),
1182
+ onNetworkRecovery: createNetworkRecoveryProgressHandler(runtime),
1183
+ });
1184
+ } catch (error) {
1185
+ if (!assignmentState.obsolete) {
1186
+ const terminalOutcome = error instanceof WorkerNetworkFailure ? error.outcome : undefined;
1187
+ if (terminalOutcome || networkInterruptedOutcomes.length > 0) {
1188
+ finalizeWorkerCost(runtime.workerCostState, accountingWorker, {
1189
+ workerCostTotal:
1190
+ terminalOutcome?.workerCostTotal ??
1191
+ networkInterruptedOutcomes.reduce((total, interrupted) => total + interrupted.workerCostTotal, 0),
1192
+ });
1193
+ }
1194
+ throw error;
1195
+ }
1196
+ // A cancellation-aware custom runner may reject instead of returning
1197
+ // an aborted outcome. Preserve historical evidence, but never let that
1198
+ // obsolete rejection terminate or update the replacement assignment.
1199
+ outcome = mergeWorkerRecoveryOutcomes(
1200
+ networkInterruptedOutcomes,
1201
+ buildWorkerSessionCreationFailureOutcome(workerOptions, error),
1202
+ );
1203
+ outcome.aborted = true;
1204
+ }
1205
+ finalizeWorkerCost(runtime.workerCostState, accountingWorker, outcome);
424
1206
 
425
1207
  // A revision may have been accepted while the worker was running. Let all
426
1208
  // already-received guidance settle, then resolve this exact task identity
@@ -465,6 +1247,9 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
465
1247
  activeTask = undefined;
466
1248
  activeTaskReference = undefined;
467
1249
  activeAttempt = undefined;
1250
+ if (activeWorkerAssignment === assignmentState) {
1251
+ activeWorkerAssignment = undefined;
1252
+ }
468
1253
 
469
1254
  let taskCommitHash: string | undefined;
470
1255
  let taskCommitError: string | undefined;
@@ -587,6 +1372,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
587
1372
  attempts,
588
1373
  taskProgress,
589
1374
  workerCostTotal: runtime.workerCostState.total,
1375
+ workerUsageTotal: aggregateWorkerUsage(outcomes),
1376
+ workerSessionMetrics: snapshotWorkerSessionMetrics(runtime.workerSessionMetrics),
590
1377
  commit: options.commit,
591
1378
  goal: runtime.goal,
592
1379
  error: failure,
@@ -668,6 +1455,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
668
1455
  attempts,
669
1456
  taskProgress,
670
1457
  workerCostTotal: runtime.workerCostState.total,
1458
+ workerUsageTotal: aggregateWorkerUsage(outcomes),
1459
+ workerSessionMetrics: snapshotWorkerSessionMetrics(runtime.workerSessionMetrics),
671
1460
  commit: options.commit,
672
1461
  goal: runtime.goal,
673
1462
  error: resultError,
@@ -680,6 +1469,11 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
680
1469
  taskProgress,
681
1470
  });
682
1471
  return result;
1472
+ } finally {
1473
+ runtime.progressClosed = true;
1474
+ runtime.activeNetworkRecoveries.clear();
1475
+ removeSteeringProcessor?.();
1476
+ await workerSessionOwner?.dispose();
683
1477
  }
684
1478
  }
685
1479
 
@@ -749,19 +1543,59 @@ async function extractTodoMarkdownWithOneRepair(
749
1543
  }
750
1544
 
751
1545
  async function requestTodoPlan(inputText: string, runtime: RuntimeOptions): Promise<string> {
752
- return runtime.todoPlanner({
753
- inputText,
754
- cwd: runtime.cwd,
755
- runDir: runtime.runDir,
756
- thinkingLevel: runtime.todoThinking,
757
- model: runtime.workerModel,
758
- abortSignal: runtime.abortSignal,
759
- timeoutMs: runtime.todoTimeoutMs,
760
- gracefulShutdownMs: runtime.todoGracefulShutdownMs,
761
- sessionFactory: runtime.todoSessionFactory,
762
- onDiagnostic: (diagnostic) => recordPlannerDiagnostic(runtime, diagnostic),
763
- goal: runtime.goal,
764
- });
1546
+ return runPlannerOperationWithNetworkRecovery(
1547
+ {
1548
+ inputText,
1549
+ cwd: runtime.cwd,
1550
+ runDir: runtime.runDir,
1551
+ thinkingLevel: runtime.todoThinking,
1552
+ model: runtime.workerModel,
1553
+ abortSignal: runtime.abortSignal,
1554
+ timeoutMs: runtime.todoTimeoutMs,
1555
+ gracefulShutdownMs: runtime.todoGracefulShutdownMs,
1556
+ sessionFactory: runtime.todoSessionFactory,
1557
+ networkRecovery: runtime.networkRecovery,
1558
+ onDiagnostic: (diagnostic) => recordPlannerDiagnostic(runtime, diagnostic),
1559
+ goal: runtime.goal,
1560
+ },
1561
+ runtime,
1562
+ );
1563
+ }
1564
+
1565
+ /**
1566
+ * Retry a side-effect-free planner request only after its provider boundary has
1567
+ * failed. The default planner disables tools and disposes every session before
1568
+ * rejecting, so each retry rotates unsafe conversation state while replaying
1569
+ * only the complete immutable planning context. Recovery owns no planner
1570
+ * repair/attempt counter, and each fresh call retains the planner timeout;
1571
+ * backoff remains governed solely by the separate outage deadline.
1572
+ */
1573
+ async function runPlannerOperationWithNetworkRecovery(
1574
+ plannerOptions: TodoPlannerOptions,
1575
+ runtime: RuntimeOptions,
1576
+ ): Promise<string> {
1577
+ const run = (recoverySignal?: AbortSignal) =>
1578
+ runtime.todoPlanner({
1579
+ ...plannerOptions,
1580
+ abortSignal: combineAbortSignals(plannerOptions.abortSignal, recoverySignal),
1581
+ });
1582
+
1583
+ try {
1584
+ return await run();
1585
+ } catch (initialFailure) {
1586
+ const classification = classifyNetworkFailure(initialFailure);
1587
+ if (!runtime.networkRecovery.enabled || !classification.recoverable) {
1588
+ throw initialFailure;
1589
+ }
1590
+ const recovered = await recoverNetworkOperation({
1591
+ initialFailure,
1592
+ config: runtime.networkRecovery,
1593
+ signal: plannerOptions.abortSignal,
1594
+ onEvent: createNetworkRecoveryProgressHandler(runtime),
1595
+ retry: ({ signal }) => run(signal),
1596
+ });
1597
+ return recovered.value;
1598
+ }
765
1599
  }
766
1600
 
767
1601
  async function generateSteeringPlanRevision(options: {
@@ -794,21 +1628,25 @@ async function generateSteeringPlanRevision(options: {
794
1628
  }
795
1629
  : undefined,
796
1630
  planner: ({ prompt, request }) =>
797
- options.runtime.todoPlanner({
798
- inputText: prompt,
799
- plannerPrompt: prompt,
800
- planRevision: request,
801
- cwd: options.runtime.cwd,
802
- runDir: options.runtime.runDir,
803
- thinkingLevel: options.runtime.todoThinking,
804
- model: options.runtime.workerModel,
805
- abortSignal: options.runtime.abortSignal,
806
- timeoutMs: options.runtime.todoTimeoutMs,
807
- gracefulShutdownMs: options.runtime.todoGracefulShutdownMs,
808
- sessionFactory: options.runtime.todoSessionFactory,
809
- onDiagnostic: (diagnostic) => recordPlannerDiagnostic(options.runtime, diagnostic),
810
- goal: options.runtime.goal,
811
- }),
1631
+ runPlannerOperationWithNetworkRecovery(
1632
+ {
1633
+ inputText: prompt,
1634
+ plannerPrompt: prompt,
1635
+ planRevision: request,
1636
+ cwd: options.runtime.cwd,
1637
+ runDir: options.runtime.runDir,
1638
+ thinkingLevel: options.runtime.todoThinking,
1639
+ model: options.runtime.workerModel,
1640
+ abortSignal: options.runtime.abortSignal,
1641
+ timeoutMs: options.runtime.todoTimeoutMs,
1642
+ gracefulShutdownMs: options.runtime.todoGracefulShutdownMs,
1643
+ sessionFactory: options.runtime.todoSessionFactory,
1644
+ networkRecovery: options.runtime.networkRecovery,
1645
+ onDiagnostic: (diagnostic) => recordPlannerDiagnostic(options.runtime, diagnostic),
1646
+ goal: options.runtime.goal,
1647
+ },
1648
+ options.runtime,
1649
+ ),
812
1650
  });
813
1651
  }
814
1652
 
@@ -1031,7 +1869,10 @@ async function runTodoPlannerPrompt(options: {
1031
1869
  if (promptResult.error) {
1032
1870
  const message = `TODO planner failed: ${promptResult.error}`;
1033
1871
  options.onDiagnostic?.(plannerPromptDiagnostic("failure", message, promptResult));
1034
- throw new TodoGenerationError(message);
1872
+ throw new TodoGenerationError(
1873
+ message,
1874
+ promptResult.failure === undefined ? undefined : { cause: promptResult.failure },
1875
+ );
1035
1876
  }
1036
1877
  if (!promptResult.assistantText) {
1037
1878
  const message = "TODO planner did not return assistant text.";
@@ -1077,6 +1918,15 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
1077
1918
  const workerModelName = options.workerModelName ?? parsedWorkerConfig.modelName;
1078
1919
  const workerModel = workerModelName ? undefined : options.workerModel;
1079
1920
  const goal = normalizeOptionalText(options.goal);
1921
+ const workerSessionReuseConfig = resolveWorkerSessionReuseConfig({
1922
+ enabled: options.workerSessionReuse ?? parsedWorkerConfig.workerSessionReuseEnabled,
1923
+ contextThresholdPercent:
1924
+ options.workerSessionReuseContextThresholdPercent ?? parsedWorkerConfig.workerSessionReuseContextThresholdPercent,
1925
+ });
1926
+ const networkRecovery = resolveNetworkRecoveryConfig({
1927
+ ...parsedWorkerConfig.networkRecovery,
1928
+ ...options.networkRecovery,
1929
+ });
1080
1930
 
1081
1931
  return {
1082
1932
  cwd,
@@ -1098,7 +1948,11 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
1098
1948
  goal,
1099
1949
  taskThinking: options.taskThinking ?? DEFAULT_COORDINATOR_OPTIONS.taskThinking,
1100
1950
  todoThinking: options.todoThinking ?? DEFAULT_COORDINATOR_OPTIONS.todoThinking,
1951
+ workerSessionReuse: workerSessionReuseConfig.enabled,
1952
+ workerSessionReuseContextThresholdPercent: workerSessionReuseConfig.contextThresholdPercent,
1953
+ networkRecovery,
1101
1954
  workerRunner: options.workerRunner ?? runWorkerTask,
1955
+ useRetainedWorkerLifecycle: options.workerRunner === undefined,
1102
1956
  todoPlanner: options.todoPlanner ?? runTodoPlanner,
1103
1957
  abortSignal: options.abortSignal,
1104
1958
  workerSessionFactory: options.workerSessionFactory,
@@ -1110,8 +1964,13 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
1110
1964
  workerTextByWorker: new Map(),
1111
1965
  workerTextPublishedLengthByWorker: new Map(),
1112
1966
  plannerDiagnostics: [],
1967
+ workerSessionMetrics: createWorkerSessionMetrics(),
1113
1968
  steeringQueue: options.steeringQueue,
1114
1969
  onPlanRevisionAccepted: options.onPlanRevisionAccepted,
1970
+ onNetworkRecovery: options.onNetworkRecovery,
1971
+ progressClosed: false,
1972
+ networkRecoverySequence: 0,
1973
+ activeNetworkRecoveries: new Map(),
1115
1974
  };
1116
1975
  }
1117
1976
 
@@ -1120,7 +1979,8 @@ function emitProgress(
1120
1979
  message: string,
1121
1980
  update: Omit<CoordinatorProgressUpdate, "message" | "runId" | "todoPath" | "resultPath" | "workerCostTotal">,
1122
1981
  ): void {
1123
- runtime.onProgress?.({
1982
+ if (runtime.progressClosed) return;
1983
+ const progress: CoordinatorProgressUpdate = {
1124
1984
  message,
1125
1985
  runId: runtime.runId,
1126
1986
  todoPath: runtime.todoPath,
@@ -1128,9 +1988,107 @@ function emitProgress(
1128
1988
  workerCostTotal: runtime.workerCostState.total,
1129
1989
  ...update,
1130
1990
  goal: runtime.goal,
1991
+ };
1992
+ runtime.lastProgress = progress;
1993
+ const activeRecovery = latestNetworkRecovery(runtime.activeNetworkRecoveries);
1994
+ if (activeRecovery) {
1995
+ publishNetworkRecoveryProgress(runtime, activeRecovery);
1996
+ } else {
1997
+ runtime.onProgress?.(progress);
1998
+ }
1999
+ }
2000
+
2001
+ /**
2002
+ * Bridge one recovery lifecycle into coordinator progress without replacing the
2003
+ * last stable task/phase update. A recovered operation restores whichever
2004
+ * ordinary status is current; terminal failures are left for the normal final
2005
+ * status path. Operation IDs prevent an older concurrent recovery from
2006
+ * repainting a newer outage or completion.
2007
+ */
2008
+ function createNetworkRecoveryProgressHandler(runtime: RuntimeOptions): (event: NetworkRecoveryEvent) => void {
2009
+ const operationId = ++runtime.networkRecoverySequence;
2010
+ let cleaned = false;
2011
+
2012
+ return (event) => {
2013
+ runtime.onNetworkRecovery?.(event);
2014
+ if (cleaned || runtime.progressClosed) return;
2015
+
2016
+ if (event.type === "cleanup") {
2017
+ cleaned = true;
2018
+ runtime.activeNetworkRecoveries.delete(operationId);
2019
+ return;
2020
+ }
2021
+
2022
+ if (isTerminalNetworkRecoveryEvent(event.type)) {
2023
+ runtime.activeNetworkRecoveries.delete(operationId);
2024
+ if (event.type === "recovered") {
2025
+ const active = latestNetworkRecovery(runtime.activeNetworkRecoveries);
2026
+ if (active) {
2027
+ publishNetworkRecoveryProgress(runtime, active);
2028
+ } else if (runtime.lastProgress) {
2029
+ runtime.onProgress?.({ ...runtime.lastProgress, workerCostTotal: runtime.workerCostState.total });
2030
+ }
2031
+ }
2032
+ return;
2033
+ }
2034
+
2035
+ runtime.activeNetworkRecoveries.set(operationId, event);
2036
+ if (operationId === latestNetworkRecoveryId(runtime.activeNetworkRecoveries)) {
2037
+ publishNetworkRecoveryProgress(runtime, event);
2038
+ }
2039
+ };
2040
+ }
2041
+
2042
+ function publishNetworkRecoveryProgress(runtime: RuntimeOptions, event: NetworkRecoveryEvent): void {
2043
+ if (runtime.progressClosed) return;
2044
+ const stable = runtime.lastProgress;
2045
+ const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
2046
+ const message = formatNetworkRecoveryStatus(event);
2047
+ runtime.onProgress?.({
2048
+ message,
2049
+ phase: "network_wait",
2050
+ runId: runtime.runId,
2051
+ todoPath: runtime.todoPath,
2052
+ resultPath: runtime.taskResultPath,
2053
+ workerCostTotal: runtime.workerCostState.total,
2054
+ goal: runtime.goal,
2055
+ taskId: stable?.taskId,
2056
+ title: stable?.title,
2057
+ attempt: stable?.attempt,
2058
+ totalTasks: stable?.totalTasks,
2059
+ currentTask: stable?.currentTask,
2060
+ subtasks: stable?.subtasks,
2061
+ taskProgress: stable?.taskProgress,
2062
+ activeStatus: message,
2063
+ networkRecoveryEvent: event.type,
2064
+ networkRetryCount: event.state.retryCount,
2065
+ networkOutageElapsedMs: event.state.elapsedMs,
2066
+ networkNextRetryAtMs: event.state.nextRetryAtMs,
2067
+ networkNextRetryInMs:
2068
+ event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
2069
+ networkFailureReason: event.state.lastFailure.reason,
1131
2070
  });
1132
2071
  }
1133
2072
 
2073
+ function latestNetworkRecovery(
2074
+ recoveries: ReadonlyMap<number, NetworkRecoveryEvent>,
2075
+ ): NetworkRecoveryEvent | undefined {
2076
+ const id = latestNetworkRecoveryId(recoveries);
2077
+ return id === undefined ? undefined : recoveries.get(id);
2078
+ }
2079
+
2080
+ function latestNetworkRecoveryId(recoveries: ReadonlyMap<number, NetworkRecoveryEvent>): number | undefined {
2081
+ let latest: number | undefined;
2082
+ for (const id of recoveries.keys()) {
2083
+ if (latest === undefined || id > latest) latest = id;
2084
+ }
2085
+ return latest;
2086
+ }
2087
+
2088
+ function isTerminalNetworkRecoveryEvent(type: NetworkRecoveryEventType): boolean {
2089
+ return type === "recovered" || type === "failed" || type === "cancelled" || type === "outage_expired";
2090
+ }
2091
+
1134
2092
  function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDiagnostic): void {
1135
2093
  const normalized: PlannerDiagnostic = {
1136
2094
  kind: diagnostic.kind,
@@ -1156,6 +2114,41 @@ function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDia
1156
2114
  });
1157
2115
  }
1158
2116
 
2117
+ function aggregateWorkerUsage(outcomes: readonly SessionOutcome[]): WorkerUsageTotals | undefined {
2118
+ const usage = outcomes.flatMap((outcome) => (outcome.workerUsage ? [outcome.workerUsage] : []));
2119
+ if (usage.length === 0) {
2120
+ return undefined;
2121
+ }
2122
+ return usage.reduce<WorkerUsageTotals>(
2123
+ (total, item) => ({
2124
+ input: total.input + item.input,
2125
+ output: total.output + item.output,
2126
+ cacheRead: total.cacheRead + item.cacheRead,
2127
+ cacheWrite: total.cacheWrite + item.cacheWrite,
2128
+ total: total.total + item.total,
2129
+ }),
2130
+ { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
2131
+ );
2132
+ }
2133
+
2134
+ function createWorkerSessionMetrics(): WorkerSessionMetrics {
2135
+ return { starts: 0, reuses: 0, rotations: 0, retained: 0, rotationReasons: {} };
2136
+ }
2137
+
2138
+ function snapshotWorkerSessionMetrics(metrics: WorkerSessionMetrics): WorkerSessionMetrics {
2139
+ return { ...metrics, rotationReasons: { ...metrics.rotationReasons } };
2140
+ }
2141
+
2142
+ function recordWorkerSessionMetric(metrics: WorkerSessionMetrics, diagnostic: WorkerSessionDiagnostic): void {
2143
+ if (diagnostic.event === "session_started") metrics.starts += 1;
2144
+ if (diagnostic.event === "session_reused") metrics.reuses += 1;
2145
+ if (diagnostic.event === "session_retained") metrics.retained += 1;
2146
+ if (diagnostic.event === "session_rotated") {
2147
+ metrics.rotations += 1;
2148
+ metrics.rotationReasons[diagnostic.reasonCode] = (metrics.rotationReasons[diagnostic.reasonCode] ?? 0) + 1;
2149
+ }
2150
+ }
2151
+
1159
2152
  function createWorkerCostState(): WorkerCostState {
1160
2153
  return {
1161
2154
  total: 0,
@@ -1192,9 +2185,9 @@ function recordLiveWorkerCost(
1192
2185
 
1193
2186
  function finalizeWorkerCost(
1194
2187
  state: WorkerCostState,
1195
- outcome: Pick<SessionOutcome, "task" | "attempt" | "workerCostTotal">,
2188
+ worker: string,
2189
+ outcome: Pick<SessionOutcome, "workerCostTotal">,
1196
2190
  ): void {
1197
- const worker = workerKey(outcome.task.taskId, outcome.attempt);
1198
2191
  state.finalizedByWorker.set(worker, finiteNonNegativeNumber(outcome.workerCostTotal) ?? 0);
1199
2192
  state.liveByWorker.delete(worker);
1200
2193
  for (const messageKey of state.liveByMessage.keys()) {
@@ -1263,6 +2256,43 @@ function subtaskProgress(
1263
2256
  });
1264
2257
  }
1265
2258
 
2259
+ function emitWorkerSessionProgress(
2260
+ runtime: RuntimeOptions,
2261
+ tasks: readonly Task[],
2262
+ task: Pick<Task, "taskId" | "title" | "statusItems">,
2263
+ attempts: readonly TaskAttemptSummary[],
2264
+ attempt: number,
2265
+ diagnostic: WorkerSessionDiagnostic,
2266
+ ): void {
2267
+ recordWorkerSessionMetric(runtime.workerSessionMetrics, diagnostic);
2268
+ const contextText =
2269
+ diagnostic.contextUsagePercent === undefined
2270
+ ? ""
2271
+ : ` at ${diagnostic.contextUsagePercent.toFixed(1)}% context usage`;
2272
+ const action =
2273
+ diagnostic.event === "session_started"
2274
+ ? "started"
2275
+ : diagnostic.event === "session_reused"
2276
+ ? "reused"
2277
+ : diagnostic.event === "session_rotated"
2278
+ ? "rotated"
2279
+ : "retained";
2280
+ emitProgress(runtime, `Worker session ${action}${contextText} (${diagnostic.reasonCode}).`, {
2281
+ phase: "worker_session",
2282
+ taskId: task.taskId,
2283
+ title: task.title,
2284
+ attempt,
2285
+ status: "in_progress",
2286
+ activeStatus: `Worker session ${action}`,
2287
+ workerSessionEvent: diagnostic.event,
2288
+ workerSessionReason: diagnostic.reasonCode,
2289
+ workerSessionContextUsagePercent: diagnostic.contextUsagePercent,
2290
+ workerSessionContextThresholdPercent: diagnostic.contextThresholdPercent,
2291
+ ...currentTaskProgress(task, "in_progress"),
2292
+ taskProgress: buildTaskProgressModel({ tasks, attempts, currentTaskId: task.taskId }),
2293
+ });
2294
+ }
2295
+
1266
2296
  function emitWorkerEventProgress(
1267
2297
  runtime: RuntimeOptions,
1268
2298
  tasks: readonly Task[],
@@ -1278,6 +2308,7 @@ function emitWorkerEventProgress(
1278
2308
  usageCostTotal?: number;
1279
2309
  usageCostKey?: string;
1280
2310
  },
2311
+ accountingWorker = workerKey(task.taskId, attempt),
1281
2312
  ): void {
1282
2313
  const worker = workerKey(task.taskId, attempt);
1283
2314
  let activeStatus = runtime.workerActivityByWorker.get(worker);
@@ -1318,7 +2349,7 @@ function emitWorkerEventProgress(
1318
2349
  }
1319
2350
 
1320
2351
  const costChanged =
1321
- event.usageCostTotal !== undefined && recordLiveWorkerCost(runtime.workerCostState, worker, event);
2352
+ event.usageCostTotal !== undefined && recordLiveWorkerCost(runtime.workerCostState, accountingWorker, event);
1322
2353
 
1323
2354
  if (event.type === "message_end" && event.activity) {
1324
2355
  emitProgress(runtime, event.activity, {
@@ -1390,7 +2421,7 @@ function emitWorkerEventProgress(
1390
2421
  }
1391
2422
 
1392
2423
  function stripToolOutcomePrefix(activity: string): string {
1393
- return activity.replace(/^(?:(?:Finished|Failed):\s*)+/i, "");
2424
+ return activity.trim().replace(/^(?:(?:Finished|Failed):\s*)+/i, "");
1394
2425
  }
1395
2426
 
1396
2427
  function activeStatusFromWorkerText(text: string): string {
@@ -1591,6 +2622,43 @@ async function appendCommitNote(pathname: string, result: CommitAfterSessionResu
1591
2622
  await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
1592
2623
  }
1593
2624
 
2625
+ async function appendNetworkInterruptionEvidence(
2626
+ pathname: string,
2627
+ outcome: SessionOutcome,
2628
+ networkRetry: number,
2629
+ ): Promise<void> {
2630
+ const summary = extractResultSummary(outcome.assistantText || "").trim() || "TASK_RESULT:\nstatus: unknown";
2631
+ const lines = [
2632
+ "",
2633
+ `## TODO ${outcome.task.taskId} — ${outcome.task.title} (ordinary attempt ${outcome.attempt}, network interruption ${networkRetry})`,
2634
+ "",
2635
+ "Disposition: transient provider/transport failure; this is durable evidence, not an ordinary task attempt.",
2636
+ `Started: ${outcome.startedAt}`,
2637
+ `Ended: ${outcome.endedAt}`,
2638
+ `Worker error: ${outcome.error ?? "transient provider or transport failure"}`,
2639
+ ];
2640
+ if (outcome.sessionId) lines.push(`Session ID: ${outcome.sessionId}`);
2641
+ if (outcome.sessionFile) lines.push(`Session file: ${outcome.sessionFile}`);
2642
+ if (outcome.workerCostSource || outcome.workerCostTotal > 0) {
2643
+ lines.push(`Worker cost: ${outcome.workerCostTotal} (${outcome.workerCostSource ?? "unavailable"})`);
2644
+ }
2645
+ if (outcome.workerUsage) {
2646
+ lines.push(
2647
+ `Worker token usage: input=${outcome.workerUsage.input}, output=${outcome.workerUsage.output}, cacheRead=${outcome.workerUsage.cacheRead}, cacheWrite=${outcome.workerUsage.cacheWrite}, total=${outcome.workerUsage.total}`,
2648
+ );
2649
+ }
2650
+ lines.push(
2651
+ "",
2652
+ "Safety: the replacement session must inspect the working tree and this evidence before continuing; completed side effects must not be blindly replayed.",
2653
+ "",
2654
+ "```text",
2655
+ summary,
2656
+ "```",
2657
+ "",
2658
+ );
2659
+ await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
2660
+ }
2661
+
1594
2662
  async function appendTaskResult(
1595
2663
  pathname: string,
1596
2664
  task: Task,
@@ -1629,6 +2697,25 @@ async function appendTaskResult(
1629
2697
  if (outcome.contextObservations.length > 0) {
1630
2698
  lines.push("", "Context observations:", ...outcome.contextObservations.map((item) => `- ${item}`));
1631
2699
  }
2700
+ if (outcome.workerCostSource || outcome.workerCostTotal > 0) {
2701
+ lines.push(`Worker cost: ${outcome.workerCostTotal} (${outcome.workerCostSource ?? "unavailable"})`);
2702
+ }
2703
+ if (outcome.workerUsage) {
2704
+ lines.push(
2705
+ `Worker token usage: input=${outcome.workerUsage.input}, output=${outcome.workerUsage.output}, cacheRead=${outcome.workerUsage.cacheRead}, cacheWrite=${outcome.workerUsage.cacheWrite}, total=${outcome.workerUsage.total}`,
2706
+ );
2707
+ }
2708
+ if (outcome.sessionDiagnostics?.length) {
2709
+ lines.push(
2710
+ "",
2711
+ "Worker session diagnostics:",
2712
+ ...outcome.sessionDiagnostics.map((item) => {
2713
+ const context =
2714
+ item.contextUsagePercent === undefined ? "" : ` context=${item.contextUsagePercent.toFixed(1)}%`;
2715
+ return `- event=${item.event} reason=${item.reasonCode}${context}`;
2716
+ }),
2717
+ );
2718
+ }
1632
2719
  if (outcome.compactionEvents.length > 0) {
1633
2720
  lines.push("", "Compaction events:", ...outcome.compactionEvents.map((item) => `- ${item}`));
1634
2721
  }