pi-long-task 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,18 @@ import type {
10
10
  } from "./types.ts";
11
11
  import { commitAfterSession, gitDirtyPaths, shouldCommitOutcome, type CommitAfterSessionResult } from "./git.ts";
12
12
  import { formatCoordinatorResultMessage } from "./render.ts";
13
+ import { classifyNetworkFailure } from "./network_failure.ts";
14
+ import {
15
+ formatNetworkRecoveryStatus,
16
+ recoverNetworkOperation,
17
+ type NetworkRecoveryEvent,
18
+ type NetworkRecoveryEventType,
19
+ } from "./network_recovery.ts";
20
+ import {
21
+ DEFAULT_NETWORK_RECOVERY_CONFIG,
22
+ resolveNetworkRecoveryConfig,
23
+ type NetworkRecoveryConfig,
24
+ } from "./network_recovery_config.ts";
13
25
  import { extractResultSummary, hasCompleteTaskResult } from "./result_writer.ts";
14
26
  import { runGuardedSessionPrompt } from "./session_guard.ts";
15
27
  import {
@@ -79,6 +91,7 @@ export const DEFAULT_COORDINATOR_OPTIONS = {
79
91
  todoThinking: "xhigh",
80
92
  workerSessionReuse: DEFAULT_WORKER_SESSION_REUSE_ENABLED,
81
93
  workerSessionReuseContextThresholdPercent: DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
94
+ networkRecovery: DEFAULT_NETWORK_RECOVERY_CONFIG,
82
95
  } as const;
83
96
 
84
97
  export type WorkerRunner = (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
@@ -88,6 +101,7 @@ export type CoordinatorProgressPhase =
88
101
  | "task_start"
89
102
  | "worker_session"
90
103
  | "worker_tool"
104
+ | "network_wait"
91
105
  | "task_done"
92
106
  | "task_blocked"
93
107
  | "task_failed"
@@ -150,6 +164,12 @@ export interface CoordinatorProgressUpdate {
150
164
  workerSessionReason?: string;
151
165
  workerSessionContextUsagePercent?: number;
152
166
  workerSessionContextThresholdPercent?: number;
167
+ networkRecoveryEvent?: NetworkRecoveryEventType;
168
+ networkRetryCount?: number;
169
+ networkOutageElapsedMs?: number;
170
+ networkNextRetryAtMs?: number;
171
+ networkNextRetryInMs?: number;
172
+ networkFailureReason?: string;
153
173
  }
154
174
 
155
175
  export type CoordinatorProgressHandler = (update: CoordinatorProgressUpdate) => void;
@@ -182,6 +202,8 @@ export interface RunCoordinatorOptions extends PiLongTaskInput {
182
202
  steeringQueue?: SerializedSteeringQueue;
183
203
  /** Runs after rebase/validation and immediately before the revision is atomically persisted. */
184
204
  onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
205
+ /** Receives coordinator-level outage lifecycle events for parent orchestrators and status integrations. */
206
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
185
207
  }
186
208
 
187
209
  export interface TodoPlannerOptions {
@@ -200,6 +222,8 @@ export interface TodoPlannerOptions {
200
222
  plannerPrompt?: string;
201
223
  /** Structured revision context supplied alongside plannerPrompt. */
202
224
  planRevision?: Readonly<PlanRevisionRequest>;
225
+ /** Normalized coordinator recovery policy; network wait is excluded from operation timeouts. */
226
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
203
227
  }
204
228
 
205
229
  export interface TaskAttemptSummary {
@@ -280,6 +304,7 @@ interface RuntimeOptions {
280
304
  todoThinking: string;
281
305
  workerSessionReuse: boolean;
282
306
  workerSessionReuseContextThresholdPercent: number;
307
+ networkRecovery: NetworkRecoveryConfig;
283
308
  todoTimeoutMs: number;
284
309
  todoGracefulShutdownMs: number;
285
310
  workerRunner: WorkerRunner;
@@ -298,6 +323,11 @@ interface RuntimeOptions {
298
323
  workerSessionMetrics: WorkerSessionMetrics;
299
324
  steeringQueue?: SerializedSteeringQueue;
300
325
  onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
326
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
327
+ lastProgress?: CoordinatorProgressUpdate;
328
+ progressClosed: boolean;
329
+ networkRecoverySequence: number;
330
+ activeNetworkRecoveries: Map<number, NetworkRecoveryEvent>;
301
331
  }
302
332
 
303
333
  type RetainedWorkerReuseScope = "sequential_task" | "partial_continuation";
@@ -733,6 +763,176 @@ function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortS
733
763
  return AbortSignal.any(available);
734
764
  }
735
765
 
766
+ /** Retains the full worker result while exposing its provider error to the shared classifier. */
767
+ class WorkerNetworkFailure extends Error {
768
+ readonly outcome: SessionOutcome;
769
+
770
+ constructor(outcome: SessionOutcome) {
771
+ const message = outcome.error ?? "worker network operation failed";
772
+ super(message, { cause: workerFailureValue(outcome) });
773
+ this.name = "WorkerNetworkFailure";
774
+ this.outcome = outcome;
775
+ }
776
+ }
777
+
778
+ interface WorkerRecoveryExecutionOptions {
779
+ workerOptions: RunWorkerTaskOptions;
780
+ run: (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
781
+ taskResultPath: string;
782
+ networkRecovery: Readonly<NetworkRecoveryConfig>;
783
+ signal?: AbortSignal;
784
+ onInterruption?: (outcome: SessionOutcome) => void;
785
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
786
+ }
787
+
788
+ /**
789
+ * Run one ordinary worker attempt, replacing only transport-failed sessions.
790
+ * Pi owns bounded request retries inside session.prompt(); therefore an outcome
791
+ * reaches this boundary only after those retries have settled. Coordinator
792
+ * probes retain the same task/attempt and always receive a recovery prompt.
793
+ */
794
+ async function runWorkerAttemptWithNetworkRecovery(options: WorkerRecoveryExecutionOptions): Promise<SessionOutcome> {
795
+ const interrupted: SessionOutcome[] = [];
796
+ const thrownErrors = new Map<SessionOutcome, unknown>();
797
+ const execute = async (workerOptions: RunWorkerTaskOptions): Promise<SessionOutcome> => {
798
+ try {
799
+ return await options.run(workerOptions);
800
+ } catch (error) {
801
+ const outcome = buildWorkerSessionCreationFailureOutcome(workerOptions, error);
802
+ thrownErrors.set(outcome, error);
803
+ return outcome;
804
+ }
805
+ };
806
+ const recordRecoverableInterruption = async (outcome: SessionOutcome): Promise<void> => {
807
+ await appendNetworkInterruptionEvidence(options.taskResultPath, outcome, interrupted.length + 1);
808
+ interrupted.push(outcome);
809
+ options.onInterruption?.(outcome);
810
+ };
811
+
812
+ const initial = await execute(options.workerOptions);
813
+ const initialFailure = workerFailureValue(initial);
814
+ const initialClassification = initialFailure === undefined ? undefined : classifyNetworkFailure(initialFailure);
815
+ if (initialClassification && isFailFastWorkerFailure(initialClassification.reason)) {
816
+ throw new WorkerNetworkFailure(initial);
817
+ }
818
+ if (!options.networkRecovery.enabled || !initialClassification?.recoverable) {
819
+ if (thrownErrors.has(initial)) throw thrownErrors.get(initial);
820
+ return initial;
821
+ }
822
+ await recordRecoverableInterruption(initial);
823
+
824
+ try {
825
+ const recovered = await recoverNetworkOperation({
826
+ initialFailure: new WorkerNetworkFailure(initial),
827
+ config: options.networkRecovery,
828
+ signal: options.signal,
829
+ onEvent: options.onNetworkRecovery,
830
+ retry: async ({ retryCount, signal }) => {
831
+ const previous = interrupted.at(-1)!;
832
+ const resumed = await execute({
833
+ ...options.workerOptions,
834
+ // The recovery signal includes both run cancellation and the outage
835
+ // deadline without replacing the assignment/steering cancellation.
836
+ abortSignal: combineAbortSignals(options.workerOptions.abortSignal, signal),
837
+ networkRecoveryContext: {
838
+ retryCount,
839
+ durableEvidencePath: options.taskResultPath,
840
+ priorSessionId: previous.sessionId,
841
+ failure: previous.error ?? "transient provider or transport failure",
842
+ },
843
+ });
844
+ const resumedFailure = workerFailureValue(resumed);
845
+ const classification = resumedFailure === undefined ? undefined : classifyNetworkFailure(resumedFailure);
846
+ if (classification?.recoverable) {
847
+ await recordRecoverableInterruption(resumed);
848
+ }
849
+ if (resumed.error) {
850
+ if (
851
+ thrownErrors.has(resumed) &&
852
+ !classification?.recoverable &&
853
+ !isFailFastWorkerFailure(classification!.reason)
854
+ ) {
855
+ throw thrownErrors.get(resumed);
856
+ }
857
+ throw new WorkerNetworkFailure(resumed);
858
+ }
859
+ return resumed;
860
+ },
861
+ });
862
+ return mergeWorkerRecoveryOutcomes(interrupted, recovered.value);
863
+ } catch (error) {
864
+ // If connectivity recovered but the fresh session failed deterministically,
865
+ // hand its outcome back to the ordinary worker failure path immediately.
866
+ if (error instanceof WorkerNetworkFailure) {
867
+ const merged = mergeWorkerRecoveryOutcomes(interrupted, error.outcome);
868
+ const mergedFailure = workerFailureValue(merged);
869
+ const classification = mergedFailure === undefined ? undefined : classifyNetworkFailure(mergedFailure);
870
+ if (classification && isFailFastWorkerFailure(classification.reason)) {
871
+ throw new WorkerNetworkFailure(merged);
872
+ }
873
+ return merged;
874
+ }
875
+ throw error;
876
+ }
877
+ }
878
+
879
+ function workerFailureValue(outcome: SessionOutcome): unknown {
880
+ return outcome.failure ?? outcome.error;
881
+ }
882
+
883
+ function isFailFastWorkerFailure(reason: ReturnType<typeof classifyNetworkFailure>["reason"]): boolean {
884
+ return [
885
+ "authentication",
886
+ "authorization",
887
+ "billing",
888
+ "quota_exhausted",
889
+ "invalid_model",
890
+ "invalid_request",
891
+ "http_client_error",
892
+ "non_retryable_server_error",
893
+ ].includes(reason);
894
+ }
895
+
896
+ function mergeWorkerRecoveryOutcomes(interrupted: readonly SessionOutcome[], final: SessionOutcome): SessionOutcome {
897
+ if (interrupted.length === 0) return final;
898
+ const usage = addWorkerUsage([...interrupted.map((item) => item.workerUsage), final.workerUsage]);
899
+ return {
900
+ ...final,
901
+ startedAt: interrupted[0].startedAt,
902
+ contextObservations: [
903
+ ...interrupted.flatMap((item, index) => [
904
+ `network interruption ${index + 1}: ${item.error ?? "transient provider or transport failure"}`,
905
+ ...item.contextObservations,
906
+ ]),
907
+ ...final.contextObservations,
908
+ ],
909
+ compactionEvents: [...interrupted.flatMap((item) => item.compactionEvents), ...final.compactionEvents],
910
+ events: [...interrupted.flatMap((item) => item.events), ...final.events],
911
+ workerCostTotal: [...interrupted, final].reduce((total, item) => total + item.workerCostTotal, 0),
912
+ workerCostSource: "network_recovery_aggregate",
913
+ workerUsage: usage,
914
+ sessionDiagnostics: [
915
+ ...interrupted.flatMap((item) => item.sessionDiagnostics ?? []),
916
+ ...(final.sessionDiagnostics ?? []),
917
+ ],
918
+ };
919
+ }
920
+
921
+ function addWorkerUsage(values: Array<WorkerUsageTotals | undefined>): WorkerUsageTotals | undefined {
922
+ const available = values.filter((value): value is WorkerUsageTotals => Boolean(value));
923
+ if (available.length === 0) return undefined;
924
+ return available.reduce<WorkerUsageTotals>(
925
+ (total, value) => ({
926
+ input: total.input + value.input,
927
+ output: total.output + value.output,
928
+ cacheRead: total.cacheRead + value.cacheRead,
929
+ cacheWrite: total.cacheWrite + value.cacheWrite,
930
+ total: total.total + value.total,
931
+ }),
932
+ { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
933
+ );
934
+ }
935
+
736
936
  export function workerSessionHealthForOutcome(
737
937
  outcome: Pick<SessionOutcome, "timedOut" | "aborted" | "error" | "assistantText">,
738
938
  cancelled = false,
@@ -949,6 +1149,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
949
1149
  thinkingLevel: runtime.taskThinking,
950
1150
  abortSignal: combineAbortSignals(runtime.abortSignal, assignmentController.signal),
951
1151
  sessionFactory: runtime.workerSessionFactory,
1152
+ networkRecovery: runtime.networkRecovery,
952
1153
  now: runtime.now,
953
1154
  onEvent: (event) => {
954
1155
  if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
@@ -966,18 +1167,39 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
966
1167
  },
967
1168
  };
968
1169
  let outcome: SessionOutcome;
1170
+ const networkInterruptedOutcomes: SessionOutcome[] = [];
969
1171
  try {
970
- outcome = workerSessionOwner
971
- ? await workerSessionOwner.run(workerOptions, assignmentIdentity)
972
- : await runtime.workerRunner(workerOptions);
1172
+ outcome = await runWorkerAttemptWithNetworkRecovery({
1173
+ workerOptions,
1174
+ run: (resumedOptions) =>
1175
+ workerSessionOwner
1176
+ ? workerSessionOwner.run(resumedOptions, assignmentIdentity)
1177
+ : runtime.workerRunner(resumedOptions),
1178
+ taskResultPath: runtime.taskResultPath,
1179
+ networkRecovery: runtime.networkRecovery,
1180
+ signal: workerOptions.abortSignal,
1181
+ onInterruption: (interrupted) => networkInterruptedOutcomes.push(interrupted),
1182
+ onNetworkRecovery: createNetworkRecoveryProgressHandler(runtime),
1183
+ });
973
1184
  } catch (error) {
974
1185
  if (!assignmentState.obsolete) {
1186
+ const terminalOutcome = error instanceof WorkerNetworkFailure ? error.outcome : undefined;
1187
+ if (terminalOutcome || networkInterruptedOutcomes.length > 0) {
1188
+ finalizeWorkerCost(runtime.workerCostState, accountingWorker, {
1189
+ workerCostTotal:
1190
+ terminalOutcome?.workerCostTotal ??
1191
+ networkInterruptedOutcomes.reduce((total, interrupted) => total + interrupted.workerCostTotal, 0),
1192
+ });
1193
+ }
975
1194
  throw error;
976
1195
  }
977
1196
  // A cancellation-aware custom runner may reject instead of returning
978
1197
  // an aborted outcome. Preserve historical evidence, but never let that
979
1198
  // obsolete rejection terminate or update the replacement assignment.
980
- outcome = buildWorkerSessionCreationFailureOutcome(workerOptions, error);
1199
+ outcome = mergeWorkerRecoveryOutcomes(
1200
+ networkInterruptedOutcomes,
1201
+ buildWorkerSessionCreationFailureOutcome(workerOptions, error),
1202
+ );
981
1203
  outcome.aborted = true;
982
1204
  }
983
1205
  finalizeWorkerCost(runtime.workerCostState, accountingWorker, outcome);
@@ -1248,6 +1470,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
1248
1470
  });
1249
1471
  return result;
1250
1472
  } finally {
1473
+ runtime.progressClosed = true;
1474
+ runtime.activeNetworkRecoveries.clear();
1251
1475
  removeSteeringProcessor?.();
1252
1476
  await workerSessionOwner?.dispose();
1253
1477
  }
@@ -1319,19 +1543,59 @@ async function extractTodoMarkdownWithOneRepair(
1319
1543
  }
1320
1544
 
1321
1545
  async function requestTodoPlan(inputText: string, runtime: RuntimeOptions): Promise<string> {
1322
- return runtime.todoPlanner({
1323
- inputText,
1324
- cwd: runtime.cwd,
1325
- runDir: runtime.runDir,
1326
- thinkingLevel: runtime.todoThinking,
1327
- model: runtime.workerModel,
1328
- abortSignal: runtime.abortSignal,
1329
- timeoutMs: runtime.todoTimeoutMs,
1330
- gracefulShutdownMs: runtime.todoGracefulShutdownMs,
1331
- sessionFactory: runtime.todoSessionFactory,
1332
- onDiagnostic: (diagnostic) => recordPlannerDiagnostic(runtime, diagnostic),
1333
- goal: runtime.goal,
1334
- });
1546
+ return runPlannerOperationWithNetworkRecovery(
1547
+ {
1548
+ inputText,
1549
+ cwd: runtime.cwd,
1550
+ runDir: runtime.runDir,
1551
+ thinkingLevel: runtime.todoThinking,
1552
+ model: runtime.workerModel,
1553
+ abortSignal: runtime.abortSignal,
1554
+ timeoutMs: runtime.todoTimeoutMs,
1555
+ gracefulShutdownMs: runtime.todoGracefulShutdownMs,
1556
+ sessionFactory: runtime.todoSessionFactory,
1557
+ networkRecovery: runtime.networkRecovery,
1558
+ onDiagnostic: (diagnostic) => recordPlannerDiagnostic(runtime, diagnostic),
1559
+ goal: runtime.goal,
1560
+ },
1561
+ runtime,
1562
+ );
1563
+ }
1564
+
1565
+ /**
1566
+ * Retry a side-effect-free planner request only after its provider boundary has
1567
+ * failed. The default planner disables tools and disposes every session before
1568
+ * rejecting, so each retry rotates unsafe conversation state while replaying
1569
+ * only the complete immutable planning context. Recovery owns no planner
1570
+ * repair/attempt counter, and each fresh call retains the planner timeout;
1571
+ * backoff remains governed solely by the separate outage deadline.
1572
+ */
1573
+ async function runPlannerOperationWithNetworkRecovery(
1574
+ plannerOptions: TodoPlannerOptions,
1575
+ runtime: RuntimeOptions,
1576
+ ): Promise<string> {
1577
+ const run = (recoverySignal?: AbortSignal) =>
1578
+ runtime.todoPlanner({
1579
+ ...plannerOptions,
1580
+ abortSignal: combineAbortSignals(plannerOptions.abortSignal, recoverySignal),
1581
+ });
1582
+
1583
+ try {
1584
+ return await run();
1585
+ } catch (initialFailure) {
1586
+ const classification = classifyNetworkFailure(initialFailure);
1587
+ if (!runtime.networkRecovery.enabled || !classification.recoverable) {
1588
+ throw initialFailure;
1589
+ }
1590
+ const recovered = await recoverNetworkOperation({
1591
+ initialFailure,
1592
+ config: runtime.networkRecovery,
1593
+ signal: plannerOptions.abortSignal,
1594
+ onEvent: createNetworkRecoveryProgressHandler(runtime),
1595
+ retry: ({ signal }) => run(signal),
1596
+ });
1597
+ return recovered.value;
1598
+ }
1335
1599
  }
1336
1600
 
1337
1601
  async function generateSteeringPlanRevision(options: {
@@ -1364,21 +1628,25 @@ async function generateSteeringPlanRevision(options: {
1364
1628
  }
1365
1629
  : undefined,
1366
1630
  planner: ({ prompt, request }) =>
1367
- options.runtime.todoPlanner({
1368
- inputText: prompt,
1369
- plannerPrompt: prompt,
1370
- planRevision: request,
1371
- cwd: options.runtime.cwd,
1372
- runDir: options.runtime.runDir,
1373
- thinkingLevel: options.runtime.todoThinking,
1374
- model: options.runtime.workerModel,
1375
- abortSignal: options.runtime.abortSignal,
1376
- timeoutMs: options.runtime.todoTimeoutMs,
1377
- gracefulShutdownMs: options.runtime.todoGracefulShutdownMs,
1378
- sessionFactory: options.runtime.todoSessionFactory,
1379
- onDiagnostic: (diagnostic) => recordPlannerDiagnostic(options.runtime, diagnostic),
1380
- goal: options.runtime.goal,
1381
- }),
1631
+ runPlannerOperationWithNetworkRecovery(
1632
+ {
1633
+ inputText: prompt,
1634
+ plannerPrompt: prompt,
1635
+ planRevision: request,
1636
+ cwd: options.runtime.cwd,
1637
+ runDir: options.runtime.runDir,
1638
+ thinkingLevel: options.runtime.todoThinking,
1639
+ model: options.runtime.workerModel,
1640
+ abortSignal: options.runtime.abortSignal,
1641
+ timeoutMs: options.runtime.todoTimeoutMs,
1642
+ gracefulShutdownMs: options.runtime.todoGracefulShutdownMs,
1643
+ sessionFactory: options.runtime.todoSessionFactory,
1644
+ networkRecovery: options.runtime.networkRecovery,
1645
+ onDiagnostic: (diagnostic) => recordPlannerDiagnostic(options.runtime, diagnostic),
1646
+ goal: options.runtime.goal,
1647
+ },
1648
+ options.runtime,
1649
+ ),
1382
1650
  });
1383
1651
  }
1384
1652
 
@@ -1601,7 +1869,10 @@ async function runTodoPlannerPrompt(options: {
1601
1869
  if (promptResult.error) {
1602
1870
  const message = `TODO planner failed: ${promptResult.error}`;
1603
1871
  options.onDiagnostic?.(plannerPromptDiagnostic("failure", message, promptResult));
1604
- throw new TodoGenerationError(message);
1872
+ throw new TodoGenerationError(
1873
+ message,
1874
+ promptResult.failure === undefined ? undefined : { cause: promptResult.failure },
1875
+ );
1605
1876
  }
1606
1877
  if (!promptResult.assistantText) {
1607
1878
  const message = "TODO planner did not return assistant text.";
@@ -1652,6 +1923,10 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
1652
1923
  contextThresholdPercent:
1653
1924
  options.workerSessionReuseContextThresholdPercent ?? parsedWorkerConfig.workerSessionReuseContextThresholdPercent,
1654
1925
  });
1926
+ const networkRecovery = resolveNetworkRecoveryConfig({
1927
+ ...parsedWorkerConfig.networkRecovery,
1928
+ ...options.networkRecovery,
1929
+ });
1655
1930
 
1656
1931
  return {
1657
1932
  cwd,
@@ -1675,6 +1950,7 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
1675
1950
  todoThinking: options.todoThinking ?? DEFAULT_COORDINATOR_OPTIONS.todoThinking,
1676
1951
  workerSessionReuse: workerSessionReuseConfig.enabled,
1677
1952
  workerSessionReuseContextThresholdPercent: workerSessionReuseConfig.contextThresholdPercent,
1953
+ networkRecovery,
1678
1954
  workerRunner: options.workerRunner ?? runWorkerTask,
1679
1955
  useRetainedWorkerLifecycle: options.workerRunner === undefined,
1680
1956
  todoPlanner: options.todoPlanner ?? runTodoPlanner,
@@ -1691,6 +1967,10 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
1691
1967
  workerSessionMetrics: createWorkerSessionMetrics(),
1692
1968
  steeringQueue: options.steeringQueue,
1693
1969
  onPlanRevisionAccepted: options.onPlanRevisionAccepted,
1970
+ onNetworkRecovery: options.onNetworkRecovery,
1971
+ progressClosed: false,
1972
+ networkRecoverySequence: 0,
1973
+ activeNetworkRecoveries: new Map(),
1694
1974
  };
1695
1975
  }
1696
1976
 
@@ -1699,7 +1979,8 @@ function emitProgress(
1699
1979
  message: string,
1700
1980
  update: Omit<CoordinatorProgressUpdate, "message" | "runId" | "todoPath" | "resultPath" | "workerCostTotal">,
1701
1981
  ): void {
1702
- runtime.onProgress?.({
1982
+ if (runtime.progressClosed) return;
1983
+ const progress: CoordinatorProgressUpdate = {
1703
1984
  message,
1704
1985
  runId: runtime.runId,
1705
1986
  todoPath: runtime.todoPath,
@@ -1707,9 +1988,107 @@ function emitProgress(
1707
1988
  workerCostTotal: runtime.workerCostState.total,
1708
1989
  ...update,
1709
1990
  goal: runtime.goal,
1991
+ };
1992
+ runtime.lastProgress = progress;
1993
+ const activeRecovery = latestNetworkRecovery(runtime.activeNetworkRecoveries);
1994
+ if (activeRecovery) {
1995
+ publishNetworkRecoveryProgress(runtime, activeRecovery);
1996
+ } else {
1997
+ runtime.onProgress?.(progress);
1998
+ }
1999
+ }
2000
+
2001
+ /**
2002
+ * Bridge one recovery lifecycle into coordinator progress without replacing the
2003
+ * last stable task/phase update. A recovered operation restores whichever
2004
+ * ordinary status is current; terminal failures are left for the normal final
2005
+ * status path. Operation IDs prevent an older concurrent recovery from
2006
+ * repainting a newer outage or completion.
2007
+ */
2008
+ function createNetworkRecoveryProgressHandler(runtime: RuntimeOptions): (event: NetworkRecoveryEvent) => void {
2009
+ const operationId = ++runtime.networkRecoverySequence;
2010
+ let cleaned = false;
2011
+
2012
+ return (event) => {
2013
+ runtime.onNetworkRecovery?.(event);
2014
+ if (cleaned || runtime.progressClosed) return;
2015
+
2016
+ if (event.type === "cleanup") {
2017
+ cleaned = true;
2018
+ runtime.activeNetworkRecoveries.delete(operationId);
2019
+ return;
2020
+ }
2021
+
2022
+ if (isTerminalNetworkRecoveryEvent(event.type)) {
2023
+ runtime.activeNetworkRecoveries.delete(operationId);
2024
+ if (event.type === "recovered") {
2025
+ const active = latestNetworkRecovery(runtime.activeNetworkRecoveries);
2026
+ if (active) {
2027
+ publishNetworkRecoveryProgress(runtime, active);
2028
+ } else if (runtime.lastProgress) {
2029
+ runtime.onProgress?.({ ...runtime.lastProgress, workerCostTotal: runtime.workerCostState.total });
2030
+ }
2031
+ }
2032
+ return;
2033
+ }
2034
+
2035
+ runtime.activeNetworkRecoveries.set(operationId, event);
2036
+ if (operationId === latestNetworkRecoveryId(runtime.activeNetworkRecoveries)) {
2037
+ publishNetworkRecoveryProgress(runtime, event);
2038
+ }
2039
+ };
2040
+ }
2041
+
2042
+ function publishNetworkRecoveryProgress(runtime: RuntimeOptions, event: NetworkRecoveryEvent): void {
2043
+ if (runtime.progressClosed) return;
2044
+ const stable = runtime.lastProgress;
2045
+ const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
2046
+ const message = formatNetworkRecoveryStatus(event);
2047
+ runtime.onProgress?.({
2048
+ message,
2049
+ phase: "network_wait",
2050
+ runId: runtime.runId,
2051
+ todoPath: runtime.todoPath,
2052
+ resultPath: runtime.taskResultPath,
2053
+ workerCostTotal: runtime.workerCostState.total,
2054
+ goal: runtime.goal,
2055
+ taskId: stable?.taskId,
2056
+ title: stable?.title,
2057
+ attempt: stable?.attempt,
2058
+ totalTasks: stable?.totalTasks,
2059
+ currentTask: stable?.currentTask,
2060
+ subtasks: stable?.subtasks,
2061
+ taskProgress: stable?.taskProgress,
2062
+ activeStatus: message,
2063
+ networkRecoveryEvent: event.type,
2064
+ networkRetryCount: event.state.retryCount,
2065
+ networkOutageElapsedMs: event.state.elapsedMs,
2066
+ networkNextRetryAtMs: event.state.nextRetryAtMs,
2067
+ networkNextRetryInMs:
2068
+ event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
2069
+ networkFailureReason: event.state.lastFailure.reason,
1710
2070
  });
1711
2071
  }
1712
2072
 
2073
+ function latestNetworkRecovery(
2074
+ recoveries: ReadonlyMap<number, NetworkRecoveryEvent>,
2075
+ ): NetworkRecoveryEvent | undefined {
2076
+ const id = latestNetworkRecoveryId(recoveries);
2077
+ return id === undefined ? undefined : recoveries.get(id);
2078
+ }
2079
+
2080
+ function latestNetworkRecoveryId(recoveries: ReadonlyMap<number, NetworkRecoveryEvent>): number | undefined {
2081
+ let latest: number | undefined;
2082
+ for (const id of recoveries.keys()) {
2083
+ if (latest === undefined || id > latest) latest = id;
2084
+ }
2085
+ return latest;
2086
+ }
2087
+
2088
+ function isTerminalNetworkRecoveryEvent(type: NetworkRecoveryEventType): boolean {
2089
+ return type === "recovered" || type === "failed" || type === "cancelled" || type === "outage_expired";
2090
+ }
2091
+
1713
2092
  function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDiagnostic): void {
1714
2093
  const normalized: PlannerDiagnostic = {
1715
2094
  kind: diagnostic.kind,
@@ -2243,6 +2622,43 @@ async function appendCommitNote(pathname: string, result: CommitAfterSessionResu
2243
2622
  await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
2244
2623
  }
2245
2624
 
2625
+ async function appendNetworkInterruptionEvidence(
2626
+ pathname: string,
2627
+ outcome: SessionOutcome,
2628
+ networkRetry: number,
2629
+ ): Promise<void> {
2630
+ const summary = extractResultSummary(outcome.assistantText || "").trim() || "TASK_RESULT:\nstatus: unknown";
2631
+ const lines = [
2632
+ "",
2633
+ `## TODO ${outcome.task.taskId} — ${outcome.task.title} (ordinary attempt ${outcome.attempt}, network interruption ${networkRetry})`,
2634
+ "",
2635
+ "Disposition: transient provider/transport failure; this is durable evidence, not an ordinary task attempt.",
2636
+ `Started: ${outcome.startedAt}`,
2637
+ `Ended: ${outcome.endedAt}`,
2638
+ `Worker error: ${outcome.error ?? "transient provider or transport failure"}`,
2639
+ ];
2640
+ if (outcome.sessionId) lines.push(`Session ID: ${outcome.sessionId}`);
2641
+ if (outcome.sessionFile) lines.push(`Session file: ${outcome.sessionFile}`);
2642
+ if (outcome.workerCostSource || outcome.workerCostTotal > 0) {
2643
+ lines.push(`Worker cost: ${outcome.workerCostTotal} (${outcome.workerCostSource ?? "unavailable"})`);
2644
+ }
2645
+ if (outcome.workerUsage) {
2646
+ lines.push(
2647
+ `Worker token usage: input=${outcome.workerUsage.input}, output=${outcome.workerUsage.output}, cacheRead=${outcome.workerUsage.cacheRead}, cacheWrite=${outcome.workerUsage.cacheWrite}, total=${outcome.workerUsage.total}`,
2648
+ );
2649
+ }
2650
+ lines.push(
2651
+ "",
2652
+ "Safety: the replacement session must inspect the working tree and this evidence before continuing; completed side effects must not be blindly replayed.",
2653
+ "",
2654
+ "```text",
2655
+ summary,
2656
+ "```",
2657
+ "",
2658
+ );
2659
+ await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
2660
+ }
2661
+
2246
2662
  async function appendTaskResult(
2247
2663
  pathname: string,
2248
2664
  task: Task,
@@ -13,6 +13,7 @@ import {
13
13
  } from "./goal_spec.ts";
14
14
  import type { GoalLoopState } from "./goal_loop.ts";
15
15
  import type { GoalStateStore } from "./goal_state.ts";
16
+ import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
16
17
 
17
18
  export type GoalDiscoveryEntrypoint = "pi_goal_task" | "pi_long_task";
18
19
  export type GoalDiscoveryRoute = "discovery" | "direct";
@@ -41,6 +42,7 @@ export interface GoalDiscoveryRunnerOptions {
41
42
  model?: unknown;
42
43
  modelName?: string;
43
44
  thinkingLevel?: string;
45
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
44
46
  now: () => Date;
45
47
  }
46
48