pi-long-task 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,6 +13,7 @@ import {
13
13
  } from "./goal_spec.ts";
14
14
  import type { GoalLoopState } from "./goal_loop.ts";
15
15
  import type { GoalStateStore } from "./goal_state.ts";
16
+ import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
16
17
 
17
18
  export type GoalDiscoveryEntrypoint = "pi_goal_task" | "pi_long_task";
18
19
  export type GoalDiscoveryRoute = "discovery" | "direct";
@@ -41,6 +42,7 @@ export interface GoalDiscoveryRunnerOptions {
41
42
  model?: unknown;
42
43
  modelName?: string;
43
44
  thinkingLevel?: string;
45
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
44
46
  now: () => Date;
45
47
  }
46
48
 
package/src/goal_loop.ts CHANGED
@@ -92,6 +92,13 @@ export interface GoalWorkerResultState {
92
92
 
93
93
  export type GoalReviewerDecision = "complete" | "incomplete" | "blocked" | "failed";
94
94
 
95
+ export interface GoalReviewerRecoveryState {
96
+ interruptions: number;
97
+ evidencePaths: string[];
98
+ reviewerCostTotal: number;
99
+ updatedAt: string;
100
+ }
101
+
95
102
  export interface GoalReviewerResultState {
96
103
  decision: GoalReviewerDecision;
97
104
  complete: boolean;
@@ -130,6 +137,7 @@ export interface GoalIterationState {
130
137
  deadlineAt?: string;
131
138
  generatedTodo?: GeneratedTodoState;
132
139
  workerResult?: GoalWorkerResultState;
140
+ reviewerRecovery?: GoalReviewerRecoveryState;
133
141
  reviewerResult?: GoalReviewerResultState;
134
142
  completion?: GoalCompletionState;
135
143
  }
@@ -402,6 +410,70 @@ export function recordReviewerResult(
402
410
  });
403
411
  }
404
412
 
413
+ export function excludeNetworkOutageFromGoalDeadlines(
414
+ state: GoalLoopState,
415
+ outageMs: number,
416
+ operation: "planner" | "reviewer",
417
+ options: { now?: Date } = {},
418
+ ): GoalLoopState {
419
+ if (!Number.isFinite(outageMs) || outageMs <= 0 || isTerminalGoalLoopStatus(state.status)) {
420
+ return state;
421
+ }
422
+
423
+ const excludedMs = Math.ceil(outageMs);
424
+ const timestamp = (options.now ?? new Date()).toISOString();
425
+ const iterations = state.iterations.map((iteration) =>
426
+ iteration.iteration === state.currentIteration && iteration.deadlineAt
427
+ ? { ...iteration, deadlineAt: shiftTimestamp(iteration.deadlineAt, excludedMs), updatedAt: timestamp }
428
+ : iteration,
429
+ );
430
+ return withTrace(
431
+ {
432
+ ...state,
433
+ deadlineAt: state.deadlineAt ? shiftTimestamp(state.deadlineAt, excludedMs) : undefined,
434
+ iterations,
435
+ updatedAt: timestamp,
436
+ },
437
+ {
438
+ timestamp,
439
+ phase: state.phase,
440
+ event: "network_wait_excluded",
441
+ message: `Excluded ${excludedMs}ms of ${operation} network recovery from goal-loop deadlines.`,
442
+ iteration: state.currentIteration || undefined,
443
+ details: { operation, outageMs: excludedMs },
444
+ },
445
+ );
446
+ }
447
+
448
+ export function recordReviewerRecoveryEvidence(
449
+ state: GoalLoopState,
450
+ iterationNumber: number,
451
+ recovery: Omit<GoalReviewerRecoveryState, "updatedAt">,
452
+ options: { now?: Date } = {},
453
+ ): GoalLoopState {
454
+ const timestamp = (options.now ?? new Date()).toISOString();
455
+ return updateIteration(state, iterationNumber, ["todo_executed", "failed"], timestamp, (item) => ({
456
+ item: {
457
+ ...item,
458
+ updatedAt: timestamp,
459
+ reviewerRecovery: { ...recovery, evidencePaths: [...recovery.evidencePaths], updatedAt: timestamp },
460
+ },
461
+ phase: state.phase,
462
+ trace: {
463
+ timestamp,
464
+ phase: state.phase,
465
+ event: "reviewer_network_interrupted",
466
+ message: `Preserved ${recovery.interruptions} interrupted reviewer call(s) during network recovery.`,
467
+ iteration: iterationNumber,
468
+ details: {
469
+ interruptions: recovery.interruptions,
470
+ evidencePaths: recovery.evidencePaths,
471
+ reviewerCostTotal: recovery.reviewerCostTotal,
472
+ },
473
+ },
474
+ }));
475
+ }
476
+
405
477
  export function failGoalLoop(
406
478
  state: GoalLoopState,
407
479
  reason: string,
@@ -597,6 +669,10 @@ function withTrace(state: GoalLoopState, event: GoalLoopTraceEvent): GoalLoopSta
597
669
  return { ...state, trace: [...state.trace, event] };
598
670
  }
599
671
 
672
+ function shiftTimestamp(timestamp: string, deltaMs: number): string {
673
+ return new Date(Date.parse(timestamp) + deltaMs).toISOString();
674
+ }
675
+
600
676
  function positiveInteger(value: number | undefined, fallback: number): number {
601
677
  return optionalPositiveInteger(value) ?? fallback;
602
678
  }
@@ -20,6 +20,17 @@ import {
20
20
  } from "./goal_loop.ts";
21
21
  import { GoalStateStore } from "./goal_state.ts";
22
22
  import type { GoalSpecification } from "./goal_spec.ts";
23
+ import {
24
+ formatNetworkRecoveryStatus,
25
+ type NetworkRecoveryEvent,
26
+ type NetworkRecoveryEventType,
27
+ } from "./network_recovery.ts";
28
+ import {
29
+ resolveNetworkRecoveryConfig,
30
+ type NetworkRecoveryConfig,
31
+ type NetworkRecoveryConfigInput,
32
+ } from "./network_recovery_config.ts";
33
+ import { validatePlannerGracefulShutdownMs, validatePlannerTimeoutMs } from "./planner_config.ts";
23
34
  import { runGoalReviewSession, type GoalReviewResult, type GoalReviewerRunner } from "./goal_review.ts";
24
35
  import {
25
36
  runGoalTodoExecutionLongTask,
@@ -43,6 +54,7 @@ export type GoalLoopProgressPhase =
43
54
  | "todo_executed"
44
55
  | "review_start"
45
56
  | "reviewed"
57
+ | "network_wait"
46
58
  | "complete";
47
59
 
48
60
  export interface GoalLoopProgressUpdate {
@@ -70,6 +82,12 @@ export interface GoalLoopProgressUpdate {
70
82
  reviewerCostTotal: number;
71
83
  totalCost: number;
72
84
  childProgress?: CoordinatorProgressUpdate;
85
+ networkRecoveryEvent?: NetworkRecoveryEventType;
86
+ networkRetryCount?: number;
87
+ networkOutageElapsedMs?: number;
88
+ networkNextRetryAtMs?: number;
89
+ networkNextRetryInMs?: number;
90
+ networkFailureReason?: string;
73
91
  }
74
92
 
75
93
  export type GoalLoopProgressHandler = (update: GoalLoopProgressUpdate) => void;
@@ -92,10 +110,14 @@ export interface RunGoalLoopOptions extends GoalLoopLimitInput {
92
110
  thinkingLevel?: string;
93
111
  maxBashTimeoutMs?: number;
94
112
  maxAttemptsPerTask?: number;
113
+ todoTimeoutMs?: number;
114
+ todoGracefulShutdownMs?: number;
115
+ networkRecovery?: NetworkRecoveryConfigInput;
95
116
  commit?: boolean;
96
117
  now?: () => Date;
97
118
  onWorkerProgress?: (update: CoordinatorProgressUpdate) => void;
98
119
  onProgress?: GoalLoopProgressHandler;
120
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
99
121
  }
100
122
 
101
123
  export interface GoalLoopRunResult {
@@ -120,6 +142,9 @@ export class GoalLoopOrchestratorError extends Error {
120
142
 
121
143
  export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoopRunResult> {
122
144
  const now = options.now ?? (() => new Date());
145
+ const networkRecovery = resolveNetworkRecoveryConfig(options.networkRecovery);
146
+ const todoTimeoutMs = validatePlannerTimeoutMs(options.todoTimeoutMs);
147
+ const todoGracefulShutdownMs = validatePlannerGracefulShutdownMs(options.todoGracefulShutdownMs);
123
148
  let state =
124
149
  options.initialState ??
125
150
  createGoalLoopState({
@@ -149,7 +174,9 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
149
174
  await store.saveState(state);
150
175
  await store.initializeResultIfMissing(state);
151
176
  await store.appendNewTraceEvents(await store.durableTraceLength(), state);
177
+ let progressClosed = false;
152
178
  const publish = (phase: GoalLoopProgressPhase, message: string, extra: Partial<GoalLoopProgressUpdate> = {}) => {
179
+ if (progressClosed) return;
153
180
  options.onProgress?.({
154
181
  message,
155
182
  phase,
@@ -173,6 +200,39 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
173
200
  ...extra,
174
201
  });
175
202
  };
203
+ const recoveryProgress = (
204
+ restorePhase: GoalLoopProgressPhase,
205
+ restoreMessage: string,
206
+ extra: Partial<GoalLoopProgressUpdate> = {},
207
+ ) => {
208
+ let cleaned = false;
209
+ return (event: NetworkRecoveryEvent) => {
210
+ options.onNetworkRecovery?.(event);
211
+ if (cleaned || progressClosed) return;
212
+ if (event.type === "cleanup") {
213
+ cleaned = true;
214
+ return;
215
+ }
216
+ if (event.type === "recovered") {
217
+ publish(restorePhase, restoreMessage, extra);
218
+ return;
219
+ }
220
+ if (isTerminalNetworkRecoveryProgressEvent(event.type)) return;
221
+
222
+ const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
223
+ const message = formatNetworkRecoveryStatus(event);
224
+ publish("network_wait", message, {
225
+ ...extra,
226
+ networkRecoveryEvent: event.type,
227
+ networkRetryCount: event.state.retryCount,
228
+ networkOutageElapsedMs: event.state.elapsedMs,
229
+ networkNextRetryAtMs: event.state.nextRetryAtMs,
230
+ networkNextRetryInMs:
231
+ event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
232
+ networkFailureReason: event.state.lastFailure.reason,
233
+ });
234
+ };
235
+ };
176
236
 
177
237
  publish("goal_start", `Starting goal loop: ${state.goal}`);
178
238
 
@@ -186,6 +246,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
186
246
  options,
187
247
  now,
188
248
  publish,
249
+ networkRecovery,
189
250
  });
190
251
  }
191
252
 
@@ -226,8 +287,16 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
226
287
  modelName: options.modelName,
227
288
  thinkingLevel: options.thinkingLevel,
228
289
  maxBashTimeoutMs: options.maxBashTimeoutMs,
290
+ todoTimeoutMs,
291
+ todoGracefulShutdownMs,
292
+ networkRecovery,
229
293
  now,
230
294
  goalSpecification,
295
+ onNetworkRecovery: recoveryProgress(
296
+ "todo_generation_start",
297
+ `Goal iteration ${nextIteration}: generating TODO markdown.`,
298
+ { iteration: nextIteration },
299
+ ),
231
300
  });
232
301
  generationResults.push(generation);
233
302
  state = generation.state;
@@ -251,8 +320,16 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
251
320
  modelName: options.modelName,
252
321
  thinkingLevel: options.thinkingLevel,
253
322
  maxBashTimeoutMs: options.maxBashTimeoutMs,
323
+ todoTimeoutMs,
324
+ todoGracefulShutdownMs,
325
+ networkRecovery,
254
326
  now,
255
327
  goalSpecification,
328
+ onNetworkRecovery: recoveryProgress(
329
+ "todo_generation_start",
330
+ `Goal iteration ${current.iteration}: resuming TODO generation.`,
331
+ { iteration: current.iteration },
332
+ ),
256
333
  });
257
334
  generationResults.push(generation);
258
335
  state = generation.state;
@@ -278,6 +355,9 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
278
355
  thinkingLevel: options.thinkingLevel,
279
356
  maxBashTimeoutMs: options.maxBashTimeoutMs,
280
357
  maxAttemptsPerTask: options.maxAttemptsPerTask,
358
+ todoTimeoutMs,
359
+ todoGracefulShutdownMs,
360
+ networkRecovery,
281
361
  commit: options.commit,
282
362
  now,
283
363
  onProgress: (update) => {
@@ -324,7 +404,13 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
324
404
  thinkingLevel: options.thinkingLevel,
325
405
  now,
326
406
  goalSpecification,
407
+ networkRecovery,
327
408
  timeoutMs: remainingReviewTimeout(state, current.deadlineAt, now()),
409
+ onNetworkRecovery: recoveryProgress(
410
+ "review_start",
411
+ `Goal iteration ${current.iteration}: reviewing goal completion.`,
412
+ { iteration: current.iteration },
413
+ ),
328
414
  });
329
415
  reviewResults.push(review);
330
416
  state = review.state;
@@ -365,6 +451,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
365
451
  }
366
452
 
367
453
  publish("complete", `Goal loop ${state.status}: ${state.completion?.reason ?? "finished"}`);
454
+ progressClosed = true;
368
455
 
369
456
  return {
370
457
  state,
@@ -385,6 +472,7 @@ async function maybeRunGoalDiscovery(options: {
385
472
  options: RunGoalLoopOptions;
386
473
  now: () => Date;
387
474
  publish: (phase: GoalLoopProgressPhase, message: string, extra?: Partial<GoalLoopProgressUpdate>) => void;
475
+ networkRecovery: Readonly<NetworkRecoveryConfig>;
388
476
  }): Promise<GoalSpecification | undefined> {
389
477
  if (options.discoveryDecision.route !== "discovery") {
390
478
  return options.existingSpecification;
@@ -407,6 +495,7 @@ async function maybeRunGoalDiscovery(options: {
407
495
  model: options.options.model,
408
496
  modelName: options.options.modelName,
409
497
  thinkingLevel: options.options.thinkingLevel,
498
+ networkRecovery: options.networkRecovery,
410
499
  now: options.now,
411
500
  });
412
501
  await options.store.saveGoalSpecification(spec);
@@ -467,7 +556,11 @@ function accumulatedWorkerCost(state: GoalLoopState): number {
467
556
  }
468
557
 
469
558
  function accumulatedReviewerCost(state: GoalLoopState): number {
470
- return sumFinite(state.iterations.map((iteration) => iteration.reviewerResult?.reviewerCostTotal));
559
+ return sumFinite(
560
+ state.iterations.map(
561
+ (iteration) => iteration.reviewerResult?.reviewerCostTotal ?? iteration.reviewerRecovery?.reviewerCostTotal,
562
+ ),
563
+ );
471
564
  }
472
565
 
473
566
  function sumFinite(values: Array<number | undefined>): number {
@@ -477,6 +570,10 @@ function sumFinite(values: Array<number | undefined>): number {
477
570
  );
478
571
  }
479
572
 
573
+ function isTerminalNetworkRecoveryProgressEvent(type: NetworkRecoveryEventType): boolean {
574
+ return type === "failed" || type === "cancelled" || type === "outage_expired";
575
+ }
576
+
480
577
  function requiredGoal(goal: string | undefined): string {
481
578
  const trimmed = goal?.trim();
482
579
  if (!trimmed) {
@@ -3,18 +3,22 @@ import path from "node:path";
3
3
 
4
4
  import {
5
5
  cancelGoalLoop,
6
+ excludeNetworkOutageFromGoalDeadlines,
6
7
  type GoalIterationState,
7
8
  type GoalLoopState,
8
9
  type GoalReviewerDecision,
9
10
  type GoalReviewerResultState,
11
+ recordReviewerRecoveryEvidence,
10
12
  recordReviewerResult,
11
13
  } from "./goal_loop.ts";
12
14
  import { GoalStateStore } from "./goal_state.ts";
13
15
  import { runGuardedSessionPrompt } from "./session_guard.ts";
14
16
  import { goalSpecificationToMarkdown, type GoalSpecification } from "./goal_spec.ts";
17
+ import { classifyNetworkFailure } from "./network_failure.ts";
18
+ import { recoverNetworkOperation, type NetworkRecoveryEvent } from "./network_recovery.ts";
19
+ import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
15
20
  import {
16
21
  createIsolatedWorkerSession,
17
- DEFAULT_WORKER_TOOLS,
18
22
  workerUsageCostFromEvent,
19
23
  workerUsageCostFromStats,
20
24
  workerUsageCostKeyFromEvent,
@@ -23,6 +27,8 @@ import {
23
27
 
24
28
  export const GOAL_REVIEW_PAYLOAD_FILE = "REVIEW_TASK.md";
25
29
  export const GOAL_REVIEW_RAW_FILE = "REVIEW_RESULT_RAW.txt";
30
+ /** Reviewer retries are constrained to inspection and read-only verification. */
31
+ export const GOAL_REVIEWER_TOOLS = ["read", "bash", "grep", "find", "ls"] as const;
26
32
 
27
33
  export interface GoalReviewOptions {
28
34
  state: GoalLoopState;
@@ -36,7 +42,9 @@ export interface GoalReviewOptions {
36
42
  now?: () => Date;
37
43
  sessionFactory?: WorkerSessionFactory;
38
44
  goalSpecification?: GoalSpecification;
45
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
39
46
  timeoutMs?: number;
47
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
40
48
  }
41
49
 
42
50
  export interface GoalReviewResult {
@@ -59,6 +67,7 @@ export interface GoalReviewerRunnerOptions {
59
67
  modelName?: string;
60
68
  thinkingLevel?: string;
61
69
  sessionFactory?: WorkerSessionFactory;
70
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
62
71
  }
63
72
 
64
73
  export interface GoalReviewerSessionResult {
@@ -69,6 +78,8 @@ export interface GoalReviewerSessionResult {
69
78
  timedOut?: boolean;
70
79
  aborted?: boolean;
71
80
  error?: string;
81
+ /** Untouched provider/transport failure retained for coordinator recovery classification. */
82
+ failure?: unknown;
72
83
  }
73
84
 
74
85
  export type GoalReviewerRunner = (options: GoalReviewerRunnerOptions) => Promise<GoalReviewerSessionResult>;
@@ -91,7 +102,7 @@ export class GoalReviewError extends Error {
91
102
  export async function runGoalReviewSession(options: GoalReviewOptions): Promise<GoalReviewResult> {
92
103
  const now = options.now ?? (() => new Date());
93
104
  let state = options.state;
94
- const previousTraceLength = state.trace.length;
105
+ let previousTraceLength = state.trace.length;
95
106
  const store =
96
107
  options.store ?? new GoalStateStore({ cwd: options.cwd, goalRunId: state.goalRunId, goalRunDir: state.goalRunDir });
97
108
 
@@ -116,19 +127,78 @@ export async function runGoalReviewSession(options: GoalReviewOptions): Promise<
116
127
  throw new GoalReviewError("Goal review was aborted before starting.", { state });
117
128
  }
118
129
 
119
- let sessionResult: GoalReviewerSessionResult;
130
+ const interruptedResults: GoalReviewerSessionResult[] = [];
131
+ const interruptedEvidencePaths: string[] = [];
132
+ let excludedOutageMs = 0;
133
+ const captureOutage = (event: NetworkRecoveryEvent) => {
134
+ if (event.type === "cleanup") {
135
+ excludedOutageMs += event.state.elapsedMs;
136
+ }
137
+ options.onNetworkRecovery?.(event);
138
+ };
139
+
140
+ let sessionResult: GoalReviewerSessionResult | undefined;
141
+ let sessionFailure: unknown;
120
142
  try {
121
- sessionResult = await (options.reviewerRunner ?? runGoalReviewerSession)({
122
- prompt: payload,
123
- cwd: path.resolve(options.cwd ?? process.cwd()),
143
+ sessionResult = await runReviewerWithNetworkRecovery({
144
+ runner: options.reviewerRunner ?? runGoalReviewerSession,
145
+ runnerOptions: {
146
+ prompt: payload,
147
+ cwd: path.resolve(options.cwd ?? process.cwd()),
148
+ abortSignal: options.abortSignal,
149
+ timeoutMs: options.timeoutMs ?? state.limits.reviewerTimeoutMs,
150
+ model: options.model,
151
+ modelName: options.modelName,
152
+ thinkingLevel: options.thinkingLevel,
153
+ sessionFactory: options.sessionFactory,
154
+ networkRecovery: options.networkRecovery,
155
+ },
156
+ networkRecovery: options.networkRecovery,
124
157
  abortSignal: options.abortSignal,
125
- timeoutMs: options.timeoutMs ?? state.limits.reviewerTimeoutMs,
126
- model: options.model,
127
- modelName: options.modelName,
128
- thinkingLevel: options.thinkingLevel,
129
- sessionFactory: options.sessionFactory,
158
+ now,
159
+ interruptedResults,
160
+ interruptedEvidencePaths,
161
+ iterationDir,
162
+ onInterruption: async () => {
163
+ state = recordReviewerRecoveryEvidence(
164
+ state,
165
+ iteration.iteration,
166
+ {
167
+ interruptions: interruptedResults.length,
168
+ evidencePaths: interruptedEvidencePaths,
169
+ reviewerCostTotal: reviewerCost(interruptedResults),
170
+ },
171
+ { now: now() },
172
+ );
173
+ await persistStateChange(store, previousTraceLength, state);
174
+ previousTraceLength = state.trace.length;
175
+ },
176
+ onRecoveryEvent: captureOutage,
130
177
  });
131
178
  } catch (error) {
179
+ sessionFailure = error;
180
+ }
181
+
182
+ if (excludedOutageMs > 0) {
183
+ state = excludeNetworkOutageFromGoalDeadlines(state, excludedOutageMs, "reviewer", { now: now() });
184
+ }
185
+ if (state.trace.length !== previousTraceLength) {
186
+ await persistStateChange(store, previousTraceLength, state);
187
+ previousTraceLength = state.trace.length;
188
+ }
189
+
190
+ if (sessionFailure !== undefined && options.abortSignal?.aborted) {
191
+ const reason = `Goal review was aborted during network recovery: ${errorMessage(
192
+ options.abortSignal.reason ?? sessionFailure,
193
+ )}`;
194
+ state = cancelGoalLoop(state, reason, { now: now() });
195
+ await persistStateChange(store, previousTraceLength, state);
196
+ await store.writeIterationSnapshot(currentIteration(state, iteration.iteration));
197
+ throw new GoalReviewError(reason, { cause: sessionFailure, state });
198
+ }
199
+
200
+ if (sessionFailure !== undefined) {
201
+ const interruptedResult = combineReviewerSessionCosts(interruptedResults);
132
202
  const failure = await recordReviewFailure({
133
203
  state,
134
204
  iteration,
@@ -136,17 +206,23 @@ export async function runGoalReviewSession(options: GoalReviewOptions): Promise<
136
206
  previousTraceLength,
137
207
  payloadPath,
138
208
  rawReviewPath,
139
- message: `Reviewer session failed: ${errorMessage(error)}`,
140
- error,
209
+ message: `Reviewer session failed: ${errorMessage(sessionFailure)}`,
210
+ error: sessionFailure,
211
+ sessionResult: interruptedResult,
141
212
  now,
142
213
  });
143
214
  throw new GoalReviewError(failure.reviewerResult.summary, {
144
- cause: error,
215
+ cause: sessionFailure,
145
216
  state: failure.state,
146
217
  reviewerResult: failure.reviewerResult,
147
218
  });
148
219
  }
149
220
 
221
+ sessionResult = combineReviewerSessionCosts(interruptedResults, sessionResult);
222
+ if (!sessionResult) {
223
+ throw new GoalReviewError("Reviewer session ended without a result.", { state });
224
+ }
225
+
150
226
  const rawReviewerOutput = sessionResult.assistantText;
151
227
  await writeFile(rawReviewPath, rawReviewerOutput, "utf8");
152
228
 
@@ -367,7 +443,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
367
443
  }
368
444
  const factoryResult = await sessionFactory({
369
445
  cwd: options.cwd,
370
- tools: DEFAULT_WORKER_TOOLS,
446
+ tools: GOAL_REVIEWER_TOOLS,
371
447
  model: options.model,
372
448
  modelName: options.modelName,
373
449
  thinkingLevel: options.thinkingLevel,
@@ -414,6 +490,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
414
490
  timedOut: promptResult.timedOut,
415
491
  aborted: promptResult.aborted,
416
492
  error: promptResult.error,
493
+ failure: promptResult.failure,
417
494
  };
418
495
  } catch (exc) {
419
496
  return {
@@ -422,6 +499,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
422
499
  reviewerSessionFile: session?.sessionFile,
423
500
  reviewerCostTotal,
424
501
  error: errorMessage(exc),
502
+ failure: exc,
425
503
  };
426
504
  } finally {
427
505
  if (session) {
@@ -434,6 +512,119 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
434
512
  }
435
513
  }
436
514
 
515
+ async function runReviewerWithNetworkRecovery(options: {
516
+ runner: GoalReviewerRunner;
517
+ runnerOptions: GoalReviewerRunnerOptions;
518
+ networkRecovery: Readonly<NetworkRecoveryConfig> | undefined;
519
+ abortSignal: AbortSignal | undefined;
520
+ now: () => Date;
521
+ interruptedResults: GoalReviewerSessionResult[];
522
+ interruptedEvidencePaths: string[];
523
+ iterationDir: string;
524
+ onInterruption: () => Promise<void>;
525
+ onRecoveryEvent: (event: NetworkRecoveryEvent) => void;
526
+ }): Promise<GoalReviewerSessionResult> {
527
+ const run = async (recoverySignal?: AbortSignal): Promise<GoalReviewerSessionResult> => {
528
+ const result = await options.runner({
529
+ ...options.runnerOptions,
530
+ abortSignal: combineAbortSignals(options.abortSignal, recoverySignal),
531
+ });
532
+ const networkFailure = recoverableReviewerFailure(result);
533
+ if (networkFailure === undefined) {
534
+ return result;
535
+ }
536
+
537
+ options.interruptedResults.push(result);
538
+ options.interruptedEvidencePaths.push(
539
+ await writeInterruptedReviewEvidence(options.iterationDir, options.interruptedResults.length, result),
540
+ );
541
+ await options.onInterruption();
542
+ throw new ReviewerNetworkFailure(networkFailure, result);
543
+ };
544
+
545
+ try {
546
+ return await run();
547
+ } catch (initialFailure) {
548
+ const classification = classifyNetworkFailure(initialFailure);
549
+ if (!options.networkRecovery?.enabled || !classification.recoverable) {
550
+ throw initialFailure;
551
+ }
552
+ const recovered = await recoverNetworkOperation({
553
+ initialFailure,
554
+ config: options.networkRecovery,
555
+ signal: options.abortSignal,
556
+ now: () => options.now().getTime(),
557
+ onEvent: options.onRecoveryEvent,
558
+ retry: ({ signal }) => run(signal),
559
+ });
560
+ return recovered.value;
561
+ }
562
+ }
563
+
564
+ class ReviewerNetworkFailure extends Error {
565
+ readonly sessionResult: GoalReviewerSessionResult;
566
+
567
+ constructor(failure: unknown, sessionResult: GoalReviewerSessionResult) {
568
+ super(sessionResult.error ?? errorMessage(failure), { cause: failure });
569
+ this.name = "ReviewerNetworkFailure";
570
+ this.sessionResult = sessionResult;
571
+ }
572
+ }
573
+
574
+ function recoverableReviewerFailure(result: GoalReviewerSessionResult): unknown | undefined {
575
+ if (result.aborted || result.timedOut || !result.error) {
576
+ return undefined;
577
+ }
578
+ const failure = result.failure ?? new Error(result.error);
579
+ return classifyNetworkFailure(failure).recoverable ? failure : undefined;
580
+ }
581
+
582
+ async function writeInterruptedReviewEvidence(
583
+ iterationDir: string,
584
+ interruption: number,
585
+ result: GoalReviewerSessionResult,
586
+ ): Promise<string> {
587
+ const outputPath = path.join(
588
+ iterationDir,
589
+ `REVIEW_RESULT_NETWORK_INTERRUPTED_${String(interruption).padStart(2, "0")}.txt`,
590
+ );
591
+ const evidence = result.assistantText.trim()
592
+ ? result.assistantText
593
+ : `[No assistant text was captured.]\n\n${result.error ?? "Transient reviewer network failure."}\n`;
594
+ await writeFile(outputPath, evidence, "utf8");
595
+ return outputPath;
596
+ }
597
+
598
+ function reviewerCost(results: readonly GoalReviewerSessionResult[]): number {
599
+ return results.reduce(
600
+ (total, result) =>
601
+ total +
602
+ (typeof result.reviewerCostTotal === "number" && Number.isFinite(result.reviewerCostTotal)
603
+ ? result.reviewerCostTotal
604
+ : 0),
605
+ 0,
606
+ );
607
+ }
608
+
609
+ function combineReviewerSessionCosts(
610
+ interrupted: readonly GoalReviewerSessionResult[],
611
+ final?: GoalReviewerSessionResult,
612
+ ): GoalReviewerSessionResult | undefined {
613
+ if (!final && interrupted.length === 0) {
614
+ return undefined;
615
+ }
616
+ const base = final ?? interrupted.at(-1)!;
617
+ const reviewerCostTotal = reviewerCost([...interrupted, ...(final ? [final] : [])]);
618
+ return { ...base, reviewerCostTotal };
619
+ }
620
+
621
+ function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined {
622
+ const available = signals.filter((signal): signal is AbortSignal => Boolean(signal));
623
+ if (available.length === 0) return undefined;
624
+ if (available.length === 1) return available[0];
625
+ return AbortSignal.any(available);
626
+ }
627
+
437
628
  function currentReviewableIteration(state: GoalLoopState): GoalIterationState {
438
629
  const iteration = currentIteration(state, state.currentIteration);
439
630
  if (iteration.status !== "todo_executed" && iteration.status !== "failed") {
@@ -15,6 +15,7 @@ import {
15
15
  recordWorkerResult,
16
16
  } from "./goal_loop.ts";
17
17
  import { GoalStateStore } from "./goal_state.ts";
18
+ import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
18
19
  import { validateTodoMarkdown } from "./todo_generator.ts";
19
20
 
20
21
  export const GOAL_TODO_EXECUTION_PROGRESS_FILE = "WORKER_PROGRESS.jsonl";
@@ -32,6 +33,9 @@ export interface GoalTodoExecutionOptions {
32
33
  thinkingLevel?: string;
33
34
  maxBashTimeoutMs?: number;
34
35
  maxAttemptsPerTask?: number;
36
+ todoTimeoutMs?: number;
37
+ todoGracefulShutdownMs?: number;
38
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
35
39
  commit?: boolean;
36
40
  now?: () => Date;
37
41
  onProgress?: (update: CoordinatorProgressUpdate) => void;
@@ -152,6 +156,9 @@ export async function runGoalTodoExecutionLongTask(
152
156
  taskTimeoutMs: childTimeoutMs,
153
157
  maxBashTimeoutMs: options.maxBashTimeoutMs,
154
158
  maxAttemptsPerTask: options.maxAttemptsPerTask,
159
+ todoTimeoutMs: options.todoTimeoutMs,
160
+ todoGracefulShutdownMs: options.todoGracefulShutdownMs,
161
+ networkRecovery: options.networkRecovery,
155
162
  onProgress: (update) => {
156
163
  progressEvents.push(update);
157
164
  options.onProgress?.(update);