pi-long-task 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,6 +13,7 @@ import {
13
13
  } from "./goal_spec.ts";
14
14
  import type { GoalLoopState } from "./goal_loop.ts";
15
15
  import type { GoalStateStore } from "./goal_state.ts";
16
+ import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
16
17
 
17
18
  export type GoalDiscoveryEntrypoint = "pi_goal_task" | "pi_long_task";
18
19
  export type GoalDiscoveryRoute = "discovery" | "direct";
@@ -41,6 +42,7 @@ export interface GoalDiscoveryRunnerOptions {
41
42
  model?: unknown;
42
43
  modelName?: string;
43
44
  thinkingLevel?: string;
45
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
44
46
  now: () => Date;
45
47
  }
46
48
 
package/src/goal_loop.ts CHANGED
@@ -92,6 +92,13 @@ export interface GoalWorkerResultState {
92
92
 
93
93
  export type GoalReviewerDecision = "complete" | "incomplete" | "blocked" | "failed";
94
94
 
95
+ export interface GoalReviewerRecoveryState {
96
+ interruptions: number;
97
+ evidencePaths: string[];
98
+ reviewerCostTotal: number;
99
+ updatedAt: string;
100
+ }
101
+
95
102
  export interface GoalReviewerResultState {
96
103
  decision: GoalReviewerDecision;
97
104
  complete: boolean;
@@ -130,6 +137,7 @@ export interface GoalIterationState {
130
137
  deadlineAt?: string;
131
138
  generatedTodo?: GeneratedTodoState;
132
139
  workerResult?: GoalWorkerResultState;
140
+ reviewerRecovery?: GoalReviewerRecoveryState;
133
141
  reviewerResult?: GoalReviewerResultState;
134
142
  completion?: GoalCompletionState;
135
143
  }
@@ -402,6 +410,70 @@ export function recordReviewerResult(
402
410
  });
403
411
  }
404
412
 
413
+ export function excludeNetworkOutageFromGoalDeadlines(
414
+ state: GoalLoopState,
415
+ outageMs: number,
416
+ operation: "planner" | "reviewer",
417
+ options: { now?: Date } = {},
418
+ ): GoalLoopState {
419
+ if (!Number.isFinite(outageMs) || outageMs <= 0 || isTerminalGoalLoopStatus(state.status)) {
420
+ return state;
421
+ }
422
+
423
+ const excludedMs = Math.ceil(outageMs);
424
+ const timestamp = (options.now ?? new Date()).toISOString();
425
+ const iterations = state.iterations.map((iteration) =>
426
+ iteration.iteration === state.currentIteration && iteration.deadlineAt
427
+ ? { ...iteration, deadlineAt: shiftTimestamp(iteration.deadlineAt, excludedMs), updatedAt: timestamp }
428
+ : iteration,
429
+ );
430
+ return withTrace(
431
+ {
432
+ ...state,
433
+ deadlineAt: state.deadlineAt ? shiftTimestamp(state.deadlineAt, excludedMs) : undefined,
434
+ iterations,
435
+ updatedAt: timestamp,
436
+ },
437
+ {
438
+ timestamp,
439
+ phase: state.phase,
440
+ event: "network_wait_excluded",
441
+ message: `Excluded ${excludedMs}ms of ${operation} network recovery from goal-loop deadlines.`,
442
+ iteration: state.currentIteration || undefined,
443
+ details: { operation, outageMs: excludedMs },
444
+ },
445
+ );
446
+ }
447
+
448
+ export function recordReviewerRecoveryEvidence(
449
+ state: GoalLoopState,
450
+ iterationNumber: number,
451
+ recovery: Omit<GoalReviewerRecoveryState, "updatedAt">,
452
+ options: { now?: Date } = {},
453
+ ): GoalLoopState {
454
+ const timestamp = (options.now ?? new Date()).toISOString();
455
+ return updateIteration(state, iterationNumber, ["todo_executed", "failed"], timestamp, (item) => ({
456
+ item: {
457
+ ...item,
458
+ updatedAt: timestamp,
459
+ reviewerRecovery: { ...recovery, evidencePaths: [...recovery.evidencePaths], updatedAt: timestamp },
460
+ },
461
+ phase: state.phase,
462
+ trace: {
463
+ timestamp,
464
+ phase: state.phase,
465
+ event: "reviewer_network_interrupted",
466
+ message: `Preserved ${recovery.interruptions} interrupted reviewer call(s) during network recovery.`,
467
+ iteration: iterationNumber,
468
+ details: {
469
+ interruptions: recovery.interruptions,
470
+ evidencePaths: recovery.evidencePaths,
471
+ reviewerCostTotal: recovery.reviewerCostTotal,
472
+ },
473
+ },
474
+ }));
475
+ }
476
+
405
477
  export function failGoalLoop(
406
478
  state: GoalLoopState,
407
479
  reason: string,
@@ -597,6 +669,10 @@ function withTrace(state: GoalLoopState, event: GoalLoopTraceEvent): GoalLoopSta
597
669
  return { ...state, trace: [...state.trace, event] };
598
670
  }
599
671
 
672
+ function shiftTimestamp(timestamp: string, deltaMs: number): string {
673
+ return new Date(Date.parse(timestamp) + deltaMs).toISOString();
674
+ }
675
+
600
676
  function positiveInteger(value: number | undefined, fallback: number): number {
601
677
  return optionalPositiveInteger(value) ?? fallback;
602
678
  }
@@ -20,6 +20,16 @@ import {
20
20
  } from "./goal_loop.ts";
21
21
  import { GoalStateStore } from "./goal_state.ts";
22
22
  import type { GoalSpecification } from "./goal_spec.ts";
23
+ import {
24
+ formatNetworkRecoveryStatus,
25
+ type NetworkRecoveryEvent,
26
+ type NetworkRecoveryEventType,
27
+ } from "./network_recovery.ts";
28
+ import {
29
+ resolveNetworkRecoveryConfig,
30
+ type NetworkRecoveryConfig,
31
+ type NetworkRecoveryConfigInput,
32
+ } from "./network_recovery_config.ts";
23
33
  import { runGoalReviewSession, type GoalReviewResult, type GoalReviewerRunner } from "./goal_review.ts";
24
34
  import {
25
35
  runGoalTodoExecutionLongTask,
@@ -43,6 +53,7 @@ export type GoalLoopProgressPhase =
43
53
  | "todo_executed"
44
54
  | "review_start"
45
55
  | "reviewed"
56
+ | "network_wait"
46
57
  | "complete";
47
58
 
48
59
  export interface GoalLoopProgressUpdate {
@@ -70,6 +81,12 @@ export interface GoalLoopProgressUpdate {
70
81
  reviewerCostTotal: number;
71
82
  totalCost: number;
72
83
  childProgress?: CoordinatorProgressUpdate;
84
+ networkRecoveryEvent?: NetworkRecoveryEventType;
85
+ networkRetryCount?: number;
86
+ networkOutageElapsedMs?: number;
87
+ networkNextRetryAtMs?: number;
88
+ networkNextRetryInMs?: number;
89
+ networkFailureReason?: string;
73
90
  }
74
91
 
75
92
  export type GoalLoopProgressHandler = (update: GoalLoopProgressUpdate) => void;
@@ -92,10 +109,12 @@ export interface RunGoalLoopOptions extends GoalLoopLimitInput {
92
109
  thinkingLevel?: string;
93
110
  maxBashTimeoutMs?: number;
94
111
  maxAttemptsPerTask?: number;
112
+ networkRecovery?: NetworkRecoveryConfigInput;
95
113
  commit?: boolean;
96
114
  now?: () => Date;
97
115
  onWorkerProgress?: (update: CoordinatorProgressUpdate) => void;
98
116
  onProgress?: GoalLoopProgressHandler;
117
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
99
118
  }
100
119
 
101
120
  export interface GoalLoopRunResult {
@@ -120,6 +139,7 @@ export class GoalLoopOrchestratorError extends Error {
120
139
 
121
140
  export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoopRunResult> {
122
141
  const now = options.now ?? (() => new Date());
142
+ const networkRecovery = resolveNetworkRecoveryConfig(options.networkRecovery);
123
143
  let state =
124
144
  options.initialState ??
125
145
  createGoalLoopState({
@@ -149,7 +169,9 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
149
169
  await store.saveState(state);
150
170
  await store.initializeResultIfMissing(state);
151
171
  await store.appendNewTraceEvents(await store.durableTraceLength(), state);
172
+ let progressClosed = false;
152
173
  const publish = (phase: GoalLoopProgressPhase, message: string, extra: Partial<GoalLoopProgressUpdate> = {}) => {
174
+ if (progressClosed) return;
153
175
  options.onProgress?.({
154
176
  message,
155
177
  phase,
@@ -173,6 +195,39 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
173
195
  ...extra,
174
196
  });
175
197
  };
198
+ const recoveryProgress = (
199
+ restorePhase: GoalLoopProgressPhase,
200
+ restoreMessage: string,
201
+ extra: Partial<GoalLoopProgressUpdate> = {},
202
+ ) => {
203
+ let cleaned = false;
204
+ return (event: NetworkRecoveryEvent) => {
205
+ options.onNetworkRecovery?.(event);
206
+ if (cleaned || progressClosed) return;
207
+ if (event.type === "cleanup") {
208
+ cleaned = true;
209
+ return;
210
+ }
211
+ if (event.type === "recovered") {
212
+ publish(restorePhase, restoreMessage, extra);
213
+ return;
214
+ }
215
+ if (isTerminalNetworkRecoveryProgressEvent(event.type)) return;
216
+
217
+ const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
218
+ const message = formatNetworkRecoveryStatus(event);
219
+ publish("network_wait", message, {
220
+ ...extra,
221
+ networkRecoveryEvent: event.type,
222
+ networkRetryCount: event.state.retryCount,
223
+ networkOutageElapsedMs: event.state.elapsedMs,
224
+ networkNextRetryAtMs: event.state.nextRetryAtMs,
225
+ networkNextRetryInMs:
226
+ event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
227
+ networkFailureReason: event.state.lastFailure.reason,
228
+ });
229
+ };
230
+ };
176
231
 
177
232
  publish("goal_start", `Starting goal loop: ${state.goal}`);
178
233
 
@@ -186,6 +241,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
186
241
  options,
187
242
  now,
188
243
  publish,
244
+ networkRecovery,
189
245
  });
190
246
  }
191
247
 
@@ -226,8 +282,14 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
226
282
  modelName: options.modelName,
227
283
  thinkingLevel: options.thinkingLevel,
228
284
  maxBashTimeoutMs: options.maxBashTimeoutMs,
285
+ networkRecovery,
229
286
  now,
230
287
  goalSpecification,
288
+ onNetworkRecovery: recoveryProgress(
289
+ "todo_generation_start",
290
+ `Goal iteration ${nextIteration}: generating TODO markdown.`,
291
+ { iteration: nextIteration },
292
+ ),
231
293
  });
232
294
  generationResults.push(generation);
233
295
  state = generation.state;
@@ -251,8 +313,14 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
251
313
  modelName: options.modelName,
252
314
  thinkingLevel: options.thinkingLevel,
253
315
  maxBashTimeoutMs: options.maxBashTimeoutMs,
316
+ networkRecovery,
254
317
  now,
255
318
  goalSpecification,
319
+ onNetworkRecovery: recoveryProgress(
320
+ "todo_generation_start",
321
+ `Goal iteration ${current.iteration}: resuming TODO generation.`,
322
+ { iteration: current.iteration },
323
+ ),
256
324
  });
257
325
  generationResults.push(generation);
258
326
  state = generation.state;
@@ -278,6 +346,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
278
346
  thinkingLevel: options.thinkingLevel,
279
347
  maxBashTimeoutMs: options.maxBashTimeoutMs,
280
348
  maxAttemptsPerTask: options.maxAttemptsPerTask,
349
+ networkRecovery,
281
350
  commit: options.commit,
282
351
  now,
283
352
  onProgress: (update) => {
@@ -324,7 +393,13 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
324
393
  thinkingLevel: options.thinkingLevel,
325
394
  now,
326
395
  goalSpecification,
396
+ networkRecovery,
327
397
  timeoutMs: remainingReviewTimeout(state, current.deadlineAt, now()),
398
+ onNetworkRecovery: recoveryProgress(
399
+ "review_start",
400
+ `Goal iteration ${current.iteration}: reviewing goal completion.`,
401
+ { iteration: current.iteration },
402
+ ),
328
403
  });
329
404
  reviewResults.push(review);
330
405
  state = review.state;
@@ -365,6 +440,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
365
440
  }
366
441
 
367
442
  publish("complete", `Goal loop ${state.status}: ${state.completion?.reason ?? "finished"}`);
443
+ progressClosed = true;
368
444
 
369
445
  return {
370
446
  state,
@@ -385,6 +461,7 @@ async function maybeRunGoalDiscovery(options: {
385
461
  options: RunGoalLoopOptions;
386
462
  now: () => Date;
387
463
  publish: (phase: GoalLoopProgressPhase, message: string, extra?: Partial<GoalLoopProgressUpdate>) => void;
464
+ networkRecovery: Readonly<NetworkRecoveryConfig>;
388
465
  }): Promise<GoalSpecification | undefined> {
389
466
  if (options.discoveryDecision.route !== "discovery") {
390
467
  return options.existingSpecification;
@@ -407,6 +484,7 @@ async function maybeRunGoalDiscovery(options: {
407
484
  model: options.options.model,
408
485
  modelName: options.options.modelName,
409
486
  thinkingLevel: options.options.thinkingLevel,
487
+ networkRecovery: options.networkRecovery,
410
488
  now: options.now,
411
489
  });
412
490
  await options.store.saveGoalSpecification(spec);
@@ -467,7 +545,11 @@ function accumulatedWorkerCost(state: GoalLoopState): number {
467
545
  }
468
546
 
469
547
  function accumulatedReviewerCost(state: GoalLoopState): number {
470
- return sumFinite(state.iterations.map((iteration) => iteration.reviewerResult?.reviewerCostTotal));
548
+ return sumFinite(
549
+ state.iterations.map(
550
+ (iteration) => iteration.reviewerResult?.reviewerCostTotal ?? iteration.reviewerRecovery?.reviewerCostTotal,
551
+ ),
552
+ );
471
553
  }
472
554
 
473
555
  function sumFinite(values: Array<number | undefined>): number {
@@ -477,6 +559,10 @@ function sumFinite(values: Array<number | undefined>): number {
477
559
  );
478
560
  }
479
561
 
562
+ function isTerminalNetworkRecoveryProgressEvent(type: NetworkRecoveryEventType): boolean {
563
+ return type === "failed" || type === "cancelled" || type === "outage_expired";
564
+ }
565
+
480
566
  function requiredGoal(goal: string | undefined): string {
481
567
  const trimmed = goal?.trim();
482
568
  if (!trimmed) {
@@ -3,18 +3,22 @@ import path from "node:path";
3
3
 
4
4
  import {
5
5
  cancelGoalLoop,
6
+ excludeNetworkOutageFromGoalDeadlines,
6
7
  type GoalIterationState,
7
8
  type GoalLoopState,
8
9
  type GoalReviewerDecision,
9
10
  type GoalReviewerResultState,
11
+ recordReviewerRecoveryEvidence,
10
12
  recordReviewerResult,
11
13
  } from "./goal_loop.ts";
12
14
  import { GoalStateStore } from "./goal_state.ts";
13
15
  import { runGuardedSessionPrompt } from "./session_guard.ts";
14
16
  import { goalSpecificationToMarkdown, type GoalSpecification } from "./goal_spec.ts";
17
+ import { classifyNetworkFailure } from "./network_failure.ts";
18
+ import { recoverNetworkOperation, type NetworkRecoveryEvent } from "./network_recovery.ts";
19
+ import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
15
20
  import {
16
21
  createIsolatedWorkerSession,
17
- DEFAULT_WORKER_TOOLS,
18
22
  workerUsageCostFromEvent,
19
23
  workerUsageCostFromStats,
20
24
  workerUsageCostKeyFromEvent,
@@ -23,6 +27,8 @@ import {
23
27
 
24
28
  export const GOAL_REVIEW_PAYLOAD_FILE = "REVIEW_TASK.md";
25
29
  export const GOAL_REVIEW_RAW_FILE = "REVIEW_RESULT_RAW.txt";
30
+ /** Reviewer retries are constrained to inspection and read-only verification. */
31
+ export const GOAL_REVIEWER_TOOLS = ["read", "bash", "grep", "find", "ls"] as const;
26
32
 
27
33
  export interface GoalReviewOptions {
28
34
  state: GoalLoopState;
@@ -36,7 +42,9 @@ export interface GoalReviewOptions {
36
42
  now?: () => Date;
37
43
  sessionFactory?: WorkerSessionFactory;
38
44
  goalSpecification?: GoalSpecification;
45
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
39
46
  timeoutMs?: number;
47
+ onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
40
48
  }
41
49
 
42
50
  export interface GoalReviewResult {
@@ -59,6 +67,7 @@ export interface GoalReviewerRunnerOptions {
59
67
  modelName?: string;
60
68
  thinkingLevel?: string;
61
69
  sessionFactory?: WorkerSessionFactory;
70
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
62
71
  }
63
72
 
64
73
  export interface GoalReviewerSessionResult {
@@ -69,6 +78,8 @@ export interface GoalReviewerSessionResult {
69
78
  timedOut?: boolean;
70
79
  aborted?: boolean;
71
80
  error?: string;
81
+ /** Untouched provider/transport failure retained for coordinator recovery classification. */
82
+ failure?: unknown;
72
83
  }
73
84
 
74
85
  export type GoalReviewerRunner = (options: GoalReviewerRunnerOptions) => Promise<GoalReviewerSessionResult>;
@@ -91,7 +102,7 @@ export class GoalReviewError extends Error {
91
102
  export async function runGoalReviewSession(options: GoalReviewOptions): Promise<GoalReviewResult> {
92
103
  const now = options.now ?? (() => new Date());
93
104
  let state = options.state;
94
- const previousTraceLength = state.trace.length;
105
+ let previousTraceLength = state.trace.length;
95
106
  const store =
96
107
  options.store ?? new GoalStateStore({ cwd: options.cwd, goalRunId: state.goalRunId, goalRunDir: state.goalRunDir });
97
108
 
@@ -116,19 +127,78 @@ export async function runGoalReviewSession(options: GoalReviewOptions): Promise<
116
127
  throw new GoalReviewError("Goal review was aborted before starting.", { state });
117
128
  }
118
129
 
119
- let sessionResult: GoalReviewerSessionResult;
130
+ const interruptedResults: GoalReviewerSessionResult[] = [];
131
+ const interruptedEvidencePaths: string[] = [];
132
+ let excludedOutageMs = 0;
133
+ const captureOutage = (event: NetworkRecoveryEvent) => {
134
+ if (event.type === "cleanup") {
135
+ excludedOutageMs += event.state.elapsedMs;
136
+ }
137
+ options.onNetworkRecovery?.(event);
138
+ };
139
+
140
+ let sessionResult: GoalReviewerSessionResult | undefined;
141
+ let sessionFailure: unknown;
120
142
  try {
121
- sessionResult = await (options.reviewerRunner ?? runGoalReviewerSession)({
122
- prompt: payload,
123
- cwd: path.resolve(options.cwd ?? process.cwd()),
143
+ sessionResult = await runReviewerWithNetworkRecovery({
144
+ runner: options.reviewerRunner ?? runGoalReviewerSession,
145
+ runnerOptions: {
146
+ prompt: payload,
147
+ cwd: path.resolve(options.cwd ?? process.cwd()),
148
+ abortSignal: options.abortSignal,
149
+ timeoutMs: options.timeoutMs ?? state.limits.reviewerTimeoutMs,
150
+ model: options.model,
151
+ modelName: options.modelName,
152
+ thinkingLevel: options.thinkingLevel,
153
+ sessionFactory: options.sessionFactory,
154
+ networkRecovery: options.networkRecovery,
155
+ },
156
+ networkRecovery: options.networkRecovery,
124
157
  abortSignal: options.abortSignal,
125
- timeoutMs: options.timeoutMs ?? state.limits.reviewerTimeoutMs,
126
- model: options.model,
127
- modelName: options.modelName,
128
- thinkingLevel: options.thinkingLevel,
129
- sessionFactory: options.sessionFactory,
158
+ now,
159
+ interruptedResults,
160
+ interruptedEvidencePaths,
161
+ iterationDir,
162
+ onInterruption: async () => {
163
+ state = recordReviewerRecoveryEvidence(
164
+ state,
165
+ iteration.iteration,
166
+ {
167
+ interruptions: interruptedResults.length,
168
+ evidencePaths: interruptedEvidencePaths,
169
+ reviewerCostTotal: reviewerCost(interruptedResults),
170
+ },
171
+ { now: now() },
172
+ );
173
+ await persistStateChange(store, previousTraceLength, state);
174
+ previousTraceLength = state.trace.length;
175
+ },
176
+ onRecoveryEvent: captureOutage,
130
177
  });
131
178
  } catch (error) {
179
+ sessionFailure = error;
180
+ }
181
+
182
+ if (excludedOutageMs > 0) {
183
+ state = excludeNetworkOutageFromGoalDeadlines(state, excludedOutageMs, "reviewer", { now: now() });
184
+ }
185
+ if (state.trace.length !== previousTraceLength) {
186
+ await persistStateChange(store, previousTraceLength, state);
187
+ previousTraceLength = state.trace.length;
188
+ }
189
+
190
+ if (sessionFailure !== undefined && options.abortSignal?.aborted) {
191
+ const reason = `Goal review was aborted during network recovery: ${errorMessage(
192
+ options.abortSignal.reason ?? sessionFailure,
193
+ )}`;
194
+ state = cancelGoalLoop(state, reason, { now: now() });
195
+ await persistStateChange(store, previousTraceLength, state);
196
+ await store.writeIterationSnapshot(currentIteration(state, iteration.iteration));
197
+ throw new GoalReviewError(reason, { cause: sessionFailure, state });
198
+ }
199
+
200
+ if (sessionFailure !== undefined) {
201
+ const interruptedResult = combineReviewerSessionCosts(interruptedResults);
132
202
  const failure = await recordReviewFailure({
133
203
  state,
134
204
  iteration,
@@ -136,17 +206,23 @@ export async function runGoalReviewSession(options: GoalReviewOptions): Promise<
136
206
  previousTraceLength,
137
207
  payloadPath,
138
208
  rawReviewPath,
139
- message: `Reviewer session failed: ${errorMessage(error)}`,
140
- error,
209
+ message: `Reviewer session failed: ${errorMessage(sessionFailure)}`,
210
+ error: sessionFailure,
211
+ sessionResult: interruptedResult,
141
212
  now,
142
213
  });
143
214
  throw new GoalReviewError(failure.reviewerResult.summary, {
144
- cause: error,
215
+ cause: sessionFailure,
145
216
  state: failure.state,
146
217
  reviewerResult: failure.reviewerResult,
147
218
  });
148
219
  }
149
220
 
221
+ sessionResult = combineReviewerSessionCosts(interruptedResults, sessionResult);
222
+ if (!sessionResult) {
223
+ throw new GoalReviewError("Reviewer session ended without a result.", { state });
224
+ }
225
+
150
226
  const rawReviewerOutput = sessionResult.assistantText;
151
227
  await writeFile(rawReviewPath, rawReviewerOutput, "utf8");
152
228
 
@@ -367,7 +443,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
367
443
  }
368
444
  const factoryResult = await sessionFactory({
369
445
  cwd: options.cwd,
370
- tools: DEFAULT_WORKER_TOOLS,
446
+ tools: GOAL_REVIEWER_TOOLS,
371
447
  model: options.model,
372
448
  modelName: options.modelName,
373
449
  thinkingLevel: options.thinkingLevel,
@@ -414,6 +490,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
414
490
  timedOut: promptResult.timedOut,
415
491
  aborted: promptResult.aborted,
416
492
  error: promptResult.error,
493
+ failure: promptResult.failure,
417
494
  };
418
495
  } catch (exc) {
419
496
  return {
@@ -422,6 +499,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
422
499
  reviewerSessionFile: session?.sessionFile,
423
500
  reviewerCostTotal,
424
501
  error: errorMessage(exc),
502
+ failure: exc,
425
503
  };
426
504
  } finally {
427
505
  if (session) {
@@ -434,6 +512,119 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
434
512
  }
435
513
  }
436
514
 
515
+ async function runReviewerWithNetworkRecovery(options: {
516
+ runner: GoalReviewerRunner;
517
+ runnerOptions: GoalReviewerRunnerOptions;
518
+ networkRecovery: Readonly<NetworkRecoveryConfig> | undefined;
519
+ abortSignal: AbortSignal | undefined;
520
+ now: () => Date;
521
+ interruptedResults: GoalReviewerSessionResult[];
522
+ interruptedEvidencePaths: string[];
523
+ iterationDir: string;
524
+ onInterruption: () => Promise<void>;
525
+ onRecoveryEvent: (event: NetworkRecoveryEvent) => void;
526
+ }): Promise<GoalReviewerSessionResult> {
527
+ const run = async (recoverySignal?: AbortSignal): Promise<GoalReviewerSessionResult> => {
528
+ const result = await options.runner({
529
+ ...options.runnerOptions,
530
+ abortSignal: combineAbortSignals(options.abortSignal, recoverySignal),
531
+ });
532
+ const networkFailure = recoverableReviewerFailure(result);
533
+ if (networkFailure === undefined) {
534
+ return result;
535
+ }
536
+
537
+ options.interruptedResults.push(result);
538
+ options.interruptedEvidencePaths.push(
539
+ await writeInterruptedReviewEvidence(options.iterationDir, options.interruptedResults.length, result),
540
+ );
541
+ await options.onInterruption();
542
+ throw new ReviewerNetworkFailure(networkFailure, result);
543
+ };
544
+
545
+ try {
546
+ return await run();
547
+ } catch (initialFailure) {
548
+ const classification = classifyNetworkFailure(initialFailure);
549
+ if (!options.networkRecovery?.enabled || !classification.recoverable) {
550
+ throw initialFailure;
551
+ }
552
+ const recovered = await recoverNetworkOperation({
553
+ initialFailure,
554
+ config: options.networkRecovery,
555
+ signal: options.abortSignal,
556
+ now: () => options.now().getTime(),
557
+ onEvent: options.onRecoveryEvent,
558
+ retry: ({ signal }) => run(signal),
559
+ });
560
+ return recovered.value;
561
+ }
562
+ }
563
+
564
+ class ReviewerNetworkFailure extends Error {
565
+ readonly sessionResult: GoalReviewerSessionResult;
566
+
567
+ constructor(failure: unknown, sessionResult: GoalReviewerSessionResult) {
568
+ super(sessionResult.error ?? errorMessage(failure), { cause: failure });
569
+ this.name = "ReviewerNetworkFailure";
570
+ this.sessionResult = sessionResult;
571
+ }
572
+ }
573
+
574
+ function recoverableReviewerFailure(result: GoalReviewerSessionResult): unknown | undefined {
575
+ if (result.aborted || result.timedOut || !result.error) {
576
+ return undefined;
577
+ }
578
+ const failure = result.failure ?? new Error(result.error);
579
+ return classifyNetworkFailure(failure).recoverable ? failure : undefined;
580
+ }
581
+
582
+ async function writeInterruptedReviewEvidence(
583
+ iterationDir: string,
584
+ interruption: number,
585
+ result: GoalReviewerSessionResult,
586
+ ): Promise<string> {
587
+ const outputPath = path.join(
588
+ iterationDir,
589
+ `REVIEW_RESULT_NETWORK_INTERRUPTED_${String(interruption).padStart(2, "0")}.txt`,
590
+ );
591
+ const evidence = result.assistantText.trim()
592
+ ? result.assistantText
593
+ : `[No assistant text was captured.]\n\n${result.error ?? "Transient reviewer network failure."}\n`;
594
+ await writeFile(outputPath, evidence, "utf8");
595
+ return outputPath;
596
+ }
597
+
598
+ function reviewerCost(results: readonly GoalReviewerSessionResult[]): number {
599
+ return results.reduce(
600
+ (total, result) =>
601
+ total +
602
+ (typeof result.reviewerCostTotal === "number" && Number.isFinite(result.reviewerCostTotal)
603
+ ? result.reviewerCostTotal
604
+ : 0),
605
+ 0,
606
+ );
607
+ }
608
+
609
+ function combineReviewerSessionCosts(
610
+ interrupted: readonly GoalReviewerSessionResult[],
611
+ final?: GoalReviewerSessionResult,
612
+ ): GoalReviewerSessionResult | undefined {
613
+ if (!final && interrupted.length === 0) {
614
+ return undefined;
615
+ }
616
+ const base = final ?? interrupted.at(-1)!;
617
+ const reviewerCostTotal = reviewerCost([...interrupted, ...(final ? [final] : [])]);
618
+ return { ...base, reviewerCostTotal };
619
+ }
620
+
621
+ function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined {
622
+ const available = signals.filter((signal): signal is AbortSignal => Boolean(signal));
623
+ if (available.length === 0) return undefined;
624
+ if (available.length === 1) return available[0];
625
+ return AbortSignal.any(available);
626
+ }
627
+
437
628
  function currentReviewableIteration(state: GoalLoopState): GoalIterationState {
438
629
  const iteration = currentIteration(state, state.currentIteration);
439
630
  if (iteration.status !== "todo_executed" && iteration.status !== "failed") {
@@ -15,6 +15,7 @@ import {
15
15
  recordWorkerResult,
16
16
  } from "./goal_loop.ts";
17
17
  import { GoalStateStore } from "./goal_state.ts";
18
+ import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
18
19
  import { validateTodoMarkdown } from "./todo_generator.ts";
19
20
 
20
21
  export const GOAL_TODO_EXECUTION_PROGRESS_FILE = "WORKER_PROGRESS.jsonl";
@@ -32,6 +33,7 @@ export interface GoalTodoExecutionOptions {
32
33
  thinkingLevel?: string;
33
34
  maxBashTimeoutMs?: number;
34
35
  maxAttemptsPerTask?: number;
36
+ networkRecovery?: Readonly<NetworkRecoveryConfig>;
35
37
  commit?: boolean;
36
38
  now?: () => Date;
37
39
  onProgress?: (update: CoordinatorProgressUpdate) => void;
@@ -152,6 +154,7 @@ export async function runGoalTodoExecutionLongTask(
152
154
  taskTimeoutMs: childTimeoutMs,
153
155
  maxBashTimeoutMs: options.maxBashTimeoutMs,
154
156
  maxAttemptsPerTask: options.maxAttemptsPerTask,
157
+ networkRecovery: options.networkRecovery,
155
158
  onProgress: (update) => {
156
159
  progressEvents.push(update);
157
160
  options.onProgress?.(update);