pi-long-task 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +153 -3
- package/package.json +1 -1
- package/src/coordinator.ts +861 -62
- package/src/goal_discovery.ts +2 -0
- package/src/goal_loop.ts +76 -0
- package/src/goal_orchestrator.ts +98 -1
- package/src/goal_review.ts +206 -15
- package/src/goal_todo_execution.ts +7 -0
- package/src/goal_todo_generation.ts +108 -13
- package/src/index.ts +15 -1
- package/src/network_failure.ts +574 -0
- package/src/network_recovery.ts +395 -0
- package/src/network_recovery_config.ts +89 -0
- package/src/planner_config.ts +214 -0
- package/src/planner_progress.ts +156 -0
- package/src/render.ts +38 -0
- package/src/session_guard.ts +120 -7
- package/src/todo_generator.ts +84 -7
- package/src/types.ts +68 -0
- package/src/worker_capabilities.ts +103 -0
- package/src/worker_config.ts +148 -7
- package/src/worker_session.ts +33 -1
package/src/goal_discovery.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
} from "./goal_spec.ts";
|
|
14
14
|
import type { GoalLoopState } from "./goal_loop.ts";
|
|
15
15
|
import type { GoalStateStore } from "./goal_state.ts";
|
|
16
|
+
import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
|
|
16
17
|
|
|
17
18
|
export type GoalDiscoveryEntrypoint = "pi_goal_task" | "pi_long_task";
|
|
18
19
|
export type GoalDiscoveryRoute = "discovery" | "direct";
|
|
@@ -41,6 +42,7 @@ export interface GoalDiscoveryRunnerOptions {
|
|
|
41
42
|
model?: unknown;
|
|
42
43
|
modelName?: string;
|
|
43
44
|
thinkingLevel?: string;
|
|
45
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
44
46
|
now: () => Date;
|
|
45
47
|
}
|
|
46
48
|
|
package/src/goal_loop.ts
CHANGED
|
@@ -92,6 +92,13 @@ export interface GoalWorkerResultState {
|
|
|
92
92
|
|
|
93
93
|
export type GoalReviewerDecision = "complete" | "incomplete" | "blocked" | "failed";
|
|
94
94
|
|
|
95
|
+
export interface GoalReviewerRecoveryState {
|
|
96
|
+
interruptions: number;
|
|
97
|
+
evidencePaths: string[];
|
|
98
|
+
reviewerCostTotal: number;
|
|
99
|
+
updatedAt: string;
|
|
100
|
+
}
|
|
101
|
+
|
|
95
102
|
export interface GoalReviewerResultState {
|
|
96
103
|
decision: GoalReviewerDecision;
|
|
97
104
|
complete: boolean;
|
|
@@ -130,6 +137,7 @@ export interface GoalIterationState {
|
|
|
130
137
|
deadlineAt?: string;
|
|
131
138
|
generatedTodo?: GeneratedTodoState;
|
|
132
139
|
workerResult?: GoalWorkerResultState;
|
|
140
|
+
reviewerRecovery?: GoalReviewerRecoveryState;
|
|
133
141
|
reviewerResult?: GoalReviewerResultState;
|
|
134
142
|
completion?: GoalCompletionState;
|
|
135
143
|
}
|
|
@@ -402,6 +410,70 @@ export function recordReviewerResult(
|
|
|
402
410
|
});
|
|
403
411
|
}
|
|
404
412
|
|
|
413
|
+
export function excludeNetworkOutageFromGoalDeadlines(
|
|
414
|
+
state: GoalLoopState,
|
|
415
|
+
outageMs: number,
|
|
416
|
+
operation: "planner" | "reviewer",
|
|
417
|
+
options: { now?: Date } = {},
|
|
418
|
+
): GoalLoopState {
|
|
419
|
+
if (!Number.isFinite(outageMs) || outageMs <= 0 || isTerminalGoalLoopStatus(state.status)) {
|
|
420
|
+
return state;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
const excludedMs = Math.ceil(outageMs);
|
|
424
|
+
const timestamp = (options.now ?? new Date()).toISOString();
|
|
425
|
+
const iterations = state.iterations.map((iteration) =>
|
|
426
|
+
iteration.iteration === state.currentIteration && iteration.deadlineAt
|
|
427
|
+
? { ...iteration, deadlineAt: shiftTimestamp(iteration.deadlineAt, excludedMs), updatedAt: timestamp }
|
|
428
|
+
: iteration,
|
|
429
|
+
);
|
|
430
|
+
return withTrace(
|
|
431
|
+
{
|
|
432
|
+
...state,
|
|
433
|
+
deadlineAt: state.deadlineAt ? shiftTimestamp(state.deadlineAt, excludedMs) : undefined,
|
|
434
|
+
iterations,
|
|
435
|
+
updatedAt: timestamp,
|
|
436
|
+
},
|
|
437
|
+
{
|
|
438
|
+
timestamp,
|
|
439
|
+
phase: state.phase,
|
|
440
|
+
event: "network_wait_excluded",
|
|
441
|
+
message: `Excluded ${excludedMs}ms of ${operation} network recovery from goal-loop deadlines.`,
|
|
442
|
+
iteration: state.currentIteration || undefined,
|
|
443
|
+
details: { operation, outageMs: excludedMs },
|
|
444
|
+
},
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
export function recordReviewerRecoveryEvidence(
|
|
449
|
+
state: GoalLoopState,
|
|
450
|
+
iterationNumber: number,
|
|
451
|
+
recovery: Omit<GoalReviewerRecoveryState, "updatedAt">,
|
|
452
|
+
options: { now?: Date } = {},
|
|
453
|
+
): GoalLoopState {
|
|
454
|
+
const timestamp = (options.now ?? new Date()).toISOString();
|
|
455
|
+
return updateIteration(state, iterationNumber, ["todo_executed", "failed"], timestamp, (item) => ({
|
|
456
|
+
item: {
|
|
457
|
+
...item,
|
|
458
|
+
updatedAt: timestamp,
|
|
459
|
+
reviewerRecovery: { ...recovery, evidencePaths: [...recovery.evidencePaths], updatedAt: timestamp },
|
|
460
|
+
},
|
|
461
|
+
phase: state.phase,
|
|
462
|
+
trace: {
|
|
463
|
+
timestamp,
|
|
464
|
+
phase: state.phase,
|
|
465
|
+
event: "reviewer_network_interrupted",
|
|
466
|
+
message: `Preserved ${recovery.interruptions} interrupted reviewer call(s) during network recovery.`,
|
|
467
|
+
iteration: iterationNumber,
|
|
468
|
+
details: {
|
|
469
|
+
interruptions: recovery.interruptions,
|
|
470
|
+
evidencePaths: recovery.evidencePaths,
|
|
471
|
+
reviewerCostTotal: recovery.reviewerCostTotal,
|
|
472
|
+
},
|
|
473
|
+
},
|
|
474
|
+
}));
|
|
475
|
+
}
|
|
476
|
+
|
|
405
477
|
export function failGoalLoop(
|
|
406
478
|
state: GoalLoopState,
|
|
407
479
|
reason: string,
|
|
@@ -597,6 +669,10 @@ function withTrace(state: GoalLoopState, event: GoalLoopTraceEvent): GoalLoopSta
|
|
|
597
669
|
return { ...state, trace: [...state.trace, event] };
|
|
598
670
|
}
|
|
599
671
|
|
|
672
|
+
function shiftTimestamp(timestamp: string, deltaMs: number): string {
|
|
673
|
+
return new Date(Date.parse(timestamp) + deltaMs).toISOString();
|
|
674
|
+
}
|
|
675
|
+
|
|
600
676
|
function positiveInteger(value: number | undefined, fallback: number): number {
|
|
601
677
|
return optionalPositiveInteger(value) ?? fallback;
|
|
602
678
|
}
|
package/src/goal_orchestrator.ts
CHANGED
|
@@ -20,6 +20,17 @@ import {
|
|
|
20
20
|
} from "./goal_loop.ts";
|
|
21
21
|
import { GoalStateStore } from "./goal_state.ts";
|
|
22
22
|
import type { GoalSpecification } from "./goal_spec.ts";
|
|
23
|
+
import {
|
|
24
|
+
formatNetworkRecoveryStatus,
|
|
25
|
+
type NetworkRecoveryEvent,
|
|
26
|
+
type NetworkRecoveryEventType,
|
|
27
|
+
} from "./network_recovery.ts";
|
|
28
|
+
import {
|
|
29
|
+
resolveNetworkRecoveryConfig,
|
|
30
|
+
type NetworkRecoveryConfig,
|
|
31
|
+
type NetworkRecoveryConfigInput,
|
|
32
|
+
} from "./network_recovery_config.ts";
|
|
33
|
+
import { validatePlannerGracefulShutdownMs, validatePlannerTimeoutMs } from "./planner_config.ts";
|
|
23
34
|
import { runGoalReviewSession, type GoalReviewResult, type GoalReviewerRunner } from "./goal_review.ts";
|
|
24
35
|
import {
|
|
25
36
|
runGoalTodoExecutionLongTask,
|
|
@@ -43,6 +54,7 @@ export type GoalLoopProgressPhase =
|
|
|
43
54
|
| "todo_executed"
|
|
44
55
|
| "review_start"
|
|
45
56
|
| "reviewed"
|
|
57
|
+
| "network_wait"
|
|
46
58
|
| "complete";
|
|
47
59
|
|
|
48
60
|
export interface GoalLoopProgressUpdate {
|
|
@@ -70,6 +82,12 @@ export interface GoalLoopProgressUpdate {
|
|
|
70
82
|
reviewerCostTotal: number;
|
|
71
83
|
totalCost: number;
|
|
72
84
|
childProgress?: CoordinatorProgressUpdate;
|
|
85
|
+
networkRecoveryEvent?: NetworkRecoveryEventType;
|
|
86
|
+
networkRetryCount?: number;
|
|
87
|
+
networkOutageElapsedMs?: number;
|
|
88
|
+
networkNextRetryAtMs?: number;
|
|
89
|
+
networkNextRetryInMs?: number;
|
|
90
|
+
networkFailureReason?: string;
|
|
73
91
|
}
|
|
74
92
|
|
|
75
93
|
export type GoalLoopProgressHandler = (update: GoalLoopProgressUpdate) => void;
|
|
@@ -92,10 +110,14 @@ export interface RunGoalLoopOptions extends GoalLoopLimitInput {
|
|
|
92
110
|
thinkingLevel?: string;
|
|
93
111
|
maxBashTimeoutMs?: number;
|
|
94
112
|
maxAttemptsPerTask?: number;
|
|
113
|
+
todoTimeoutMs?: number;
|
|
114
|
+
todoGracefulShutdownMs?: number;
|
|
115
|
+
networkRecovery?: NetworkRecoveryConfigInput;
|
|
95
116
|
commit?: boolean;
|
|
96
117
|
now?: () => Date;
|
|
97
118
|
onWorkerProgress?: (update: CoordinatorProgressUpdate) => void;
|
|
98
119
|
onProgress?: GoalLoopProgressHandler;
|
|
120
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
99
121
|
}
|
|
100
122
|
|
|
101
123
|
export interface GoalLoopRunResult {
|
|
@@ -120,6 +142,9 @@ export class GoalLoopOrchestratorError extends Error {
|
|
|
120
142
|
|
|
121
143
|
export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoopRunResult> {
|
|
122
144
|
const now = options.now ?? (() => new Date());
|
|
145
|
+
const networkRecovery = resolveNetworkRecoveryConfig(options.networkRecovery);
|
|
146
|
+
const todoTimeoutMs = validatePlannerTimeoutMs(options.todoTimeoutMs);
|
|
147
|
+
const todoGracefulShutdownMs = validatePlannerGracefulShutdownMs(options.todoGracefulShutdownMs);
|
|
123
148
|
let state =
|
|
124
149
|
options.initialState ??
|
|
125
150
|
createGoalLoopState({
|
|
@@ -149,7 +174,9 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
149
174
|
await store.saveState(state);
|
|
150
175
|
await store.initializeResultIfMissing(state);
|
|
151
176
|
await store.appendNewTraceEvents(await store.durableTraceLength(), state);
|
|
177
|
+
let progressClosed = false;
|
|
152
178
|
const publish = (phase: GoalLoopProgressPhase, message: string, extra: Partial<GoalLoopProgressUpdate> = {}) => {
|
|
179
|
+
if (progressClosed) return;
|
|
153
180
|
options.onProgress?.({
|
|
154
181
|
message,
|
|
155
182
|
phase,
|
|
@@ -173,6 +200,39 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
173
200
|
...extra,
|
|
174
201
|
});
|
|
175
202
|
};
|
|
203
|
+
const recoveryProgress = (
|
|
204
|
+
restorePhase: GoalLoopProgressPhase,
|
|
205
|
+
restoreMessage: string,
|
|
206
|
+
extra: Partial<GoalLoopProgressUpdate> = {},
|
|
207
|
+
) => {
|
|
208
|
+
let cleaned = false;
|
|
209
|
+
return (event: NetworkRecoveryEvent) => {
|
|
210
|
+
options.onNetworkRecovery?.(event);
|
|
211
|
+
if (cleaned || progressClosed) return;
|
|
212
|
+
if (event.type === "cleanup") {
|
|
213
|
+
cleaned = true;
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
if (event.type === "recovered") {
|
|
217
|
+
publish(restorePhase, restoreMessage, extra);
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
if (isTerminalNetworkRecoveryProgressEvent(event.type)) return;
|
|
221
|
+
|
|
222
|
+
const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
|
|
223
|
+
const message = formatNetworkRecoveryStatus(event);
|
|
224
|
+
publish("network_wait", message, {
|
|
225
|
+
...extra,
|
|
226
|
+
networkRecoveryEvent: event.type,
|
|
227
|
+
networkRetryCount: event.state.retryCount,
|
|
228
|
+
networkOutageElapsedMs: event.state.elapsedMs,
|
|
229
|
+
networkNextRetryAtMs: event.state.nextRetryAtMs,
|
|
230
|
+
networkNextRetryInMs:
|
|
231
|
+
event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
|
|
232
|
+
networkFailureReason: event.state.lastFailure.reason,
|
|
233
|
+
});
|
|
234
|
+
};
|
|
235
|
+
};
|
|
176
236
|
|
|
177
237
|
publish("goal_start", `Starting goal loop: ${state.goal}`);
|
|
178
238
|
|
|
@@ -186,6 +246,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
186
246
|
options,
|
|
187
247
|
now,
|
|
188
248
|
publish,
|
|
249
|
+
networkRecovery,
|
|
189
250
|
});
|
|
190
251
|
}
|
|
191
252
|
|
|
@@ -226,8 +287,16 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
226
287
|
modelName: options.modelName,
|
|
227
288
|
thinkingLevel: options.thinkingLevel,
|
|
228
289
|
maxBashTimeoutMs: options.maxBashTimeoutMs,
|
|
290
|
+
todoTimeoutMs,
|
|
291
|
+
todoGracefulShutdownMs,
|
|
292
|
+
networkRecovery,
|
|
229
293
|
now,
|
|
230
294
|
goalSpecification,
|
|
295
|
+
onNetworkRecovery: recoveryProgress(
|
|
296
|
+
"todo_generation_start",
|
|
297
|
+
`Goal iteration ${nextIteration}: generating TODO markdown.`,
|
|
298
|
+
{ iteration: nextIteration },
|
|
299
|
+
),
|
|
231
300
|
});
|
|
232
301
|
generationResults.push(generation);
|
|
233
302
|
state = generation.state;
|
|
@@ -251,8 +320,16 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
251
320
|
modelName: options.modelName,
|
|
252
321
|
thinkingLevel: options.thinkingLevel,
|
|
253
322
|
maxBashTimeoutMs: options.maxBashTimeoutMs,
|
|
323
|
+
todoTimeoutMs,
|
|
324
|
+
todoGracefulShutdownMs,
|
|
325
|
+
networkRecovery,
|
|
254
326
|
now,
|
|
255
327
|
goalSpecification,
|
|
328
|
+
onNetworkRecovery: recoveryProgress(
|
|
329
|
+
"todo_generation_start",
|
|
330
|
+
`Goal iteration ${current.iteration}: resuming TODO generation.`,
|
|
331
|
+
{ iteration: current.iteration },
|
|
332
|
+
),
|
|
256
333
|
});
|
|
257
334
|
generationResults.push(generation);
|
|
258
335
|
state = generation.state;
|
|
@@ -278,6 +355,9 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
278
355
|
thinkingLevel: options.thinkingLevel,
|
|
279
356
|
maxBashTimeoutMs: options.maxBashTimeoutMs,
|
|
280
357
|
maxAttemptsPerTask: options.maxAttemptsPerTask,
|
|
358
|
+
todoTimeoutMs,
|
|
359
|
+
todoGracefulShutdownMs,
|
|
360
|
+
networkRecovery,
|
|
281
361
|
commit: options.commit,
|
|
282
362
|
now,
|
|
283
363
|
onProgress: (update) => {
|
|
@@ -324,7 +404,13 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
324
404
|
thinkingLevel: options.thinkingLevel,
|
|
325
405
|
now,
|
|
326
406
|
goalSpecification,
|
|
407
|
+
networkRecovery,
|
|
327
408
|
timeoutMs: remainingReviewTimeout(state, current.deadlineAt, now()),
|
|
409
|
+
onNetworkRecovery: recoveryProgress(
|
|
410
|
+
"review_start",
|
|
411
|
+
`Goal iteration ${current.iteration}: reviewing goal completion.`,
|
|
412
|
+
{ iteration: current.iteration },
|
|
413
|
+
),
|
|
328
414
|
});
|
|
329
415
|
reviewResults.push(review);
|
|
330
416
|
state = review.state;
|
|
@@ -365,6 +451,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
365
451
|
}
|
|
366
452
|
|
|
367
453
|
publish("complete", `Goal loop ${state.status}: ${state.completion?.reason ?? "finished"}`);
|
|
454
|
+
progressClosed = true;
|
|
368
455
|
|
|
369
456
|
return {
|
|
370
457
|
state,
|
|
@@ -385,6 +472,7 @@ async function maybeRunGoalDiscovery(options: {
|
|
|
385
472
|
options: RunGoalLoopOptions;
|
|
386
473
|
now: () => Date;
|
|
387
474
|
publish: (phase: GoalLoopProgressPhase, message: string, extra?: Partial<GoalLoopProgressUpdate>) => void;
|
|
475
|
+
networkRecovery: Readonly<NetworkRecoveryConfig>;
|
|
388
476
|
}): Promise<GoalSpecification | undefined> {
|
|
389
477
|
if (options.discoveryDecision.route !== "discovery") {
|
|
390
478
|
return options.existingSpecification;
|
|
@@ -407,6 +495,7 @@ async function maybeRunGoalDiscovery(options: {
|
|
|
407
495
|
model: options.options.model,
|
|
408
496
|
modelName: options.options.modelName,
|
|
409
497
|
thinkingLevel: options.options.thinkingLevel,
|
|
498
|
+
networkRecovery: options.networkRecovery,
|
|
410
499
|
now: options.now,
|
|
411
500
|
});
|
|
412
501
|
await options.store.saveGoalSpecification(spec);
|
|
@@ -467,7 +556,11 @@ function accumulatedWorkerCost(state: GoalLoopState): number {
|
|
|
467
556
|
}
|
|
468
557
|
|
|
469
558
|
function accumulatedReviewerCost(state: GoalLoopState): number {
|
|
470
|
-
return sumFinite(
|
|
559
|
+
return sumFinite(
|
|
560
|
+
state.iterations.map(
|
|
561
|
+
(iteration) => iteration.reviewerResult?.reviewerCostTotal ?? iteration.reviewerRecovery?.reviewerCostTotal,
|
|
562
|
+
),
|
|
563
|
+
);
|
|
471
564
|
}
|
|
472
565
|
|
|
473
566
|
function sumFinite(values: Array<number | undefined>): number {
|
|
@@ -477,6 +570,10 @@ function sumFinite(values: Array<number | undefined>): number {
|
|
|
477
570
|
);
|
|
478
571
|
}
|
|
479
572
|
|
|
573
|
+
function isTerminalNetworkRecoveryProgressEvent(type: NetworkRecoveryEventType): boolean {
|
|
574
|
+
return type === "failed" || type === "cancelled" || type === "outage_expired";
|
|
575
|
+
}
|
|
576
|
+
|
|
480
577
|
function requiredGoal(goal: string | undefined): string {
|
|
481
578
|
const trimmed = goal?.trim();
|
|
482
579
|
if (!trimmed) {
|
package/src/goal_review.ts
CHANGED
|
@@ -3,18 +3,22 @@ import path from "node:path";
|
|
|
3
3
|
|
|
4
4
|
import {
|
|
5
5
|
cancelGoalLoop,
|
|
6
|
+
excludeNetworkOutageFromGoalDeadlines,
|
|
6
7
|
type GoalIterationState,
|
|
7
8
|
type GoalLoopState,
|
|
8
9
|
type GoalReviewerDecision,
|
|
9
10
|
type GoalReviewerResultState,
|
|
11
|
+
recordReviewerRecoveryEvidence,
|
|
10
12
|
recordReviewerResult,
|
|
11
13
|
} from "./goal_loop.ts";
|
|
12
14
|
import { GoalStateStore } from "./goal_state.ts";
|
|
13
15
|
import { runGuardedSessionPrompt } from "./session_guard.ts";
|
|
14
16
|
import { goalSpecificationToMarkdown, type GoalSpecification } from "./goal_spec.ts";
|
|
17
|
+
import { classifyNetworkFailure } from "./network_failure.ts";
|
|
18
|
+
import { recoverNetworkOperation, type NetworkRecoveryEvent } from "./network_recovery.ts";
|
|
19
|
+
import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
|
|
15
20
|
import {
|
|
16
21
|
createIsolatedWorkerSession,
|
|
17
|
-
DEFAULT_WORKER_TOOLS,
|
|
18
22
|
workerUsageCostFromEvent,
|
|
19
23
|
workerUsageCostFromStats,
|
|
20
24
|
workerUsageCostKeyFromEvent,
|
|
@@ -23,6 +27,8 @@ import {
|
|
|
23
27
|
|
|
24
28
|
export const GOAL_REVIEW_PAYLOAD_FILE = "REVIEW_TASK.md";
|
|
25
29
|
export const GOAL_REVIEW_RAW_FILE = "REVIEW_RESULT_RAW.txt";
|
|
30
|
+
/** Reviewer retries are constrained to inspection and read-only verification. */
|
|
31
|
+
export const GOAL_REVIEWER_TOOLS = ["read", "bash", "grep", "find", "ls"] as const;
|
|
26
32
|
|
|
27
33
|
export interface GoalReviewOptions {
|
|
28
34
|
state: GoalLoopState;
|
|
@@ -36,7 +42,9 @@ export interface GoalReviewOptions {
|
|
|
36
42
|
now?: () => Date;
|
|
37
43
|
sessionFactory?: WorkerSessionFactory;
|
|
38
44
|
goalSpecification?: GoalSpecification;
|
|
45
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
39
46
|
timeoutMs?: number;
|
|
47
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
40
48
|
}
|
|
41
49
|
|
|
42
50
|
export interface GoalReviewResult {
|
|
@@ -59,6 +67,7 @@ export interface GoalReviewerRunnerOptions {
|
|
|
59
67
|
modelName?: string;
|
|
60
68
|
thinkingLevel?: string;
|
|
61
69
|
sessionFactory?: WorkerSessionFactory;
|
|
70
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
62
71
|
}
|
|
63
72
|
|
|
64
73
|
export interface GoalReviewerSessionResult {
|
|
@@ -69,6 +78,8 @@ export interface GoalReviewerSessionResult {
|
|
|
69
78
|
timedOut?: boolean;
|
|
70
79
|
aborted?: boolean;
|
|
71
80
|
error?: string;
|
|
81
|
+
/** Untouched provider/transport failure retained for coordinator recovery classification. */
|
|
82
|
+
failure?: unknown;
|
|
72
83
|
}
|
|
73
84
|
|
|
74
85
|
export type GoalReviewerRunner = (options: GoalReviewerRunnerOptions) => Promise<GoalReviewerSessionResult>;
|
|
@@ -91,7 +102,7 @@ export class GoalReviewError extends Error {
|
|
|
91
102
|
export async function runGoalReviewSession(options: GoalReviewOptions): Promise<GoalReviewResult> {
|
|
92
103
|
const now = options.now ?? (() => new Date());
|
|
93
104
|
let state = options.state;
|
|
94
|
-
|
|
105
|
+
let previousTraceLength = state.trace.length;
|
|
95
106
|
const store =
|
|
96
107
|
options.store ?? new GoalStateStore({ cwd: options.cwd, goalRunId: state.goalRunId, goalRunDir: state.goalRunDir });
|
|
97
108
|
|
|
@@ -116,19 +127,78 @@ export async function runGoalReviewSession(options: GoalReviewOptions): Promise<
|
|
|
116
127
|
throw new GoalReviewError("Goal review was aborted before starting.", { state });
|
|
117
128
|
}
|
|
118
129
|
|
|
119
|
-
|
|
130
|
+
const interruptedResults: GoalReviewerSessionResult[] = [];
|
|
131
|
+
const interruptedEvidencePaths: string[] = [];
|
|
132
|
+
let excludedOutageMs = 0;
|
|
133
|
+
const captureOutage = (event: NetworkRecoveryEvent) => {
|
|
134
|
+
if (event.type === "cleanup") {
|
|
135
|
+
excludedOutageMs += event.state.elapsedMs;
|
|
136
|
+
}
|
|
137
|
+
options.onNetworkRecovery?.(event);
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
let sessionResult: GoalReviewerSessionResult | undefined;
|
|
141
|
+
let sessionFailure: unknown;
|
|
120
142
|
try {
|
|
121
|
-
sessionResult = await (
|
|
122
|
-
|
|
123
|
-
|
|
143
|
+
sessionResult = await runReviewerWithNetworkRecovery({
|
|
144
|
+
runner: options.reviewerRunner ?? runGoalReviewerSession,
|
|
145
|
+
runnerOptions: {
|
|
146
|
+
prompt: payload,
|
|
147
|
+
cwd: path.resolve(options.cwd ?? process.cwd()),
|
|
148
|
+
abortSignal: options.abortSignal,
|
|
149
|
+
timeoutMs: options.timeoutMs ?? state.limits.reviewerTimeoutMs,
|
|
150
|
+
model: options.model,
|
|
151
|
+
modelName: options.modelName,
|
|
152
|
+
thinkingLevel: options.thinkingLevel,
|
|
153
|
+
sessionFactory: options.sessionFactory,
|
|
154
|
+
networkRecovery: options.networkRecovery,
|
|
155
|
+
},
|
|
156
|
+
networkRecovery: options.networkRecovery,
|
|
124
157
|
abortSignal: options.abortSignal,
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
158
|
+
now,
|
|
159
|
+
interruptedResults,
|
|
160
|
+
interruptedEvidencePaths,
|
|
161
|
+
iterationDir,
|
|
162
|
+
onInterruption: async () => {
|
|
163
|
+
state = recordReviewerRecoveryEvidence(
|
|
164
|
+
state,
|
|
165
|
+
iteration.iteration,
|
|
166
|
+
{
|
|
167
|
+
interruptions: interruptedResults.length,
|
|
168
|
+
evidencePaths: interruptedEvidencePaths,
|
|
169
|
+
reviewerCostTotal: reviewerCost(interruptedResults),
|
|
170
|
+
},
|
|
171
|
+
{ now: now() },
|
|
172
|
+
);
|
|
173
|
+
await persistStateChange(store, previousTraceLength, state);
|
|
174
|
+
previousTraceLength = state.trace.length;
|
|
175
|
+
},
|
|
176
|
+
onRecoveryEvent: captureOutage,
|
|
130
177
|
});
|
|
131
178
|
} catch (error) {
|
|
179
|
+
sessionFailure = error;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (excludedOutageMs > 0) {
|
|
183
|
+
state = excludeNetworkOutageFromGoalDeadlines(state, excludedOutageMs, "reviewer", { now: now() });
|
|
184
|
+
}
|
|
185
|
+
if (state.trace.length !== previousTraceLength) {
|
|
186
|
+
await persistStateChange(store, previousTraceLength, state);
|
|
187
|
+
previousTraceLength = state.trace.length;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
if (sessionFailure !== undefined && options.abortSignal?.aborted) {
|
|
191
|
+
const reason = `Goal review was aborted during network recovery: ${errorMessage(
|
|
192
|
+
options.abortSignal.reason ?? sessionFailure,
|
|
193
|
+
)}`;
|
|
194
|
+
state = cancelGoalLoop(state, reason, { now: now() });
|
|
195
|
+
await persistStateChange(store, previousTraceLength, state);
|
|
196
|
+
await store.writeIterationSnapshot(currentIteration(state, iteration.iteration));
|
|
197
|
+
throw new GoalReviewError(reason, { cause: sessionFailure, state });
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (sessionFailure !== undefined) {
|
|
201
|
+
const interruptedResult = combineReviewerSessionCosts(interruptedResults);
|
|
132
202
|
const failure = await recordReviewFailure({
|
|
133
203
|
state,
|
|
134
204
|
iteration,
|
|
@@ -136,17 +206,23 @@ export async function runGoalReviewSession(options: GoalReviewOptions): Promise<
|
|
|
136
206
|
previousTraceLength,
|
|
137
207
|
payloadPath,
|
|
138
208
|
rawReviewPath,
|
|
139
|
-
message: `Reviewer session failed: ${errorMessage(
|
|
140
|
-
error,
|
|
209
|
+
message: `Reviewer session failed: ${errorMessage(sessionFailure)}`,
|
|
210
|
+
error: sessionFailure,
|
|
211
|
+
sessionResult: interruptedResult,
|
|
141
212
|
now,
|
|
142
213
|
});
|
|
143
214
|
throw new GoalReviewError(failure.reviewerResult.summary, {
|
|
144
|
-
cause:
|
|
215
|
+
cause: sessionFailure,
|
|
145
216
|
state: failure.state,
|
|
146
217
|
reviewerResult: failure.reviewerResult,
|
|
147
218
|
});
|
|
148
219
|
}
|
|
149
220
|
|
|
221
|
+
sessionResult = combineReviewerSessionCosts(interruptedResults, sessionResult);
|
|
222
|
+
if (!sessionResult) {
|
|
223
|
+
throw new GoalReviewError("Reviewer session ended without a result.", { state });
|
|
224
|
+
}
|
|
225
|
+
|
|
150
226
|
const rawReviewerOutput = sessionResult.assistantText;
|
|
151
227
|
await writeFile(rawReviewPath, rawReviewerOutput, "utf8");
|
|
152
228
|
|
|
@@ -367,7 +443,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
|
|
|
367
443
|
}
|
|
368
444
|
const factoryResult = await sessionFactory({
|
|
369
445
|
cwd: options.cwd,
|
|
370
|
-
tools:
|
|
446
|
+
tools: GOAL_REVIEWER_TOOLS,
|
|
371
447
|
model: options.model,
|
|
372
448
|
modelName: options.modelName,
|
|
373
449
|
thinkingLevel: options.thinkingLevel,
|
|
@@ -414,6 +490,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
|
|
|
414
490
|
timedOut: promptResult.timedOut,
|
|
415
491
|
aborted: promptResult.aborted,
|
|
416
492
|
error: promptResult.error,
|
|
493
|
+
failure: promptResult.failure,
|
|
417
494
|
};
|
|
418
495
|
} catch (exc) {
|
|
419
496
|
return {
|
|
@@ -422,6 +499,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
|
|
|
422
499
|
reviewerSessionFile: session?.sessionFile,
|
|
423
500
|
reviewerCostTotal,
|
|
424
501
|
error: errorMessage(exc),
|
|
502
|
+
failure: exc,
|
|
425
503
|
};
|
|
426
504
|
} finally {
|
|
427
505
|
if (session) {
|
|
@@ -434,6 +512,119 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
|
|
|
434
512
|
}
|
|
435
513
|
}
|
|
436
514
|
|
|
515
|
+
async function runReviewerWithNetworkRecovery(options: {
|
|
516
|
+
runner: GoalReviewerRunner;
|
|
517
|
+
runnerOptions: GoalReviewerRunnerOptions;
|
|
518
|
+
networkRecovery: Readonly<NetworkRecoveryConfig> | undefined;
|
|
519
|
+
abortSignal: AbortSignal | undefined;
|
|
520
|
+
now: () => Date;
|
|
521
|
+
interruptedResults: GoalReviewerSessionResult[];
|
|
522
|
+
interruptedEvidencePaths: string[];
|
|
523
|
+
iterationDir: string;
|
|
524
|
+
onInterruption: () => Promise<void>;
|
|
525
|
+
onRecoveryEvent: (event: NetworkRecoveryEvent) => void;
|
|
526
|
+
}): Promise<GoalReviewerSessionResult> {
|
|
527
|
+
const run = async (recoverySignal?: AbortSignal): Promise<GoalReviewerSessionResult> => {
|
|
528
|
+
const result = await options.runner({
|
|
529
|
+
...options.runnerOptions,
|
|
530
|
+
abortSignal: combineAbortSignals(options.abortSignal, recoverySignal),
|
|
531
|
+
});
|
|
532
|
+
const networkFailure = recoverableReviewerFailure(result);
|
|
533
|
+
if (networkFailure === undefined) {
|
|
534
|
+
return result;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
options.interruptedResults.push(result);
|
|
538
|
+
options.interruptedEvidencePaths.push(
|
|
539
|
+
await writeInterruptedReviewEvidence(options.iterationDir, options.interruptedResults.length, result),
|
|
540
|
+
);
|
|
541
|
+
await options.onInterruption();
|
|
542
|
+
throw new ReviewerNetworkFailure(networkFailure, result);
|
|
543
|
+
};
|
|
544
|
+
|
|
545
|
+
try {
|
|
546
|
+
return await run();
|
|
547
|
+
} catch (initialFailure) {
|
|
548
|
+
const classification = classifyNetworkFailure(initialFailure);
|
|
549
|
+
if (!options.networkRecovery?.enabled || !classification.recoverable) {
|
|
550
|
+
throw initialFailure;
|
|
551
|
+
}
|
|
552
|
+
const recovered = await recoverNetworkOperation({
|
|
553
|
+
initialFailure,
|
|
554
|
+
config: options.networkRecovery,
|
|
555
|
+
signal: options.abortSignal,
|
|
556
|
+
now: () => options.now().getTime(),
|
|
557
|
+
onEvent: options.onRecoveryEvent,
|
|
558
|
+
retry: ({ signal }) => run(signal),
|
|
559
|
+
});
|
|
560
|
+
return recovered.value;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
class ReviewerNetworkFailure extends Error {
|
|
565
|
+
readonly sessionResult: GoalReviewerSessionResult;
|
|
566
|
+
|
|
567
|
+
constructor(failure: unknown, sessionResult: GoalReviewerSessionResult) {
|
|
568
|
+
super(sessionResult.error ?? errorMessage(failure), { cause: failure });
|
|
569
|
+
this.name = "ReviewerNetworkFailure";
|
|
570
|
+
this.sessionResult = sessionResult;
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
function recoverableReviewerFailure(result: GoalReviewerSessionResult): unknown | undefined {
|
|
575
|
+
if (result.aborted || result.timedOut || !result.error) {
|
|
576
|
+
return undefined;
|
|
577
|
+
}
|
|
578
|
+
const failure = result.failure ?? new Error(result.error);
|
|
579
|
+
return classifyNetworkFailure(failure).recoverable ? failure : undefined;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
async function writeInterruptedReviewEvidence(
|
|
583
|
+
iterationDir: string,
|
|
584
|
+
interruption: number,
|
|
585
|
+
result: GoalReviewerSessionResult,
|
|
586
|
+
): Promise<string> {
|
|
587
|
+
const outputPath = path.join(
|
|
588
|
+
iterationDir,
|
|
589
|
+
`REVIEW_RESULT_NETWORK_INTERRUPTED_${String(interruption).padStart(2, "0")}.txt`,
|
|
590
|
+
);
|
|
591
|
+
const evidence = result.assistantText.trim()
|
|
592
|
+
? result.assistantText
|
|
593
|
+
: `[No assistant text was captured.]\n\n${result.error ?? "Transient reviewer network failure."}\n`;
|
|
594
|
+
await writeFile(outputPath, evidence, "utf8");
|
|
595
|
+
return outputPath;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
function reviewerCost(results: readonly GoalReviewerSessionResult[]): number {
|
|
599
|
+
return results.reduce(
|
|
600
|
+
(total, result) =>
|
|
601
|
+
total +
|
|
602
|
+
(typeof result.reviewerCostTotal === "number" && Number.isFinite(result.reviewerCostTotal)
|
|
603
|
+
? result.reviewerCostTotal
|
|
604
|
+
: 0),
|
|
605
|
+
0,
|
|
606
|
+
);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
function combineReviewerSessionCosts(
|
|
610
|
+
interrupted: readonly GoalReviewerSessionResult[],
|
|
611
|
+
final?: GoalReviewerSessionResult,
|
|
612
|
+
): GoalReviewerSessionResult | undefined {
|
|
613
|
+
if (!final && interrupted.length === 0) {
|
|
614
|
+
return undefined;
|
|
615
|
+
}
|
|
616
|
+
const base = final ?? interrupted.at(-1)!;
|
|
617
|
+
const reviewerCostTotal = reviewerCost([...interrupted, ...(final ? [final] : [])]);
|
|
618
|
+
return { ...base, reviewerCostTotal };
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined {
|
|
622
|
+
const available = signals.filter((signal): signal is AbortSignal => Boolean(signal));
|
|
623
|
+
if (available.length === 0) return undefined;
|
|
624
|
+
if (available.length === 1) return available[0];
|
|
625
|
+
return AbortSignal.any(available);
|
|
626
|
+
}
|
|
627
|
+
|
|
437
628
|
function currentReviewableIteration(state: GoalLoopState): GoalIterationState {
|
|
438
629
|
const iteration = currentIteration(state, state.currentIteration);
|
|
439
630
|
if (iteration.status !== "todo_executed" && iteration.status !== "failed") {
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
recordWorkerResult,
|
|
16
16
|
} from "./goal_loop.ts";
|
|
17
17
|
import { GoalStateStore } from "./goal_state.ts";
|
|
18
|
+
import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
|
|
18
19
|
import { validateTodoMarkdown } from "./todo_generator.ts";
|
|
19
20
|
|
|
20
21
|
export const GOAL_TODO_EXECUTION_PROGRESS_FILE = "WORKER_PROGRESS.jsonl";
|
|
@@ -32,6 +33,9 @@ export interface GoalTodoExecutionOptions {
|
|
|
32
33
|
thinkingLevel?: string;
|
|
33
34
|
maxBashTimeoutMs?: number;
|
|
34
35
|
maxAttemptsPerTask?: number;
|
|
36
|
+
todoTimeoutMs?: number;
|
|
37
|
+
todoGracefulShutdownMs?: number;
|
|
38
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
35
39
|
commit?: boolean;
|
|
36
40
|
now?: () => Date;
|
|
37
41
|
onProgress?: (update: CoordinatorProgressUpdate) => void;
|
|
@@ -152,6 +156,9 @@ export async function runGoalTodoExecutionLongTask(
|
|
|
152
156
|
taskTimeoutMs: childTimeoutMs,
|
|
153
157
|
maxBashTimeoutMs: options.maxBashTimeoutMs,
|
|
154
158
|
maxAttemptsPerTask: options.maxAttemptsPerTask,
|
|
159
|
+
todoTimeoutMs: options.todoTimeoutMs,
|
|
160
|
+
todoGracefulShutdownMs: options.todoGracefulShutdownMs,
|
|
161
|
+
networkRecovery: options.networkRecovery,
|
|
155
162
|
onProgress: (update) => {
|
|
156
163
|
progressEvents.push(update);
|
|
157
164
|
options.onProgress?.(update);
|