pi-long-task 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +139 -10
- package/package.json +2 -2
- package/src/coordinator.ts +1132 -45
- package/src/goal_discovery.ts +2 -0
- package/src/goal_loop.ts +76 -0
- package/src/goal_orchestrator.ts +87 -1
- package/src/goal_review.ts +206 -15
- package/src/goal_todo_execution.ts +3 -0
- package/src/goal_todo_generation.ts +96 -3
- package/src/index.ts +20 -5
- package/src/network_failure.ts +574 -0
- package/src/network_recovery.ts +395 -0
- package/src/network_recovery_config.ts +89 -0
- package/src/render.ts +2 -0
- package/src/session_guard.ts +8 -1
- package/src/todo_generator.ts +2 -2
- package/src/types.ts +32 -0
- package/src/worker_config.ts +137 -14
- package/src/worker_reuse_policy.ts +389 -0
- package/src/worker_session.ts +294 -34
package/src/goal_discovery.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
} from "./goal_spec.ts";
|
|
14
14
|
import type { GoalLoopState } from "./goal_loop.ts";
|
|
15
15
|
import type { GoalStateStore } from "./goal_state.ts";
|
|
16
|
+
import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
|
|
16
17
|
|
|
17
18
|
export type GoalDiscoveryEntrypoint = "pi_goal_task" | "pi_long_task";
|
|
18
19
|
export type GoalDiscoveryRoute = "discovery" | "direct";
|
|
@@ -41,6 +42,7 @@ export interface GoalDiscoveryRunnerOptions {
|
|
|
41
42
|
model?: unknown;
|
|
42
43
|
modelName?: string;
|
|
43
44
|
thinkingLevel?: string;
|
|
45
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
44
46
|
now: () => Date;
|
|
45
47
|
}
|
|
46
48
|
|
package/src/goal_loop.ts
CHANGED
|
@@ -92,6 +92,13 @@ export interface GoalWorkerResultState {
|
|
|
92
92
|
|
|
93
93
|
export type GoalReviewerDecision = "complete" | "incomplete" | "blocked" | "failed";
|
|
94
94
|
|
|
95
|
+
export interface GoalReviewerRecoveryState {
|
|
96
|
+
interruptions: number;
|
|
97
|
+
evidencePaths: string[];
|
|
98
|
+
reviewerCostTotal: number;
|
|
99
|
+
updatedAt: string;
|
|
100
|
+
}
|
|
101
|
+
|
|
95
102
|
export interface GoalReviewerResultState {
|
|
96
103
|
decision: GoalReviewerDecision;
|
|
97
104
|
complete: boolean;
|
|
@@ -130,6 +137,7 @@ export interface GoalIterationState {
|
|
|
130
137
|
deadlineAt?: string;
|
|
131
138
|
generatedTodo?: GeneratedTodoState;
|
|
132
139
|
workerResult?: GoalWorkerResultState;
|
|
140
|
+
reviewerRecovery?: GoalReviewerRecoveryState;
|
|
133
141
|
reviewerResult?: GoalReviewerResultState;
|
|
134
142
|
completion?: GoalCompletionState;
|
|
135
143
|
}
|
|
@@ -402,6 +410,70 @@ export function recordReviewerResult(
|
|
|
402
410
|
});
|
|
403
411
|
}
|
|
404
412
|
|
|
413
|
+
export function excludeNetworkOutageFromGoalDeadlines(
|
|
414
|
+
state: GoalLoopState,
|
|
415
|
+
outageMs: number,
|
|
416
|
+
operation: "planner" | "reviewer",
|
|
417
|
+
options: { now?: Date } = {},
|
|
418
|
+
): GoalLoopState {
|
|
419
|
+
if (!Number.isFinite(outageMs) || outageMs <= 0 || isTerminalGoalLoopStatus(state.status)) {
|
|
420
|
+
return state;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
const excludedMs = Math.ceil(outageMs);
|
|
424
|
+
const timestamp = (options.now ?? new Date()).toISOString();
|
|
425
|
+
const iterations = state.iterations.map((iteration) =>
|
|
426
|
+
iteration.iteration === state.currentIteration && iteration.deadlineAt
|
|
427
|
+
? { ...iteration, deadlineAt: shiftTimestamp(iteration.deadlineAt, excludedMs), updatedAt: timestamp }
|
|
428
|
+
: iteration,
|
|
429
|
+
);
|
|
430
|
+
return withTrace(
|
|
431
|
+
{
|
|
432
|
+
...state,
|
|
433
|
+
deadlineAt: state.deadlineAt ? shiftTimestamp(state.deadlineAt, excludedMs) : undefined,
|
|
434
|
+
iterations,
|
|
435
|
+
updatedAt: timestamp,
|
|
436
|
+
},
|
|
437
|
+
{
|
|
438
|
+
timestamp,
|
|
439
|
+
phase: state.phase,
|
|
440
|
+
event: "network_wait_excluded",
|
|
441
|
+
message: `Excluded ${excludedMs}ms of ${operation} network recovery from goal-loop deadlines.`,
|
|
442
|
+
iteration: state.currentIteration || undefined,
|
|
443
|
+
details: { operation, outageMs: excludedMs },
|
|
444
|
+
},
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
export function recordReviewerRecoveryEvidence(
|
|
449
|
+
state: GoalLoopState,
|
|
450
|
+
iterationNumber: number,
|
|
451
|
+
recovery: Omit<GoalReviewerRecoveryState, "updatedAt">,
|
|
452
|
+
options: { now?: Date } = {},
|
|
453
|
+
): GoalLoopState {
|
|
454
|
+
const timestamp = (options.now ?? new Date()).toISOString();
|
|
455
|
+
return updateIteration(state, iterationNumber, ["todo_executed", "failed"], timestamp, (item) => ({
|
|
456
|
+
item: {
|
|
457
|
+
...item,
|
|
458
|
+
updatedAt: timestamp,
|
|
459
|
+
reviewerRecovery: { ...recovery, evidencePaths: [...recovery.evidencePaths], updatedAt: timestamp },
|
|
460
|
+
},
|
|
461
|
+
phase: state.phase,
|
|
462
|
+
trace: {
|
|
463
|
+
timestamp,
|
|
464
|
+
phase: state.phase,
|
|
465
|
+
event: "reviewer_network_interrupted",
|
|
466
|
+
message: `Preserved ${recovery.interruptions} interrupted reviewer call(s) during network recovery.`,
|
|
467
|
+
iteration: iterationNumber,
|
|
468
|
+
details: {
|
|
469
|
+
interruptions: recovery.interruptions,
|
|
470
|
+
evidencePaths: recovery.evidencePaths,
|
|
471
|
+
reviewerCostTotal: recovery.reviewerCostTotal,
|
|
472
|
+
},
|
|
473
|
+
},
|
|
474
|
+
}));
|
|
475
|
+
}
|
|
476
|
+
|
|
405
477
|
export function failGoalLoop(
|
|
406
478
|
state: GoalLoopState,
|
|
407
479
|
reason: string,
|
|
@@ -597,6 +669,10 @@ function withTrace(state: GoalLoopState, event: GoalLoopTraceEvent): GoalLoopSta
|
|
|
597
669
|
return { ...state, trace: [...state.trace, event] };
|
|
598
670
|
}
|
|
599
671
|
|
|
672
|
+
function shiftTimestamp(timestamp: string, deltaMs: number): string {
|
|
673
|
+
return new Date(Date.parse(timestamp) + deltaMs).toISOString();
|
|
674
|
+
}
|
|
675
|
+
|
|
600
676
|
function positiveInteger(value: number | undefined, fallback: number): number {
|
|
601
677
|
return optionalPositiveInteger(value) ?? fallback;
|
|
602
678
|
}
|
package/src/goal_orchestrator.ts
CHANGED
|
@@ -20,6 +20,16 @@ import {
|
|
|
20
20
|
} from "./goal_loop.ts";
|
|
21
21
|
import { GoalStateStore } from "./goal_state.ts";
|
|
22
22
|
import type { GoalSpecification } from "./goal_spec.ts";
|
|
23
|
+
import {
|
|
24
|
+
formatNetworkRecoveryStatus,
|
|
25
|
+
type NetworkRecoveryEvent,
|
|
26
|
+
type NetworkRecoveryEventType,
|
|
27
|
+
} from "./network_recovery.ts";
|
|
28
|
+
import {
|
|
29
|
+
resolveNetworkRecoveryConfig,
|
|
30
|
+
type NetworkRecoveryConfig,
|
|
31
|
+
type NetworkRecoveryConfigInput,
|
|
32
|
+
} from "./network_recovery_config.ts";
|
|
23
33
|
import { runGoalReviewSession, type GoalReviewResult, type GoalReviewerRunner } from "./goal_review.ts";
|
|
24
34
|
import {
|
|
25
35
|
runGoalTodoExecutionLongTask,
|
|
@@ -43,6 +53,7 @@ export type GoalLoopProgressPhase =
|
|
|
43
53
|
| "todo_executed"
|
|
44
54
|
| "review_start"
|
|
45
55
|
| "reviewed"
|
|
56
|
+
| "network_wait"
|
|
46
57
|
| "complete";
|
|
47
58
|
|
|
48
59
|
export interface GoalLoopProgressUpdate {
|
|
@@ -70,6 +81,12 @@ export interface GoalLoopProgressUpdate {
|
|
|
70
81
|
reviewerCostTotal: number;
|
|
71
82
|
totalCost: number;
|
|
72
83
|
childProgress?: CoordinatorProgressUpdate;
|
|
84
|
+
networkRecoveryEvent?: NetworkRecoveryEventType;
|
|
85
|
+
networkRetryCount?: number;
|
|
86
|
+
networkOutageElapsedMs?: number;
|
|
87
|
+
networkNextRetryAtMs?: number;
|
|
88
|
+
networkNextRetryInMs?: number;
|
|
89
|
+
networkFailureReason?: string;
|
|
73
90
|
}
|
|
74
91
|
|
|
75
92
|
export type GoalLoopProgressHandler = (update: GoalLoopProgressUpdate) => void;
|
|
@@ -92,10 +109,12 @@ export interface RunGoalLoopOptions extends GoalLoopLimitInput {
|
|
|
92
109
|
thinkingLevel?: string;
|
|
93
110
|
maxBashTimeoutMs?: number;
|
|
94
111
|
maxAttemptsPerTask?: number;
|
|
112
|
+
networkRecovery?: NetworkRecoveryConfigInput;
|
|
95
113
|
commit?: boolean;
|
|
96
114
|
now?: () => Date;
|
|
97
115
|
onWorkerProgress?: (update: CoordinatorProgressUpdate) => void;
|
|
98
116
|
onProgress?: GoalLoopProgressHandler;
|
|
117
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
99
118
|
}
|
|
100
119
|
|
|
101
120
|
export interface GoalLoopRunResult {
|
|
@@ -120,6 +139,7 @@ export class GoalLoopOrchestratorError extends Error {
|
|
|
120
139
|
|
|
121
140
|
export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoopRunResult> {
|
|
122
141
|
const now = options.now ?? (() => new Date());
|
|
142
|
+
const networkRecovery = resolveNetworkRecoveryConfig(options.networkRecovery);
|
|
123
143
|
let state =
|
|
124
144
|
options.initialState ??
|
|
125
145
|
createGoalLoopState({
|
|
@@ -149,7 +169,9 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
149
169
|
await store.saveState(state);
|
|
150
170
|
await store.initializeResultIfMissing(state);
|
|
151
171
|
await store.appendNewTraceEvents(await store.durableTraceLength(), state);
|
|
172
|
+
let progressClosed = false;
|
|
152
173
|
const publish = (phase: GoalLoopProgressPhase, message: string, extra: Partial<GoalLoopProgressUpdate> = {}) => {
|
|
174
|
+
if (progressClosed) return;
|
|
153
175
|
options.onProgress?.({
|
|
154
176
|
message,
|
|
155
177
|
phase,
|
|
@@ -173,6 +195,39 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
173
195
|
...extra,
|
|
174
196
|
});
|
|
175
197
|
};
|
|
198
|
+
const recoveryProgress = (
|
|
199
|
+
restorePhase: GoalLoopProgressPhase,
|
|
200
|
+
restoreMessage: string,
|
|
201
|
+
extra: Partial<GoalLoopProgressUpdate> = {},
|
|
202
|
+
) => {
|
|
203
|
+
let cleaned = false;
|
|
204
|
+
return (event: NetworkRecoveryEvent) => {
|
|
205
|
+
options.onNetworkRecovery?.(event);
|
|
206
|
+
if (cleaned || progressClosed) return;
|
|
207
|
+
if (event.type === "cleanup") {
|
|
208
|
+
cleaned = true;
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
if (event.type === "recovered") {
|
|
212
|
+
publish(restorePhase, restoreMessage, extra);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
if (isTerminalNetworkRecoveryProgressEvent(event.type)) return;
|
|
216
|
+
|
|
217
|
+
const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
|
|
218
|
+
const message = formatNetworkRecoveryStatus(event);
|
|
219
|
+
publish("network_wait", message, {
|
|
220
|
+
...extra,
|
|
221
|
+
networkRecoveryEvent: event.type,
|
|
222
|
+
networkRetryCount: event.state.retryCount,
|
|
223
|
+
networkOutageElapsedMs: event.state.elapsedMs,
|
|
224
|
+
networkNextRetryAtMs: event.state.nextRetryAtMs,
|
|
225
|
+
networkNextRetryInMs:
|
|
226
|
+
event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
|
|
227
|
+
networkFailureReason: event.state.lastFailure.reason,
|
|
228
|
+
});
|
|
229
|
+
};
|
|
230
|
+
};
|
|
176
231
|
|
|
177
232
|
publish("goal_start", `Starting goal loop: ${state.goal}`);
|
|
178
233
|
|
|
@@ -186,6 +241,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
186
241
|
options,
|
|
187
242
|
now,
|
|
188
243
|
publish,
|
|
244
|
+
networkRecovery,
|
|
189
245
|
});
|
|
190
246
|
}
|
|
191
247
|
|
|
@@ -226,8 +282,14 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
226
282
|
modelName: options.modelName,
|
|
227
283
|
thinkingLevel: options.thinkingLevel,
|
|
228
284
|
maxBashTimeoutMs: options.maxBashTimeoutMs,
|
|
285
|
+
networkRecovery,
|
|
229
286
|
now,
|
|
230
287
|
goalSpecification,
|
|
288
|
+
onNetworkRecovery: recoveryProgress(
|
|
289
|
+
"todo_generation_start",
|
|
290
|
+
`Goal iteration ${nextIteration}: generating TODO markdown.`,
|
|
291
|
+
{ iteration: nextIteration },
|
|
292
|
+
),
|
|
231
293
|
});
|
|
232
294
|
generationResults.push(generation);
|
|
233
295
|
state = generation.state;
|
|
@@ -251,8 +313,14 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
251
313
|
modelName: options.modelName,
|
|
252
314
|
thinkingLevel: options.thinkingLevel,
|
|
253
315
|
maxBashTimeoutMs: options.maxBashTimeoutMs,
|
|
316
|
+
networkRecovery,
|
|
254
317
|
now,
|
|
255
318
|
goalSpecification,
|
|
319
|
+
onNetworkRecovery: recoveryProgress(
|
|
320
|
+
"todo_generation_start",
|
|
321
|
+
`Goal iteration ${current.iteration}: resuming TODO generation.`,
|
|
322
|
+
{ iteration: current.iteration },
|
|
323
|
+
),
|
|
256
324
|
});
|
|
257
325
|
generationResults.push(generation);
|
|
258
326
|
state = generation.state;
|
|
@@ -278,6 +346,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
278
346
|
thinkingLevel: options.thinkingLevel,
|
|
279
347
|
maxBashTimeoutMs: options.maxBashTimeoutMs,
|
|
280
348
|
maxAttemptsPerTask: options.maxAttemptsPerTask,
|
|
349
|
+
networkRecovery,
|
|
281
350
|
commit: options.commit,
|
|
282
351
|
now,
|
|
283
352
|
onProgress: (update) => {
|
|
@@ -324,7 +393,13 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
324
393
|
thinkingLevel: options.thinkingLevel,
|
|
325
394
|
now,
|
|
326
395
|
goalSpecification,
|
|
396
|
+
networkRecovery,
|
|
327
397
|
timeoutMs: remainingReviewTimeout(state, current.deadlineAt, now()),
|
|
398
|
+
onNetworkRecovery: recoveryProgress(
|
|
399
|
+
"review_start",
|
|
400
|
+
`Goal iteration ${current.iteration}: reviewing goal completion.`,
|
|
401
|
+
{ iteration: current.iteration },
|
|
402
|
+
),
|
|
328
403
|
});
|
|
329
404
|
reviewResults.push(review);
|
|
330
405
|
state = review.state;
|
|
@@ -365,6 +440,7 @@ export async function runGoalLoop(options: RunGoalLoopOptions): Promise<GoalLoop
|
|
|
365
440
|
}
|
|
366
441
|
|
|
367
442
|
publish("complete", `Goal loop ${state.status}: ${state.completion?.reason ?? "finished"}`);
|
|
443
|
+
progressClosed = true;
|
|
368
444
|
|
|
369
445
|
return {
|
|
370
446
|
state,
|
|
@@ -385,6 +461,7 @@ async function maybeRunGoalDiscovery(options: {
|
|
|
385
461
|
options: RunGoalLoopOptions;
|
|
386
462
|
now: () => Date;
|
|
387
463
|
publish: (phase: GoalLoopProgressPhase, message: string, extra?: Partial<GoalLoopProgressUpdate>) => void;
|
|
464
|
+
networkRecovery: Readonly<NetworkRecoveryConfig>;
|
|
388
465
|
}): Promise<GoalSpecification | undefined> {
|
|
389
466
|
if (options.discoveryDecision.route !== "discovery") {
|
|
390
467
|
return options.existingSpecification;
|
|
@@ -407,6 +484,7 @@ async function maybeRunGoalDiscovery(options: {
|
|
|
407
484
|
model: options.options.model,
|
|
408
485
|
modelName: options.options.modelName,
|
|
409
486
|
thinkingLevel: options.options.thinkingLevel,
|
|
487
|
+
networkRecovery: options.networkRecovery,
|
|
410
488
|
now: options.now,
|
|
411
489
|
});
|
|
412
490
|
await options.store.saveGoalSpecification(spec);
|
|
@@ -467,7 +545,11 @@ function accumulatedWorkerCost(state: GoalLoopState): number {
|
|
|
467
545
|
}
|
|
468
546
|
|
|
469
547
|
function accumulatedReviewerCost(state: GoalLoopState): number {
|
|
470
|
-
return sumFinite(
|
|
548
|
+
return sumFinite(
|
|
549
|
+
state.iterations.map(
|
|
550
|
+
(iteration) => iteration.reviewerResult?.reviewerCostTotal ?? iteration.reviewerRecovery?.reviewerCostTotal,
|
|
551
|
+
),
|
|
552
|
+
);
|
|
471
553
|
}
|
|
472
554
|
|
|
473
555
|
function sumFinite(values: Array<number | undefined>): number {
|
|
@@ -477,6 +559,10 @@ function sumFinite(values: Array<number | undefined>): number {
|
|
|
477
559
|
);
|
|
478
560
|
}
|
|
479
561
|
|
|
562
|
+
function isTerminalNetworkRecoveryProgressEvent(type: NetworkRecoveryEventType): boolean {
|
|
563
|
+
return type === "failed" || type === "cancelled" || type === "outage_expired";
|
|
564
|
+
}
|
|
565
|
+
|
|
480
566
|
function requiredGoal(goal: string | undefined): string {
|
|
481
567
|
const trimmed = goal?.trim();
|
|
482
568
|
if (!trimmed) {
|
package/src/goal_review.ts
CHANGED
|
@@ -3,18 +3,22 @@ import path from "node:path";
|
|
|
3
3
|
|
|
4
4
|
import {
|
|
5
5
|
cancelGoalLoop,
|
|
6
|
+
excludeNetworkOutageFromGoalDeadlines,
|
|
6
7
|
type GoalIterationState,
|
|
7
8
|
type GoalLoopState,
|
|
8
9
|
type GoalReviewerDecision,
|
|
9
10
|
type GoalReviewerResultState,
|
|
11
|
+
recordReviewerRecoveryEvidence,
|
|
10
12
|
recordReviewerResult,
|
|
11
13
|
} from "./goal_loop.ts";
|
|
12
14
|
import { GoalStateStore } from "./goal_state.ts";
|
|
13
15
|
import { runGuardedSessionPrompt } from "./session_guard.ts";
|
|
14
16
|
import { goalSpecificationToMarkdown, type GoalSpecification } from "./goal_spec.ts";
|
|
17
|
+
import { classifyNetworkFailure } from "./network_failure.ts";
|
|
18
|
+
import { recoverNetworkOperation, type NetworkRecoveryEvent } from "./network_recovery.ts";
|
|
19
|
+
import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
|
|
15
20
|
import {
|
|
16
21
|
createIsolatedWorkerSession,
|
|
17
|
-
DEFAULT_WORKER_TOOLS,
|
|
18
22
|
workerUsageCostFromEvent,
|
|
19
23
|
workerUsageCostFromStats,
|
|
20
24
|
workerUsageCostKeyFromEvent,
|
|
@@ -23,6 +27,8 @@ import {
|
|
|
23
27
|
|
|
24
28
|
export const GOAL_REVIEW_PAYLOAD_FILE = "REVIEW_TASK.md";
|
|
25
29
|
export const GOAL_REVIEW_RAW_FILE = "REVIEW_RESULT_RAW.txt";
|
|
30
|
+
/** Reviewer retries are constrained to inspection and read-only verification. */
|
|
31
|
+
export const GOAL_REVIEWER_TOOLS = ["read", "bash", "grep", "find", "ls"] as const;
|
|
26
32
|
|
|
27
33
|
export interface GoalReviewOptions {
|
|
28
34
|
state: GoalLoopState;
|
|
@@ -36,7 +42,9 @@ export interface GoalReviewOptions {
|
|
|
36
42
|
now?: () => Date;
|
|
37
43
|
sessionFactory?: WorkerSessionFactory;
|
|
38
44
|
goalSpecification?: GoalSpecification;
|
|
45
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
39
46
|
timeoutMs?: number;
|
|
47
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
40
48
|
}
|
|
41
49
|
|
|
42
50
|
export interface GoalReviewResult {
|
|
@@ -59,6 +67,7 @@ export interface GoalReviewerRunnerOptions {
|
|
|
59
67
|
modelName?: string;
|
|
60
68
|
thinkingLevel?: string;
|
|
61
69
|
sessionFactory?: WorkerSessionFactory;
|
|
70
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
62
71
|
}
|
|
63
72
|
|
|
64
73
|
export interface GoalReviewerSessionResult {
|
|
@@ -69,6 +78,8 @@ export interface GoalReviewerSessionResult {
|
|
|
69
78
|
timedOut?: boolean;
|
|
70
79
|
aborted?: boolean;
|
|
71
80
|
error?: string;
|
|
81
|
+
/** Untouched provider/transport failure retained for coordinator recovery classification. */
|
|
82
|
+
failure?: unknown;
|
|
72
83
|
}
|
|
73
84
|
|
|
74
85
|
export type GoalReviewerRunner = (options: GoalReviewerRunnerOptions) => Promise<GoalReviewerSessionResult>;
|
|
@@ -91,7 +102,7 @@ export class GoalReviewError extends Error {
|
|
|
91
102
|
export async function runGoalReviewSession(options: GoalReviewOptions): Promise<GoalReviewResult> {
|
|
92
103
|
const now = options.now ?? (() => new Date());
|
|
93
104
|
let state = options.state;
|
|
94
|
-
|
|
105
|
+
let previousTraceLength = state.trace.length;
|
|
95
106
|
const store =
|
|
96
107
|
options.store ?? new GoalStateStore({ cwd: options.cwd, goalRunId: state.goalRunId, goalRunDir: state.goalRunDir });
|
|
97
108
|
|
|
@@ -116,19 +127,78 @@ export async function runGoalReviewSession(options: GoalReviewOptions): Promise<
|
|
|
116
127
|
throw new GoalReviewError("Goal review was aborted before starting.", { state });
|
|
117
128
|
}
|
|
118
129
|
|
|
119
|
-
|
|
130
|
+
const interruptedResults: GoalReviewerSessionResult[] = [];
|
|
131
|
+
const interruptedEvidencePaths: string[] = [];
|
|
132
|
+
let excludedOutageMs = 0;
|
|
133
|
+
const captureOutage = (event: NetworkRecoveryEvent) => {
|
|
134
|
+
if (event.type === "cleanup") {
|
|
135
|
+
excludedOutageMs += event.state.elapsedMs;
|
|
136
|
+
}
|
|
137
|
+
options.onNetworkRecovery?.(event);
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
let sessionResult: GoalReviewerSessionResult | undefined;
|
|
141
|
+
let sessionFailure: unknown;
|
|
120
142
|
try {
|
|
121
|
-
sessionResult = await (
|
|
122
|
-
|
|
123
|
-
|
|
143
|
+
sessionResult = await runReviewerWithNetworkRecovery({
|
|
144
|
+
runner: options.reviewerRunner ?? runGoalReviewerSession,
|
|
145
|
+
runnerOptions: {
|
|
146
|
+
prompt: payload,
|
|
147
|
+
cwd: path.resolve(options.cwd ?? process.cwd()),
|
|
148
|
+
abortSignal: options.abortSignal,
|
|
149
|
+
timeoutMs: options.timeoutMs ?? state.limits.reviewerTimeoutMs,
|
|
150
|
+
model: options.model,
|
|
151
|
+
modelName: options.modelName,
|
|
152
|
+
thinkingLevel: options.thinkingLevel,
|
|
153
|
+
sessionFactory: options.sessionFactory,
|
|
154
|
+
networkRecovery: options.networkRecovery,
|
|
155
|
+
},
|
|
156
|
+
networkRecovery: options.networkRecovery,
|
|
124
157
|
abortSignal: options.abortSignal,
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
158
|
+
now,
|
|
159
|
+
interruptedResults,
|
|
160
|
+
interruptedEvidencePaths,
|
|
161
|
+
iterationDir,
|
|
162
|
+
onInterruption: async () => {
|
|
163
|
+
state = recordReviewerRecoveryEvidence(
|
|
164
|
+
state,
|
|
165
|
+
iteration.iteration,
|
|
166
|
+
{
|
|
167
|
+
interruptions: interruptedResults.length,
|
|
168
|
+
evidencePaths: interruptedEvidencePaths,
|
|
169
|
+
reviewerCostTotal: reviewerCost(interruptedResults),
|
|
170
|
+
},
|
|
171
|
+
{ now: now() },
|
|
172
|
+
);
|
|
173
|
+
await persistStateChange(store, previousTraceLength, state);
|
|
174
|
+
previousTraceLength = state.trace.length;
|
|
175
|
+
},
|
|
176
|
+
onRecoveryEvent: captureOutage,
|
|
130
177
|
});
|
|
131
178
|
} catch (error) {
|
|
179
|
+
sessionFailure = error;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (excludedOutageMs > 0) {
|
|
183
|
+
state = excludeNetworkOutageFromGoalDeadlines(state, excludedOutageMs, "reviewer", { now: now() });
|
|
184
|
+
}
|
|
185
|
+
if (state.trace.length !== previousTraceLength) {
|
|
186
|
+
await persistStateChange(store, previousTraceLength, state);
|
|
187
|
+
previousTraceLength = state.trace.length;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
if (sessionFailure !== undefined && options.abortSignal?.aborted) {
|
|
191
|
+
const reason = `Goal review was aborted during network recovery: ${errorMessage(
|
|
192
|
+
options.abortSignal.reason ?? sessionFailure,
|
|
193
|
+
)}`;
|
|
194
|
+
state = cancelGoalLoop(state, reason, { now: now() });
|
|
195
|
+
await persistStateChange(store, previousTraceLength, state);
|
|
196
|
+
await store.writeIterationSnapshot(currentIteration(state, iteration.iteration));
|
|
197
|
+
throw new GoalReviewError(reason, { cause: sessionFailure, state });
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (sessionFailure !== undefined) {
|
|
201
|
+
const interruptedResult = combineReviewerSessionCosts(interruptedResults);
|
|
132
202
|
const failure = await recordReviewFailure({
|
|
133
203
|
state,
|
|
134
204
|
iteration,
|
|
@@ -136,17 +206,23 @@ export async function runGoalReviewSession(options: GoalReviewOptions): Promise<
|
|
|
136
206
|
previousTraceLength,
|
|
137
207
|
payloadPath,
|
|
138
208
|
rawReviewPath,
|
|
139
|
-
message: `Reviewer session failed: ${errorMessage(
|
|
140
|
-
error,
|
|
209
|
+
message: `Reviewer session failed: ${errorMessage(sessionFailure)}`,
|
|
210
|
+
error: sessionFailure,
|
|
211
|
+
sessionResult: interruptedResult,
|
|
141
212
|
now,
|
|
142
213
|
});
|
|
143
214
|
throw new GoalReviewError(failure.reviewerResult.summary, {
|
|
144
|
-
cause:
|
|
215
|
+
cause: sessionFailure,
|
|
145
216
|
state: failure.state,
|
|
146
217
|
reviewerResult: failure.reviewerResult,
|
|
147
218
|
});
|
|
148
219
|
}
|
|
149
220
|
|
|
221
|
+
sessionResult = combineReviewerSessionCosts(interruptedResults, sessionResult);
|
|
222
|
+
if (!sessionResult) {
|
|
223
|
+
throw new GoalReviewError("Reviewer session ended without a result.", { state });
|
|
224
|
+
}
|
|
225
|
+
|
|
150
226
|
const rawReviewerOutput = sessionResult.assistantText;
|
|
151
227
|
await writeFile(rawReviewPath, rawReviewerOutput, "utf8");
|
|
152
228
|
|
|
@@ -367,7 +443,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
|
|
|
367
443
|
}
|
|
368
444
|
const factoryResult = await sessionFactory({
|
|
369
445
|
cwd: options.cwd,
|
|
370
|
-
tools:
|
|
446
|
+
tools: GOAL_REVIEWER_TOOLS,
|
|
371
447
|
model: options.model,
|
|
372
448
|
modelName: options.modelName,
|
|
373
449
|
thinkingLevel: options.thinkingLevel,
|
|
@@ -414,6 +490,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
|
|
|
414
490
|
timedOut: promptResult.timedOut,
|
|
415
491
|
aborted: promptResult.aborted,
|
|
416
492
|
error: promptResult.error,
|
|
493
|
+
failure: promptResult.failure,
|
|
417
494
|
};
|
|
418
495
|
} catch (exc) {
|
|
419
496
|
return {
|
|
@@ -422,6 +499,7 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
|
|
|
422
499
|
reviewerSessionFile: session?.sessionFile,
|
|
423
500
|
reviewerCostTotal,
|
|
424
501
|
error: errorMessage(exc),
|
|
502
|
+
failure: exc,
|
|
425
503
|
};
|
|
426
504
|
} finally {
|
|
427
505
|
if (session) {
|
|
@@ -434,6 +512,119 @@ export async function runGoalReviewerSession(options: GoalReviewerRunnerOptions)
|
|
|
434
512
|
}
|
|
435
513
|
}
|
|
436
514
|
|
|
515
|
+
async function runReviewerWithNetworkRecovery(options: {
|
|
516
|
+
runner: GoalReviewerRunner;
|
|
517
|
+
runnerOptions: GoalReviewerRunnerOptions;
|
|
518
|
+
networkRecovery: Readonly<NetworkRecoveryConfig> | undefined;
|
|
519
|
+
abortSignal: AbortSignal | undefined;
|
|
520
|
+
now: () => Date;
|
|
521
|
+
interruptedResults: GoalReviewerSessionResult[];
|
|
522
|
+
interruptedEvidencePaths: string[];
|
|
523
|
+
iterationDir: string;
|
|
524
|
+
onInterruption: () => Promise<void>;
|
|
525
|
+
onRecoveryEvent: (event: NetworkRecoveryEvent) => void;
|
|
526
|
+
}): Promise<GoalReviewerSessionResult> {
|
|
527
|
+
const run = async (recoverySignal?: AbortSignal): Promise<GoalReviewerSessionResult> => {
|
|
528
|
+
const result = await options.runner({
|
|
529
|
+
...options.runnerOptions,
|
|
530
|
+
abortSignal: combineAbortSignals(options.abortSignal, recoverySignal),
|
|
531
|
+
});
|
|
532
|
+
const networkFailure = recoverableReviewerFailure(result);
|
|
533
|
+
if (networkFailure === undefined) {
|
|
534
|
+
return result;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
options.interruptedResults.push(result);
|
|
538
|
+
options.interruptedEvidencePaths.push(
|
|
539
|
+
await writeInterruptedReviewEvidence(options.iterationDir, options.interruptedResults.length, result),
|
|
540
|
+
);
|
|
541
|
+
await options.onInterruption();
|
|
542
|
+
throw new ReviewerNetworkFailure(networkFailure, result);
|
|
543
|
+
};
|
|
544
|
+
|
|
545
|
+
try {
|
|
546
|
+
return await run();
|
|
547
|
+
} catch (initialFailure) {
|
|
548
|
+
const classification = classifyNetworkFailure(initialFailure);
|
|
549
|
+
if (!options.networkRecovery?.enabled || !classification.recoverable) {
|
|
550
|
+
throw initialFailure;
|
|
551
|
+
}
|
|
552
|
+
const recovered = await recoverNetworkOperation({
|
|
553
|
+
initialFailure,
|
|
554
|
+
config: options.networkRecovery,
|
|
555
|
+
signal: options.abortSignal,
|
|
556
|
+
now: () => options.now().getTime(),
|
|
557
|
+
onEvent: options.onRecoveryEvent,
|
|
558
|
+
retry: ({ signal }) => run(signal),
|
|
559
|
+
});
|
|
560
|
+
return recovered.value;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
class ReviewerNetworkFailure extends Error {
|
|
565
|
+
readonly sessionResult: GoalReviewerSessionResult;
|
|
566
|
+
|
|
567
|
+
constructor(failure: unknown, sessionResult: GoalReviewerSessionResult) {
|
|
568
|
+
super(sessionResult.error ?? errorMessage(failure), { cause: failure });
|
|
569
|
+
this.name = "ReviewerNetworkFailure";
|
|
570
|
+
this.sessionResult = sessionResult;
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
function recoverableReviewerFailure(result: GoalReviewerSessionResult): unknown | undefined {
|
|
575
|
+
if (result.aborted || result.timedOut || !result.error) {
|
|
576
|
+
return undefined;
|
|
577
|
+
}
|
|
578
|
+
const failure = result.failure ?? new Error(result.error);
|
|
579
|
+
return classifyNetworkFailure(failure).recoverable ? failure : undefined;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
async function writeInterruptedReviewEvidence(
|
|
583
|
+
iterationDir: string,
|
|
584
|
+
interruption: number,
|
|
585
|
+
result: GoalReviewerSessionResult,
|
|
586
|
+
): Promise<string> {
|
|
587
|
+
const outputPath = path.join(
|
|
588
|
+
iterationDir,
|
|
589
|
+
`REVIEW_RESULT_NETWORK_INTERRUPTED_${String(interruption).padStart(2, "0")}.txt`,
|
|
590
|
+
);
|
|
591
|
+
const evidence = result.assistantText.trim()
|
|
592
|
+
? result.assistantText
|
|
593
|
+
: `[No assistant text was captured.]\n\n${result.error ?? "Transient reviewer network failure."}\n`;
|
|
594
|
+
await writeFile(outputPath, evidence, "utf8");
|
|
595
|
+
return outputPath;
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
function reviewerCost(results: readonly GoalReviewerSessionResult[]): number {
|
|
599
|
+
return results.reduce(
|
|
600
|
+
(total, result) =>
|
|
601
|
+
total +
|
|
602
|
+
(typeof result.reviewerCostTotal === "number" && Number.isFinite(result.reviewerCostTotal)
|
|
603
|
+
? result.reviewerCostTotal
|
|
604
|
+
: 0),
|
|
605
|
+
0,
|
|
606
|
+
);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
function combineReviewerSessionCosts(
|
|
610
|
+
interrupted: readonly GoalReviewerSessionResult[],
|
|
611
|
+
final?: GoalReviewerSessionResult,
|
|
612
|
+
): GoalReviewerSessionResult | undefined {
|
|
613
|
+
if (!final && interrupted.length === 0) {
|
|
614
|
+
return undefined;
|
|
615
|
+
}
|
|
616
|
+
const base = final ?? interrupted.at(-1)!;
|
|
617
|
+
const reviewerCostTotal = reviewerCost([...interrupted, ...(final ? [final] : [])]);
|
|
618
|
+
return { ...base, reviewerCostTotal };
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined {
|
|
622
|
+
const available = signals.filter((signal): signal is AbortSignal => Boolean(signal));
|
|
623
|
+
if (available.length === 0) return undefined;
|
|
624
|
+
if (available.length === 1) return available[0];
|
|
625
|
+
return AbortSignal.any(available);
|
|
626
|
+
}
|
|
627
|
+
|
|
437
628
|
function currentReviewableIteration(state: GoalLoopState): GoalIterationState {
|
|
438
629
|
const iteration = currentIteration(state, state.currentIteration);
|
|
439
630
|
if (iteration.status !== "todo_executed" && iteration.status !== "failed") {
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
recordWorkerResult,
|
|
16
16
|
} from "./goal_loop.ts";
|
|
17
17
|
import { GoalStateStore } from "./goal_state.ts";
|
|
18
|
+
import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
|
|
18
19
|
import { validateTodoMarkdown } from "./todo_generator.ts";
|
|
19
20
|
|
|
20
21
|
export const GOAL_TODO_EXECUTION_PROGRESS_FILE = "WORKER_PROGRESS.jsonl";
|
|
@@ -32,6 +33,7 @@ export interface GoalTodoExecutionOptions {
|
|
|
32
33
|
thinkingLevel?: string;
|
|
33
34
|
maxBashTimeoutMs?: number;
|
|
34
35
|
maxAttemptsPerTask?: number;
|
|
36
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
35
37
|
commit?: boolean;
|
|
36
38
|
now?: () => Date;
|
|
37
39
|
onProgress?: (update: CoordinatorProgressUpdate) => void;
|
|
@@ -152,6 +154,7 @@ export async function runGoalTodoExecutionLongTask(
|
|
|
152
154
|
taskTimeoutMs: childTimeoutMs,
|
|
153
155
|
maxBashTimeoutMs: options.maxBashTimeoutMs,
|
|
154
156
|
maxAttemptsPerTask: options.maxAttemptsPerTask,
|
|
157
|
+
networkRecovery: options.networkRecovery,
|
|
155
158
|
onProgress: (update) => {
|
|
156
159
|
progressEvents.push(update);
|
|
157
160
|
options.onProgress?.(update);
|