pi-long-task 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +153 -3
- package/package.json +1 -1
- package/src/coordinator.ts +861 -62
- package/src/goal_discovery.ts +2 -0
- package/src/goal_loop.ts +76 -0
- package/src/goal_orchestrator.ts +98 -1
- package/src/goal_review.ts +206 -15
- package/src/goal_todo_execution.ts +7 -0
- package/src/goal_todo_generation.ts +108 -13
- package/src/index.ts +15 -1
- package/src/network_failure.ts +574 -0
- package/src/network_recovery.ts +395 -0
- package/src/network_recovery_config.ts +89 -0
- package/src/planner_config.ts +214 -0
- package/src/planner_progress.ts +156 -0
- package/src/render.ts +38 -0
- package/src/session_guard.ts +120 -7
- package/src/todo_generator.ts +84 -7
- package/src/types.ts +68 -0
- package/src/worker_capabilities.ts +103 -0
- package/src/worker_config.ts +148 -7
- package/src/worker_session.ts +33 -1
package/src/coordinator.ts
CHANGED
|
@@ -10,6 +10,18 @@ import type {
|
|
|
10
10
|
} from "./types.ts";
|
|
11
11
|
import { commitAfterSession, gitDirtyPaths, shouldCommitOutcome, type CommitAfterSessionResult } from "./git.ts";
|
|
12
12
|
import { formatCoordinatorResultMessage } from "./render.ts";
|
|
13
|
+
import { classifyNetworkFailure } from "./network_failure.ts";
|
|
14
|
+
import {
|
|
15
|
+
formatNetworkRecoveryStatus,
|
|
16
|
+
recoverNetworkOperation,
|
|
17
|
+
type NetworkRecoveryEvent,
|
|
18
|
+
type NetworkRecoveryEventType,
|
|
19
|
+
} from "./network_recovery.ts";
|
|
20
|
+
import {
|
|
21
|
+
DEFAULT_NETWORK_RECOVERY_CONFIG,
|
|
22
|
+
resolveNetworkRecoveryConfig,
|
|
23
|
+
type NetworkRecoveryConfig,
|
|
24
|
+
} from "./network_recovery_config.ts";
|
|
13
25
|
import { extractResultSummary, hasCompleteTaskResult } from "./result_writer.ts";
|
|
14
26
|
import { runGuardedSessionPrompt } from "./session_guard.ts";
|
|
15
27
|
import {
|
|
@@ -20,6 +32,24 @@ import {
|
|
|
20
32
|
type PlanRevisionRelevantResult,
|
|
21
33
|
} from "./plan_revision_generation.ts";
|
|
22
34
|
import { taskSemanticFingerprint, type PlanTaskState } from "./plan_revision.ts";
|
|
35
|
+
import {
|
|
36
|
+
DEFAULT_PLANNER_THINKING_LEVEL,
|
|
37
|
+
DEFAULT_PLANNER_TIMEOUT_MS,
|
|
38
|
+
resolvePlannerBudget,
|
|
39
|
+
resolvePlannerGracefulShutdownMs,
|
|
40
|
+
resolvePlannerTimeoutMs,
|
|
41
|
+
type PlannerBudget,
|
|
42
|
+
} from "./planner_config.ts";
|
|
43
|
+
import {
|
|
44
|
+
createPlannerActiveProgress,
|
|
45
|
+
createPlannerGraceProgress,
|
|
46
|
+
createPlannerStartedProgress,
|
|
47
|
+
formatFriendlyDuration,
|
|
48
|
+
plannerProgressCheckpoints,
|
|
49
|
+
type PlannerProgressEvent,
|
|
50
|
+
type PlannerProgressHandler,
|
|
51
|
+
type PlannerProgressState,
|
|
52
|
+
} from "./planner_progress.ts";
|
|
23
53
|
import {
|
|
24
54
|
PersistentTodoPlanStore,
|
|
25
55
|
planTaskReference,
|
|
@@ -41,6 +71,7 @@ import {
|
|
|
41
71
|
import { buildTaskProgressModel, type TaskProgressModel, type TaskProgressStatus } from "./task_progress.ts";
|
|
42
72
|
import {
|
|
43
73
|
applyGoalInstructionsToTodoMarkdown,
|
|
74
|
+
applyWorkerCapabilityConstraintsToTodoMarkdown,
|
|
44
75
|
buildTodoCreationPrompt,
|
|
45
76
|
buildTodoRepairPrompt,
|
|
46
77
|
extractAndValidateTodoMarkdown,
|
|
@@ -49,6 +80,7 @@ import {
|
|
|
49
80
|
validateTodoMarkdown,
|
|
50
81
|
} from "./todo_generator.ts";
|
|
51
82
|
import { parseTasks, todoGlobalInstructions, type Task } from "./todo_parser.ts";
|
|
83
|
+
import { detectUnavailableWorkerCapabilities, type WorkerCapabilityWarning } from "./worker_capabilities.ts";
|
|
52
84
|
import {
|
|
53
85
|
buildWorkerSessionCreationFailureOutcome,
|
|
54
86
|
createIsolatedWorkerSession,
|
|
@@ -72,36 +104,55 @@ export type { CoordinatorStatus } from "./types.ts";
|
|
|
72
104
|
export const DEFAULT_COORDINATOR_OPTIONS = {
|
|
73
105
|
maxAttemptsPerTask: 3,
|
|
74
106
|
taskTimeoutMs: 900_000,
|
|
75
|
-
todoTimeoutMs:
|
|
107
|
+
todoTimeoutMs: DEFAULT_PLANNER_TIMEOUT_MS,
|
|
76
108
|
todoGracefulShutdownMs: 15_000,
|
|
77
109
|
maxBashTimeoutMs: 300_000,
|
|
78
110
|
taskThinking: "high",
|
|
79
|
-
todoThinking:
|
|
111
|
+
todoThinking: DEFAULT_PLANNER_THINKING_LEVEL,
|
|
80
112
|
workerSessionReuse: DEFAULT_WORKER_SESSION_REUSE_ENABLED,
|
|
81
113
|
workerSessionReuseContextThresholdPercent: DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
|
|
114
|
+
networkRecovery: DEFAULT_NETWORK_RECOVERY_CONFIG,
|
|
82
115
|
} as const;
|
|
83
116
|
|
|
84
117
|
export type WorkerRunner = (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
|
|
85
118
|
export type CoordinatorProgressPhase =
|
|
119
|
+
| "capability_warning"
|
|
86
120
|
| "planning"
|
|
87
121
|
| "planned"
|
|
88
122
|
| "task_start"
|
|
89
123
|
| "worker_session"
|
|
90
124
|
| "worker_tool"
|
|
125
|
+
| "network_wait"
|
|
91
126
|
| "task_done"
|
|
92
127
|
| "task_blocked"
|
|
93
128
|
| "task_failed"
|
|
94
129
|
| "task_obsolete"
|
|
95
130
|
| "complete";
|
|
96
131
|
|
|
97
|
-
export type PlannerDiagnosticKind =
|
|
132
|
+
export type PlannerDiagnosticKind =
|
|
133
|
+
| "timeout"
|
|
134
|
+
| "cancelled"
|
|
135
|
+
/** @deprecated Planner cancellation is now reported as `cancelled`. */
|
|
136
|
+
| "abort"
|
|
137
|
+
| "network_recovery"
|
|
138
|
+
| "network_failure"
|
|
139
|
+
| "invalid_output"
|
|
140
|
+
| "repair_attempt"
|
|
141
|
+
| "failure";
|
|
98
142
|
|
|
99
143
|
export interface PlannerDiagnostic {
|
|
100
144
|
kind: PlannerDiagnosticKind;
|
|
101
145
|
message: string;
|
|
146
|
+
/** Present whenever output presence is known; partial content itself is deliberately omitted. */
|
|
147
|
+
partialOutputObserved?: boolean;
|
|
102
148
|
diagnostics?: string[];
|
|
103
149
|
sessionFile?: string;
|
|
104
150
|
sessionId?: string;
|
|
151
|
+
/** Network lifecycle data is separate from timeout/cancellation classification. */
|
|
152
|
+
networkRecoveryEvent?: NetworkRecoveryEventType;
|
|
153
|
+
networkFailureReason?: string;
|
|
154
|
+
networkRetryCount?: number;
|
|
155
|
+
networkOutageElapsedMs?: number;
|
|
105
156
|
}
|
|
106
157
|
|
|
107
158
|
export type PlannerDiagnosticHandler = (diagnostic: PlannerDiagnostic) => void;
|
|
@@ -144,12 +195,32 @@ export interface CoordinatorProgressUpdate {
|
|
|
144
195
|
taskProgress?: TaskProgressModel;
|
|
145
196
|
plannerDiagnostic?: PlannerDiagnosticKind;
|
|
146
197
|
plannerDiagnostics?: string[];
|
|
198
|
+
plannerPartialOutputObserved?: boolean;
|
|
147
199
|
plannerSessionFile?: string;
|
|
148
200
|
plannerSessionId?: string;
|
|
201
|
+
/** Deterministic deadline selection used by planner calls in this run. */
|
|
202
|
+
plannerBudget?: Readonly<PlannerBudget>;
|
|
203
|
+
/** Human-facing state with exact millisecond values retained for integrations. */
|
|
204
|
+
plannerProgressState?: PlannerProgressState;
|
|
205
|
+
plannerElapsedMs?: number;
|
|
206
|
+
plannerRemainingMs?: number;
|
|
207
|
+
plannerGracePeriodMs?: number;
|
|
208
|
+
plannerGraceRemainingMs?: number;
|
|
209
|
+
capabilityWarning?: Readonly<WorkerCapabilityWarning>;
|
|
149
210
|
workerSessionEvent?: WorkerSessionDiagnostic["event"];
|
|
150
211
|
workerSessionReason?: string;
|
|
151
212
|
workerSessionContextUsagePercent?: number;
|
|
152
213
|
workerSessionContextThresholdPercent?: number;
|
|
214
|
+
networkRecoveryEvent?: NetworkRecoveryEventType;
|
|
215
|
+
networkRetryCount?: number;
|
|
216
|
+
networkOutageElapsedMs?: number;
|
|
217
|
+
networkNextRetryAtMs?: number;
|
|
218
|
+
networkNextRetryInMs?: number;
|
|
219
|
+
networkFailureReason?: string;
|
|
220
|
+
/** Identifies whether recovery belongs to planning or worker execution. */
|
|
221
|
+
networkOperation?: "planner" | "worker";
|
|
222
|
+
/** Planner recovery never mutates the configured per-attempt deadline. */
|
|
223
|
+
plannerDeadlinePolicy?: "per_attempt_excludes_network_wait";
|
|
153
224
|
}
|
|
154
225
|
|
|
155
226
|
export type CoordinatorProgressHandler = (update: CoordinatorProgressUpdate) => void;
|
|
@@ -182,24 +253,39 @@ export interface RunCoordinatorOptions extends PiLongTaskInput {
|
|
|
182
253
|
steeringQueue?: SerializedSteeringQueue;
|
|
183
254
|
/** Runs after rebase/validation and immediately before the revision is atomically persisted. */
|
|
184
255
|
onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
|
|
256
|
+
/** Receives coordinator-level outage lifecycle events for parent orchestrators and status integrations. */
|
|
257
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
185
258
|
}
|
|
186
259
|
|
|
187
260
|
export interface TodoPlannerOptions {
|
|
188
261
|
inputText: string;
|
|
189
262
|
cwd: string;
|
|
190
263
|
runDir: string;
|
|
191
|
-
|
|
264
|
+
/** Defaults to the planner-only balanced level; explicit values are forwarded unchanged. */
|
|
265
|
+
thinkingLevel?: string;
|
|
192
266
|
model?: unknown;
|
|
193
267
|
abortSignal?: AbortSignal;
|
|
194
268
|
timeoutMs?: number;
|
|
195
269
|
gracefulShutdownMs?: number;
|
|
270
|
+
/** Structured record of explicit/default/adaptive deadline selection. */
|
|
271
|
+
plannerBudget?: Readonly<PlannerBudget>;
|
|
196
272
|
sessionFactory?: WorkerSessionFactory;
|
|
197
273
|
onDiagnostic?: PlannerDiagnosticHandler;
|
|
274
|
+
/** Shared human-readable timing events for CLI, TUI, and headless integrations. */
|
|
275
|
+
onProgress?: PlannerProgressHandler;
|
|
198
276
|
goal?: string;
|
|
199
277
|
/** Exact prompt for revision planners; bypasses the initial TODO-creation wrapper. */
|
|
200
278
|
plannerPrompt?: string;
|
|
201
279
|
/** Structured revision context supplied alongside plannerPrompt. */
|
|
202
280
|
planRevision?: Readonly<PlanRevisionRequest>;
|
|
281
|
+
/**
|
|
282
|
+
* Normalized coordinator recovery policy. Recovery wait is accounted on its
|
|
283
|
+
* own outage clock; every replay receives the same configured per-attempt
|
|
284
|
+
* planner deadline, so recovery settings never replace that deadline.
|
|
285
|
+
*/
|
|
286
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
287
|
+
/** Run-level constraints derived from capabilities unavailable to isolated workers. */
|
|
288
|
+
capabilityConstraints?: readonly string[];
|
|
203
289
|
}
|
|
204
290
|
|
|
205
291
|
export interface TaskAttemptSummary {
|
|
@@ -252,6 +338,10 @@ export interface CoordinatorResult {
|
|
|
252
338
|
workerUsageTotal?: WorkerUsageTotals;
|
|
253
339
|
/** Additive lifecycle counters for adaptive worker-session reuse. */
|
|
254
340
|
workerSessionMetrics?: WorkerSessionMetrics;
|
|
341
|
+
/** Deterministic deadline selection used by planner calls in this run. */
|
|
342
|
+
plannerBudget?: Readonly<PlannerBudget>;
|
|
343
|
+
/** Explicit, non-fatal warnings for requested capabilities unavailable to isolated workers. */
|
|
344
|
+
capabilityWarnings?: readonly WorkerCapabilityWarning[];
|
|
255
345
|
commit: boolean;
|
|
256
346
|
goal?: string;
|
|
257
347
|
error?: string;
|
|
@@ -280,8 +370,10 @@ interface RuntimeOptions {
|
|
|
280
370
|
todoThinking: string;
|
|
281
371
|
workerSessionReuse: boolean;
|
|
282
372
|
workerSessionReuseContextThresholdPercent: number;
|
|
373
|
+
networkRecovery: NetworkRecoveryConfig;
|
|
283
374
|
todoTimeoutMs: number;
|
|
284
375
|
todoGracefulShutdownMs: number;
|
|
376
|
+
plannerBudget: PlannerBudget;
|
|
285
377
|
workerRunner: WorkerRunner;
|
|
286
378
|
useRetainedWorkerLifecycle: boolean;
|
|
287
379
|
todoPlanner: TodoPlanner;
|
|
@@ -295,9 +387,20 @@ interface RuntimeOptions {
|
|
|
295
387
|
workerTextByWorker: Map<string, string>;
|
|
296
388
|
workerTextPublishedLengthByWorker: Map<string, number>;
|
|
297
389
|
plannerDiagnostics: PlannerDiagnostic[];
|
|
390
|
+
capabilityWarnings: WorkerCapabilityWarning[];
|
|
298
391
|
workerSessionMetrics: WorkerSessionMetrics;
|
|
299
392
|
steeringQueue?: SerializedSteeringQueue;
|
|
300
393
|
onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
|
|
394
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
395
|
+
lastProgress?: CoordinatorProgressUpdate;
|
|
396
|
+
progressClosed: boolean;
|
|
397
|
+
networkRecoverySequence: number;
|
|
398
|
+
activeNetworkRecoveries: Map<number, ActiveNetworkRecovery>;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
interface ActiveNetworkRecovery {
|
|
402
|
+
event: NetworkRecoveryEvent;
|
|
403
|
+
operation: "planner" | "worker";
|
|
301
404
|
}
|
|
302
405
|
|
|
303
406
|
type RetainedWorkerReuseScope = "sequential_task" | "partial_continuation";
|
|
@@ -733,6 +836,176 @@ function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortS
|
|
|
733
836
|
return AbortSignal.any(available);
|
|
734
837
|
}
|
|
735
838
|
|
|
839
|
+
/** Retains the full worker result while exposing its provider error to the shared classifier. */
|
|
840
|
+
class WorkerNetworkFailure extends Error {
|
|
841
|
+
readonly outcome: SessionOutcome;
|
|
842
|
+
|
|
843
|
+
constructor(outcome: SessionOutcome) {
|
|
844
|
+
const message = outcome.error ?? "worker network operation failed";
|
|
845
|
+
super(message, { cause: workerFailureValue(outcome) });
|
|
846
|
+
this.name = "WorkerNetworkFailure";
|
|
847
|
+
this.outcome = outcome;
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
interface WorkerRecoveryExecutionOptions {
|
|
852
|
+
workerOptions: RunWorkerTaskOptions;
|
|
853
|
+
run: (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
|
|
854
|
+
taskResultPath: string;
|
|
855
|
+
networkRecovery: Readonly<NetworkRecoveryConfig>;
|
|
856
|
+
signal?: AbortSignal;
|
|
857
|
+
onInterruption?: (outcome: SessionOutcome) => void;
|
|
858
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
/**
|
|
862
|
+
* Run one ordinary worker attempt, replacing only transport-failed sessions.
|
|
863
|
+
* Pi owns bounded request retries inside session.prompt(); therefore an outcome
|
|
864
|
+
* reaches this boundary only after those retries have settled. Coordinator
|
|
865
|
+
* probes retain the same task/attempt and always receive a recovery prompt.
|
|
866
|
+
*/
|
|
867
|
+
async function runWorkerAttemptWithNetworkRecovery(options: WorkerRecoveryExecutionOptions): Promise<SessionOutcome> {
|
|
868
|
+
const interrupted: SessionOutcome[] = [];
|
|
869
|
+
const thrownErrors = new Map<SessionOutcome, unknown>();
|
|
870
|
+
const execute = async (workerOptions: RunWorkerTaskOptions): Promise<SessionOutcome> => {
|
|
871
|
+
try {
|
|
872
|
+
return await options.run(workerOptions);
|
|
873
|
+
} catch (error) {
|
|
874
|
+
const outcome = buildWorkerSessionCreationFailureOutcome(workerOptions, error);
|
|
875
|
+
thrownErrors.set(outcome, error);
|
|
876
|
+
return outcome;
|
|
877
|
+
}
|
|
878
|
+
};
|
|
879
|
+
const recordRecoverableInterruption = async (outcome: SessionOutcome): Promise<void> => {
|
|
880
|
+
await appendNetworkInterruptionEvidence(options.taskResultPath, outcome, interrupted.length + 1);
|
|
881
|
+
interrupted.push(outcome);
|
|
882
|
+
options.onInterruption?.(outcome);
|
|
883
|
+
};
|
|
884
|
+
|
|
885
|
+
const initial = await execute(options.workerOptions);
|
|
886
|
+
const initialFailure = workerFailureValue(initial);
|
|
887
|
+
const initialClassification = initialFailure === undefined ? undefined : classifyNetworkFailure(initialFailure);
|
|
888
|
+
if (initialClassification && isFailFastWorkerFailure(initialClassification.reason)) {
|
|
889
|
+
throw new WorkerNetworkFailure(initial);
|
|
890
|
+
}
|
|
891
|
+
if (!options.networkRecovery.enabled || !initialClassification?.recoverable) {
|
|
892
|
+
if (thrownErrors.has(initial)) throw thrownErrors.get(initial);
|
|
893
|
+
return initial;
|
|
894
|
+
}
|
|
895
|
+
await recordRecoverableInterruption(initial);
|
|
896
|
+
|
|
897
|
+
try {
|
|
898
|
+
const recovered = await recoverNetworkOperation({
|
|
899
|
+
initialFailure: new WorkerNetworkFailure(initial),
|
|
900
|
+
config: options.networkRecovery,
|
|
901
|
+
signal: options.signal,
|
|
902
|
+
onEvent: options.onNetworkRecovery,
|
|
903
|
+
retry: async ({ retryCount, signal }) => {
|
|
904
|
+
const previous = interrupted.at(-1)!;
|
|
905
|
+
const resumed = await execute({
|
|
906
|
+
...options.workerOptions,
|
|
907
|
+
// The recovery signal includes both run cancellation and the outage
|
|
908
|
+
// deadline without replacing the assignment/steering cancellation.
|
|
909
|
+
abortSignal: combineAbortSignals(options.workerOptions.abortSignal, signal),
|
|
910
|
+
networkRecoveryContext: {
|
|
911
|
+
retryCount,
|
|
912
|
+
durableEvidencePath: options.taskResultPath,
|
|
913
|
+
priorSessionId: previous.sessionId,
|
|
914
|
+
failure: previous.error ?? "transient provider or transport failure",
|
|
915
|
+
},
|
|
916
|
+
});
|
|
917
|
+
const resumedFailure = workerFailureValue(resumed);
|
|
918
|
+
const classification = resumedFailure === undefined ? undefined : classifyNetworkFailure(resumedFailure);
|
|
919
|
+
if (classification?.recoverable) {
|
|
920
|
+
await recordRecoverableInterruption(resumed);
|
|
921
|
+
}
|
|
922
|
+
if (resumed.error) {
|
|
923
|
+
if (
|
|
924
|
+
thrownErrors.has(resumed) &&
|
|
925
|
+
!classification?.recoverable &&
|
|
926
|
+
!isFailFastWorkerFailure(classification!.reason)
|
|
927
|
+
) {
|
|
928
|
+
throw thrownErrors.get(resumed);
|
|
929
|
+
}
|
|
930
|
+
throw new WorkerNetworkFailure(resumed);
|
|
931
|
+
}
|
|
932
|
+
return resumed;
|
|
933
|
+
},
|
|
934
|
+
});
|
|
935
|
+
return mergeWorkerRecoveryOutcomes(interrupted, recovered.value);
|
|
936
|
+
} catch (error) {
|
|
937
|
+
// If connectivity recovered but the fresh session failed deterministically,
|
|
938
|
+
// hand its outcome back to the ordinary worker failure path immediately.
|
|
939
|
+
if (error instanceof WorkerNetworkFailure) {
|
|
940
|
+
const merged = mergeWorkerRecoveryOutcomes(interrupted, error.outcome);
|
|
941
|
+
const mergedFailure = workerFailureValue(merged);
|
|
942
|
+
const classification = mergedFailure === undefined ? undefined : classifyNetworkFailure(mergedFailure);
|
|
943
|
+
if (classification && isFailFastWorkerFailure(classification.reason)) {
|
|
944
|
+
throw new WorkerNetworkFailure(merged);
|
|
945
|
+
}
|
|
946
|
+
return merged;
|
|
947
|
+
}
|
|
948
|
+
throw error;
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
function workerFailureValue(outcome: SessionOutcome): unknown {
|
|
953
|
+
return outcome.failure ?? outcome.error;
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
function isFailFastWorkerFailure(reason: ReturnType<typeof classifyNetworkFailure>["reason"]): boolean {
|
|
957
|
+
return [
|
|
958
|
+
"authentication",
|
|
959
|
+
"authorization",
|
|
960
|
+
"billing",
|
|
961
|
+
"quota_exhausted",
|
|
962
|
+
"invalid_model",
|
|
963
|
+
"invalid_request",
|
|
964
|
+
"http_client_error",
|
|
965
|
+
"non_retryable_server_error",
|
|
966
|
+
].includes(reason);
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
function mergeWorkerRecoveryOutcomes(interrupted: readonly SessionOutcome[], final: SessionOutcome): SessionOutcome {
|
|
970
|
+
if (interrupted.length === 0) return final;
|
|
971
|
+
const usage = addWorkerUsage([...interrupted.map((item) => item.workerUsage), final.workerUsage]);
|
|
972
|
+
return {
|
|
973
|
+
...final,
|
|
974
|
+
startedAt: interrupted[0].startedAt,
|
|
975
|
+
contextObservations: [
|
|
976
|
+
...interrupted.flatMap((item, index) => [
|
|
977
|
+
`network interruption ${index + 1}: ${item.error ?? "transient provider or transport failure"}`,
|
|
978
|
+
...item.contextObservations,
|
|
979
|
+
]),
|
|
980
|
+
...final.contextObservations,
|
|
981
|
+
],
|
|
982
|
+
compactionEvents: [...interrupted.flatMap((item) => item.compactionEvents), ...final.compactionEvents],
|
|
983
|
+
events: [...interrupted.flatMap((item) => item.events), ...final.events],
|
|
984
|
+
workerCostTotal: [...interrupted, final].reduce((total, item) => total + item.workerCostTotal, 0),
|
|
985
|
+
workerCostSource: "network_recovery_aggregate",
|
|
986
|
+
workerUsage: usage,
|
|
987
|
+
sessionDiagnostics: [
|
|
988
|
+
...interrupted.flatMap((item) => item.sessionDiagnostics ?? []),
|
|
989
|
+
...(final.sessionDiagnostics ?? []),
|
|
990
|
+
],
|
|
991
|
+
};
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
function addWorkerUsage(values: Array<WorkerUsageTotals | undefined>): WorkerUsageTotals | undefined {
|
|
995
|
+
const available = values.filter((value): value is WorkerUsageTotals => Boolean(value));
|
|
996
|
+
if (available.length === 0) return undefined;
|
|
997
|
+
return available.reduce<WorkerUsageTotals>(
|
|
998
|
+
(total, value) => ({
|
|
999
|
+
input: total.input + value.input,
|
|
1000
|
+
output: total.output + value.output,
|
|
1001
|
+
cacheRead: total.cacheRead + value.cacheRead,
|
|
1002
|
+
cacheWrite: total.cacheWrite + value.cacheWrite,
|
|
1003
|
+
total: total.total + value.total,
|
|
1004
|
+
}),
|
|
1005
|
+
{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
1006
|
+
);
|
|
1007
|
+
}
|
|
1008
|
+
|
|
736
1009
|
export function workerSessionHealthForOutcome(
|
|
737
1010
|
outcome: Pick<SessionOutcome, "timedOut" | "aborted" | "error" | "assistantText">,
|
|
738
1011
|
cancelled = false,
|
|
@@ -756,7 +1029,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
756
1029
|
const commits: CoordinatorCommitSummary[] = [];
|
|
757
1030
|
|
|
758
1031
|
await mkdir(runtime.runDir, { recursive: true });
|
|
759
|
-
await writeFile(runtime.taskResultPath, initialTaskResultMarkdown(runtime.runId), "utf8");
|
|
1032
|
+
await writeFile(runtime.taskResultPath, initialTaskResultMarkdown(runtime.runId, runtime.capabilityWarnings), "utf8");
|
|
760
1033
|
let planningComplete = false;
|
|
761
1034
|
let latestTodoMarkdown: string | undefined;
|
|
762
1035
|
let latestTasks: Task[] = [];
|
|
@@ -772,7 +1045,14 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
772
1045
|
const protectedDirtyPathsByTask = new Map<string, Set<string>>();
|
|
773
1046
|
|
|
774
1047
|
try {
|
|
775
|
-
|
|
1048
|
+
for (const warning of runtime.capabilityWarnings) {
|
|
1049
|
+
emitProgress(runtime, warning.message, {
|
|
1050
|
+
phase: "capability_warning",
|
|
1051
|
+
status: "warning",
|
|
1052
|
+
capabilityWarning: warning,
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
emitPlannerProgress(runtime, createPlannerStartedProgress(runtime.plannerBudget, runtime.todoGracefulShutdownMs));
|
|
776
1056
|
let todoMarkdown = await generateOrNormalizeTodoMarkdown(inputText, runtime);
|
|
777
1057
|
validateTodoMarkdown(todoMarkdown);
|
|
778
1058
|
planningComplete = true;
|
|
@@ -949,6 +1229,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
949
1229
|
thinkingLevel: runtime.taskThinking,
|
|
950
1230
|
abortSignal: combineAbortSignals(runtime.abortSignal, assignmentController.signal),
|
|
951
1231
|
sessionFactory: runtime.workerSessionFactory,
|
|
1232
|
+
networkRecovery: runtime.networkRecovery,
|
|
952
1233
|
now: runtime.now,
|
|
953
1234
|
onEvent: (event) => {
|
|
954
1235
|
if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
|
|
@@ -966,18 +1247,39 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
966
1247
|
},
|
|
967
1248
|
};
|
|
968
1249
|
let outcome: SessionOutcome;
|
|
1250
|
+
const networkInterruptedOutcomes: SessionOutcome[] = [];
|
|
969
1251
|
try {
|
|
970
|
-
outcome =
|
|
971
|
-
|
|
972
|
-
:
|
|
1252
|
+
outcome = await runWorkerAttemptWithNetworkRecovery({
|
|
1253
|
+
workerOptions,
|
|
1254
|
+
run: (resumedOptions) =>
|
|
1255
|
+
workerSessionOwner
|
|
1256
|
+
? workerSessionOwner.run(resumedOptions, assignmentIdentity)
|
|
1257
|
+
: runtime.workerRunner(resumedOptions),
|
|
1258
|
+
taskResultPath: runtime.taskResultPath,
|
|
1259
|
+
networkRecovery: runtime.networkRecovery,
|
|
1260
|
+
signal: workerOptions.abortSignal,
|
|
1261
|
+
onInterruption: (interrupted) => networkInterruptedOutcomes.push(interrupted),
|
|
1262
|
+
onNetworkRecovery: createNetworkRecoveryProgressHandler(runtime),
|
|
1263
|
+
});
|
|
973
1264
|
} catch (error) {
|
|
974
1265
|
if (!assignmentState.obsolete) {
|
|
1266
|
+
const terminalOutcome = error instanceof WorkerNetworkFailure ? error.outcome : undefined;
|
|
1267
|
+
if (terminalOutcome || networkInterruptedOutcomes.length > 0) {
|
|
1268
|
+
finalizeWorkerCost(runtime.workerCostState, accountingWorker, {
|
|
1269
|
+
workerCostTotal:
|
|
1270
|
+
terminalOutcome?.workerCostTotal ??
|
|
1271
|
+
networkInterruptedOutcomes.reduce((total, interrupted) => total + interrupted.workerCostTotal, 0),
|
|
1272
|
+
});
|
|
1273
|
+
}
|
|
975
1274
|
throw error;
|
|
976
1275
|
}
|
|
977
1276
|
// A cancellation-aware custom runner may reject instead of returning
|
|
978
1277
|
// an aborted outcome. Preserve historical evidence, but never let that
|
|
979
1278
|
// obsolete rejection terminate or update the replacement assignment.
|
|
980
|
-
outcome =
|
|
1279
|
+
outcome = mergeWorkerRecoveryOutcomes(
|
|
1280
|
+
networkInterruptedOutcomes,
|
|
1281
|
+
buildWorkerSessionCreationFailureOutcome(workerOptions, error),
|
|
1282
|
+
);
|
|
981
1283
|
outcome.aborted = true;
|
|
982
1284
|
}
|
|
983
1285
|
finalizeWorkerCost(runtime.workerCostState, accountingWorker, outcome);
|
|
@@ -1152,6 +1454,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
1152
1454
|
workerCostTotal: runtime.workerCostState.total,
|
|
1153
1455
|
workerUsageTotal: aggregateWorkerUsage(outcomes),
|
|
1154
1456
|
workerSessionMetrics: snapshotWorkerSessionMetrics(runtime.workerSessionMetrics),
|
|
1457
|
+
plannerBudget: runtime.plannerBudget,
|
|
1458
|
+
capabilityWarnings: runtime.capabilityWarnings,
|
|
1155
1459
|
commit: options.commit,
|
|
1156
1460
|
goal: runtime.goal,
|
|
1157
1461
|
error: failure,
|
|
@@ -1166,10 +1470,12 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
1166
1470
|
return result;
|
|
1167
1471
|
} catch (error) {
|
|
1168
1472
|
const message = errorMessage(error);
|
|
1169
|
-
if (!planningComplete) {
|
|
1473
|
+
if (!planningComplete && !hasTerminalPlannerDiagnostic(runtime.plannerDiagnostics)) {
|
|
1170
1474
|
recordPlannerDiagnostic(runtime, {
|
|
1171
|
-
kind: "failure",
|
|
1172
|
-
message:
|
|
1475
|
+
kind: runtime.abortSignal?.aborted ? "cancelled" : "failure",
|
|
1476
|
+
message: runtime.abortSignal?.aborted
|
|
1477
|
+
? `TODO planning cancelled: ${abortSignalReason(runtime.abortSignal)}`
|
|
1478
|
+
: `TODO planning failed: ${message}`,
|
|
1173
1479
|
});
|
|
1174
1480
|
}
|
|
1175
1481
|
const resultError = !planningComplete
|
|
@@ -1235,6 +1541,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
1235
1541
|
workerCostTotal: runtime.workerCostState.total,
|
|
1236
1542
|
workerUsageTotal: aggregateWorkerUsage(outcomes),
|
|
1237
1543
|
workerSessionMetrics: snapshotWorkerSessionMetrics(runtime.workerSessionMetrics),
|
|
1544
|
+
plannerBudget: runtime.plannerBudget,
|
|
1545
|
+
capabilityWarnings: runtime.capabilityWarnings,
|
|
1238
1546
|
commit: options.commit,
|
|
1239
1547
|
goal: runtime.goal,
|
|
1240
1548
|
error: resultError,
|
|
@@ -1248,15 +1556,18 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
1248
1556
|
});
|
|
1249
1557
|
return result;
|
|
1250
1558
|
} finally {
|
|
1559
|
+
runtime.progressClosed = true;
|
|
1560
|
+
runtime.activeNetworkRecoveries.clear();
|
|
1251
1561
|
removeSteeringProcessor?.();
|
|
1252
1562
|
await workerSessionOwner?.dispose();
|
|
1253
1563
|
}
|
|
1254
1564
|
}
|
|
1255
1565
|
|
|
1256
1566
|
async function generateOrNormalizeTodoMarkdown(inputText: string, runtime: RuntimeOptions): Promise<string> {
|
|
1567
|
+
const capabilityConstraints = runtime.capabilityWarnings.map((warning) => warning.planningConstraint);
|
|
1257
1568
|
const local = todoMarkdownFromString(inputText, runtime.goal);
|
|
1258
1569
|
if (local) {
|
|
1259
|
-
return local;
|
|
1570
|
+
return applyWorkerCapabilityConstraintsToTodoMarkdown(local, capabilityConstraints);
|
|
1260
1571
|
}
|
|
1261
1572
|
|
|
1262
1573
|
const plannerText = await requestTodoPlan(inputText, runtime);
|
|
@@ -1265,6 +1576,7 @@ async function generateOrNormalizeTodoMarkdown(inputText: string, runtime: Runti
|
|
|
1265
1576
|
plannerText,
|
|
1266
1577
|
(repairPrompt) => requestTodoPlan(repairPrompt, runtime),
|
|
1267
1578
|
runtime.goal,
|
|
1579
|
+
capabilityConstraints,
|
|
1268
1580
|
{
|
|
1269
1581
|
onInvalidOutput: (validationError) =>
|
|
1270
1582
|
recordPlannerDiagnostic(runtime, {
|
|
@@ -1283,7 +1595,8 @@ async function generateOrNormalizeTodoMarkdown(inputText: string, runtime: Runti
|
|
|
1283
1595
|
}),
|
|
1284
1596
|
},
|
|
1285
1597
|
);
|
|
1286
|
-
|
|
1598
|
+
const withGoal = applyGoalInstructionsToTodoMarkdown(planned, runtime.goal);
|
|
1599
|
+
return applyWorkerCapabilityConstraintsToTodoMarkdown(withGoal, capabilityConstraints);
|
|
1287
1600
|
}
|
|
1288
1601
|
|
|
1289
1602
|
interface TodoExtractionRepairHooks {
|
|
@@ -1297,6 +1610,7 @@ async function extractTodoMarkdownWithOneRepair(
|
|
|
1297
1610
|
plannerText: string,
|
|
1298
1611
|
requestRepair: (repairPrompt: string) => Promise<string>,
|
|
1299
1612
|
goal?: string,
|
|
1613
|
+
capabilityConstraints: readonly string[] = [],
|
|
1300
1614
|
hooks: TodoExtractionRepairHooks = {},
|
|
1301
1615
|
): Promise<string> {
|
|
1302
1616
|
try {
|
|
@@ -1305,7 +1619,9 @@ async function extractTodoMarkdownWithOneRepair(
|
|
|
1305
1619
|
const validationError = errorMessage(error);
|
|
1306
1620
|
hooks.onInvalidOutput?.(validationError);
|
|
1307
1621
|
hooks.onRepairAttempt?.(validationError);
|
|
1308
|
-
const repairText = await requestRepair(
|
|
1622
|
+
const repairText = await requestRepair(
|
|
1623
|
+
buildTodoRepairPrompt(inputText, plannerText, validationError, goal, capabilityConstraints),
|
|
1624
|
+
);
|
|
1309
1625
|
try {
|
|
1310
1626
|
return extractAndValidateTodoMarkdown(repairText);
|
|
1311
1627
|
} catch (repairError) {
|
|
@@ -1319,19 +1635,126 @@ async function extractTodoMarkdownWithOneRepair(
|
|
|
1319
1635
|
}
|
|
1320
1636
|
|
|
1321
1637
|
async function requestTodoPlan(inputText: string, runtime: RuntimeOptions): Promise<string> {
|
|
1322
|
-
return
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1638
|
+
return runPlannerOperationWithNetworkRecovery(
|
|
1639
|
+
{
|
|
1640
|
+
inputText,
|
|
1641
|
+
cwd: runtime.cwd,
|
|
1642
|
+
runDir: runtime.runDir,
|
|
1643
|
+
thinkingLevel: runtime.todoThinking,
|
|
1644
|
+
model: runtime.workerModel,
|
|
1645
|
+
abortSignal: runtime.abortSignal,
|
|
1646
|
+
timeoutMs: runtime.todoTimeoutMs,
|
|
1647
|
+
gracefulShutdownMs: runtime.todoGracefulShutdownMs,
|
|
1648
|
+
plannerBudget: runtime.plannerBudget,
|
|
1649
|
+
sessionFactory: runtime.todoSessionFactory,
|
|
1650
|
+
networkRecovery: runtime.networkRecovery,
|
|
1651
|
+
capabilityConstraints: runtime.capabilityWarnings.map((warning) => warning.planningConstraint),
|
|
1652
|
+
onDiagnostic: (diagnostic) => recordPlannerDiagnostic(runtime, diagnostic),
|
|
1653
|
+
onProgress: (event) => {
|
|
1654
|
+
if (event.state !== "started") {
|
|
1655
|
+
emitPlannerProgress(runtime, event);
|
|
1656
|
+
}
|
|
1657
|
+
},
|
|
1658
|
+
goal: runtime.goal,
|
|
1659
|
+
},
|
|
1660
|
+
runtime,
|
|
1661
|
+
);
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
/**
|
|
1665
|
+
* Retry a side-effect-free planner request only after its provider boundary has
|
|
1666
|
+
* failed. The default planner disables tools and disposes every session before
|
|
1667
|
+
* rejecting, so each retry rotates unsafe conversation state while replaying
|
|
1668
|
+
* only the complete immutable planning context. Recovery owns no planner
|
|
1669
|
+
* repair/attempt counter, and each fresh call retains the exact configured
|
|
1670
|
+
* per-attempt planner timeout. Network wait is intentionally excluded and is
|
|
1671
|
+
* governed solely by the separate outage deadline; recovery cannot reset,
|
|
1672
|
+
* extend, or replace the timeout attached to any individual planner call.
|
|
1673
|
+
*/
|
|
1674
|
+
async function runPlannerOperationWithNetworkRecovery(
|
|
1675
|
+
plannerOptions: TodoPlannerOptions,
|
|
1676
|
+
runtime: RuntimeOptions,
|
|
1677
|
+
): Promise<string> {
|
|
1678
|
+
const run = (recoverySignal?: AbortSignal) =>
|
|
1679
|
+
runtime.todoPlanner({
|
|
1680
|
+
...plannerOptions,
|
|
1681
|
+
// The recovery signal carries cancellation/outage expiry only. It does
|
|
1682
|
+
// not alter timeoutMs, which remains authoritative for every replay.
|
|
1683
|
+
abortSignal: combineAbortSignals(plannerOptions.abortSignal, recoverySignal),
|
|
1684
|
+
});
|
|
1685
|
+
|
|
1686
|
+
try {
|
|
1687
|
+
return await run();
|
|
1688
|
+
} catch (initialFailure) {
|
|
1689
|
+
const classification = classifyNetworkFailure(initialFailure);
|
|
1690
|
+
if (plannerOptions.abortSignal?.aborted || classification.reason === "cancelled") {
|
|
1691
|
+
recordPlannerCancellation(runtime, plannerOptions.abortSignal, initialFailure);
|
|
1692
|
+
throw plannerCancellationError(plannerOptions.abortSignal, initialFailure);
|
|
1693
|
+
}
|
|
1694
|
+
if (!runtime.networkRecovery.enabled || !classification.recoverable) {
|
|
1695
|
+
throw initialFailure;
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1698
|
+
const publishRecovery = createNetworkRecoveryProgressHandler(runtime, "planner");
|
|
1699
|
+
let recoveryStarted = false;
|
|
1700
|
+
let lastRecoveryEvent: NetworkRecoveryEvent | undefined;
|
|
1701
|
+
const onRecoveryEvent = (event: NetworkRecoveryEvent) => {
|
|
1702
|
+
if (event.type !== "cleanup") lastRecoveryEvent = event;
|
|
1703
|
+
if (event.type === "outage_started" && !recoveryStarted) {
|
|
1704
|
+
recoveryStarted = true;
|
|
1705
|
+
recordPlannerDiagnostic(
|
|
1706
|
+
runtime,
|
|
1707
|
+
plannerNetworkDiagnostic(
|
|
1708
|
+
event,
|
|
1709
|
+
plannerOptions.timeoutMs,
|
|
1710
|
+
latestPlannerPartialOutput(runtime.plannerDiagnostics),
|
|
1711
|
+
),
|
|
1712
|
+
);
|
|
1713
|
+
}
|
|
1714
|
+
publishRecovery(event);
|
|
1715
|
+
if (event.type === "recovered") {
|
|
1716
|
+
recordPlannerDiagnostic(
|
|
1717
|
+
runtime,
|
|
1718
|
+
plannerNetworkDiagnostic(
|
|
1719
|
+
event,
|
|
1720
|
+
plannerOptions.timeoutMs,
|
|
1721
|
+
latestPlannerPartialOutput(runtime.plannerDiagnostics),
|
|
1722
|
+
),
|
|
1723
|
+
);
|
|
1724
|
+
}
|
|
1725
|
+
};
|
|
1726
|
+
|
|
1727
|
+
try {
|
|
1728
|
+
const recovered = await recoverNetworkOperation({
|
|
1729
|
+
initialFailure,
|
|
1730
|
+
config: runtime.networkRecovery,
|
|
1731
|
+
signal: plannerOptions.abortSignal,
|
|
1732
|
+
onEvent: onRecoveryEvent,
|
|
1733
|
+
retry: ({ signal }) => run(signal),
|
|
1734
|
+
});
|
|
1735
|
+
return recovered.value;
|
|
1736
|
+
} catch (recoveryFailure) {
|
|
1737
|
+
if (plannerOptions.abortSignal?.aborted) {
|
|
1738
|
+
recordPlannerCancellation(runtime, plannerOptions.abortSignal, recoveryFailure);
|
|
1739
|
+
throw plannerCancellationError(plannerOptions.abortSignal, recoveryFailure);
|
|
1740
|
+
} else if (!hasTerminalPlannerDiagnosticAfterLatestRecovery(runtime.plannerDiagnostics)) {
|
|
1741
|
+
const finalClassification = classifyNetworkFailure(recoveryFailure);
|
|
1742
|
+
recordPlannerDiagnostic(runtime, {
|
|
1743
|
+
kind: "network_failure",
|
|
1744
|
+
message: `TODO planner network recovery ended before planning completed: ${errorMessage(recoveryFailure)}`,
|
|
1745
|
+
partialOutputObserved: latestPlannerPartialOutput(runtime.plannerDiagnostics),
|
|
1746
|
+
networkRecoveryEvent:
|
|
1747
|
+
lastRecoveryEvent?.type === "outage_expired" || lastRecoveryEvent?.type === "failed"
|
|
1748
|
+
? lastRecoveryEvent.type
|
|
1749
|
+
: "failed",
|
|
1750
|
+
networkFailureReason: lastRecoveryEvent?.state.lastFailure.reason ?? finalClassification.reason,
|
|
1751
|
+
networkRetryCount: lastRecoveryEvent?.state.retryCount,
|
|
1752
|
+
networkOutageElapsedMs: lastRecoveryEvent?.state.elapsedMs,
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
throw recoveryFailure;
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1335
1758
|
}
|
|
1336
1759
|
|
|
1337
1760
|
async function generateSteeringPlanRevision(options: {
|
|
@@ -1364,21 +1787,28 @@ async function generateSteeringPlanRevision(options: {
|
|
|
1364
1787
|
}
|
|
1365
1788
|
: undefined,
|
|
1366
1789
|
planner: ({ prompt, request }) =>
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1790
|
+
runPlannerOperationWithNetworkRecovery(
|
|
1791
|
+
{
|
|
1792
|
+
inputText: prompt,
|
|
1793
|
+
plannerPrompt: prompt,
|
|
1794
|
+
planRevision: request,
|
|
1795
|
+
cwd: options.runtime.cwd,
|
|
1796
|
+
runDir: options.runtime.runDir,
|
|
1797
|
+
thinkingLevel: options.runtime.todoThinking,
|
|
1798
|
+
model: options.runtime.workerModel,
|
|
1799
|
+
abortSignal: options.runtime.abortSignal,
|
|
1800
|
+
timeoutMs: options.runtime.todoTimeoutMs,
|
|
1801
|
+
gracefulShutdownMs: options.runtime.todoGracefulShutdownMs,
|
|
1802
|
+
plannerBudget: options.runtime.plannerBudget,
|
|
1803
|
+
sessionFactory: options.runtime.todoSessionFactory,
|
|
1804
|
+
networkRecovery: options.runtime.networkRecovery,
|
|
1805
|
+
capabilityConstraints: options.runtime.capabilityWarnings.map((warning) => warning.planningConstraint),
|
|
1806
|
+
onDiagnostic: (diagnostic) => recordPlannerDiagnostic(options.runtime, diagnostic),
|
|
1807
|
+
onProgress: (event) => emitPlannerProgress(options.runtime, event),
|
|
1808
|
+
goal: options.runtime.goal,
|
|
1809
|
+
},
|
|
1810
|
+
options.runtime,
|
|
1811
|
+
),
|
|
1382
1812
|
});
|
|
1383
1813
|
}
|
|
1384
1814
|
|
|
@@ -1483,16 +1913,41 @@ function relevantPlanRevisionResults(
|
|
|
1483
1913
|
// Planner/worker lifecycle differences are audited in docs/planner-worker-lifecycle-audit.md;
|
|
1484
1914
|
// keep this function's public contract stable while moving shared prompt guarding into a helper.
|
|
1485
1915
|
export async function runTodoPlanner(options: TodoPlannerOptions): Promise<string> {
|
|
1916
|
+
const capabilityConstraints =
|
|
1917
|
+
options.capabilityConstraints ??
|
|
1918
|
+
capabilityWarningsForRequest(options.inputText, options.goal).map((warning) => warning.planningConstraint);
|
|
1919
|
+
const plannerBudget =
|
|
1920
|
+
options.plannerBudget ??
|
|
1921
|
+
resolvePlannerBudget({
|
|
1922
|
+
inputText: options.inputText,
|
|
1923
|
+
explicitTimeoutMs: options.timeoutMs,
|
|
1924
|
+
defaultTimeoutMs: DEFAULT_COORDINATOR_OPTIONS.todoTimeoutMs,
|
|
1925
|
+
});
|
|
1926
|
+
const timeoutMs = resolvePlannerTimeoutMs(options.timeoutMs, plannerBudget.timeoutMs);
|
|
1927
|
+
const gracefulShutdownMs = resolvePlannerGracefulShutdownMs(
|
|
1928
|
+
options.gracefulShutdownMs,
|
|
1929
|
+
DEFAULT_COORDINATOR_OPTIONS.todoGracefulShutdownMs,
|
|
1930
|
+
);
|
|
1931
|
+
const effectivePlannerBudget: PlannerBudget =
|
|
1932
|
+
timeoutMs === plannerBudget.timeoutMs
|
|
1933
|
+
? plannerBudget
|
|
1934
|
+
: {
|
|
1935
|
+
...plannerBudget,
|
|
1936
|
+
timeoutMs,
|
|
1937
|
+
extensionApplied: false,
|
|
1938
|
+
extensionMs: 0,
|
|
1939
|
+
source: "explicit",
|
|
1940
|
+
trigger: undefined,
|
|
1941
|
+
};
|
|
1942
|
+
notifyPlannerProgress(options.onProgress, createPlannerStartedProgress(effectivePlannerBudget, gracefulShutdownMs));
|
|
1486
1943
|
const sessionFactory = options.sessionFactory ?? createIsolatedWorkerSession;
|
|
1487
1944
|
const result = await sessionFactory({
|
|
1488
1945
|
cwd: options.cwd,
|
|
1489
1946
|
tools: [],
|
|
1490
1947
|
model: options.model,
|
|
1491
|
-
thinkingLevel: options.thinkingLevel,
|
|
1948
|
+
thinkingLevel: options.thinkingLevel ?? DEFAULT_PLANNER_THINKING_LEVEL,
|
|
1492
1949
|
});
|
|
1493
1950
|
const session = result.session;
|
|
1494
|
-
const timeoutMs = positiveMilliseconds(options.timeoutMs, DEFAULT_COORDINATOR_OPTIONS.todoTimeoutMs);
|
|
1495
|
-
const gracefulShutdownMs = options.gracefulShutdownMs ?? DEFAULT_COORDINATOR_OPTIONS.todoGracefulShutdownMs;
|
|
1496
1951
|
|
|
1497
1952
|
let plannerMarkdown: string | undefined;
|
|
1498
1953
|
let plannerError: unknown;
|
|
@@ -1500,12 +1955,14 @@ export async function runTodoPlanner(options: TodoPlannerOptions): Promise<strin
|
|
|
1500
1955
|
try {
|
|
1501
1956
|
const plannerText = await runTodoPlannerPrompt({
|
|
1502
1957
|
session,
|
|
1503
|
-
prompt: options.plannerPrompt ?? buildTodoCreationPrompt(options.inputText, options.goal),
|
|
1958
|
+
prompt: options.plannerPrompt ?? buildTodoCreationPrompt(options.inputText, options.goal, capabilityConstraints),
|
|
1504
1959
|
abortSignal: options.abortSignal,
|
|
1505
1960
|
timeoutMs,
|
|
1506
1961
|
gracefulShutdownMs,
|
|
1507
1962
|
diagnostics: result.diagnostics,
|
|
1508
1963
|
onDiagnostic: options.onDiagnostic,
|
|
1964
|
+
plannerBudget: effectivePlannerBudget,
|
|
1965
|
+
onProgress: options.onProgress,
|
|
1509
1966
|
});
|
|
1510
1967
|
|
|
1511
1968
|
plannerMarkdown = await extractTodoMarkdownWithOneRepair(
|
|
@@ -1520,8 +1977,11 @@ export async function runTodoPlanner(options: TodoPlannerOptions): Promise<strin
|
|
|
1520
1977
|
gracefulShutdownMs,
|
|
1521
1978
|
diagnostics: result.diagnostics,
|
|
1522
1979
|
onDiagnostic: options.onDiagnostic,
|
|
1980
|
+
plannerBudget: effectivePlannerBudget,
|
|
1981
|
+
onProgress: options.onProgress,
|
|
1523
1982
|
}),
|
|
1524
1983
|
options.goal,
|
|
1984
|
+
capabilityConstraints,
|
|
1525
1985
|
{
|
|
1526
1986
|
onInvalidOutput: (validationError) =>
|
|
1527
1987
|
options.onDiagnostic?.({
|
|
@@ -1565,7 +2025,8 @@ export async function runTodoPlanner(options: TodoPlannerOptions): Promise<strin
|
|
|
1565
2025
|
if (!plannerMarkdown) {
|
|
1566
2026
|
throw new TodoGenerationError("TODO planner did not return valid TODO markdown.");
|
|
1567
2027
|
}
|
|
1568
|
-
|
|
2028
|
+
const withGoal = applyGoalInstructionsToTodoMarkdown(plannerMarkdown, options.goal);
|
|
2029
|
+
return applyWorkerCapabilityConstraintsToTodoMarkdown(withGoal, capabilityConstraints);
|
|
1569
2030
|
}
|
|
1570
2031
|
|
|
1571
2032
|
async function runTodoPlannerPrompt(options: {
|
|
@@ -1576,6 +2037,8 @@ async function runTodoPlannerPrompt(options: {
|
|
|
1576
2037
|
gracefulShutdownMs: number;
|
|
1577
2038
|
diagnostics?: string[];
|
|
1578
2039
|
onDiagnostic?: PlannerDiagnosticHandler;
|
|
2040
|
+
plannerBudget: Readonly<PlannerBudget>;
|
|
2041
|
+
onProgress?: PlannerProgressHandler;
|
|
1579
2042
|
}): Promise<string> {
|
|
1580
2043
|
const promptResult = await runGuardedSessionPrompt({
|
|
1581
2044
|
session: options.session,
|
|
@@ -1585,23 +2048,56 @@ async function runTodoPlannerPrompt(options: {
|
|
|
1585
2048
|
gracefulShutdownMs: options.gracefulShutdownMs,
|
|
1586
2049
|
gracefulShutdownPrompt: buildTodoPlanningShutdownMessage(),
|
|
1587
2050
|
diagnostics: options.diagnostics,
|
|
2051
|
+
progressCheckpointsMs: plannerProgressCheckpoints(options.timeoutMs),
|
|
2052
|
+
onProgressCheckpoint: (elapsedMs) =>
|
|
2053
|
+
notifyPlannerProgress(
|
|
2054
|
+
options.onProgress,
|
|
2055
|
+
createPlannerActiveProgress(options.plannerBudget, options.gracefulShutdownMs, elapsedMs),
|
|
2056
|
+
),
|
|
2057
|
+
onGracePeriodStart: (gracePeriodMs) =>
|
|
2058
|
+
notifyPlannerProgress(options.onProgress, createPlannerGraceProgress(options.plannerBudget, gracePeriodMs)),
|
|
1588
2059
|
dispose: false,
|
|
1589
2060
|
});
|
|
1590
2061
|
|
|
2062
|
+
// Caller cancellation wins even when it arrives during grace. A hard abort
|
|
2063
|
+
// caused by grace expiry has cancelled=false and remains a timeout.
|
|
2064
|
+
if (promptResult.cancelled) {
|
|
2065
|
+
const outputState = promptResult.outputObserved
|
|
2066
|
+
? "partial output observed; content omitted"
|
|
2067
|
+
: "no planner output observed";
|
|
2068
|
+
const message = `TODO planner cancelled (${outputState}): ${promptResult.error ?? "caller cancellation"}`;
|
|
2069
|
+
options.onDiagnostic?.(plannerPromptDiagnostic("cancelled", message, promptResult));
|
|
2070
|
+
throw new TodoGenerationError(message);
|
|
2071
|
+
}
|
|
1591
2072
|
if (promptResult.timedOut) {
|
|
1592
|
-
|
|
2073
|
+
if (
|
|
2074
|
+
promptResult.completedDuringGrace &&
|
|
2075
|
+
promptResult.outputObserved &&
|
|
2076
|
+
!promptResult.error &&
|
|
2077
|
+
isSafeCompletedTodoPlannerOutput(promptResult.assistantText)
|
|
2078
|
+
) {
|
|
2079
|
+
return promptResult.assistantText;
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
const outputState = promptResult.outputObserved
|
|
2083
|
+
? "partial output observed; content omitted"
|
|
2084
|
+
: "no planner output observed";
|
|
2085
|
+
const message = `TODO planner timed out (${outputState}): ${promptResult.error ?? "time budget exceeded"}`;
|
|
1593
2086
|
options.onDiagnostic?.(plannerPromptDiagnostic("timeout", message, promptResult));
|
|
1594
2087
|
throw new TodoGenerationError(message);
|
|
1595
2088
|
}
|
|
1596
2089
|
if (promptResult.aborted) {
|
|
1597
|
-
const message = `TODO planner
|
|
1598
|
-
options.onDiagnostic?.(plannerPromptDiagnostic("
|
|
2090
|
+
const message = `TODO planner stopped: ${promptResult.error ?? "session abort"}`;
|
|
2091
|
+
options.onDiagnostic?.(plannerPromptDiagnostic("failure", message, promptResult));
|
|
1599
2092
|
throw new TodoGenerationError(message);
|
|
1600
2093
|
}
|
|
1601
2094
|
if (promptResult.error) {
|
|
1602
2095
|
const message = `TODO planner failed: ${promptResult.error}`;
|
|
1603
2096
|
options.onDiagnostic?.(plannerPromptDiagnostic("failure", message, promptResult));
|
|
1604
|
-
throw new TodoGenerationError(
|
|
2097
|
+
throw new TodoGenerationError(
|
|
2098
|
+
message,
|
|
2099
|
+
promptResult.failure === undefined ? undefined : { cause: promptResult.failure },
|
|
2100
|
+
);
|
|
1605
2101
|
}
|
|
1606
2102
|
if (!promptResult.assistantText) {
|
|
1607
2103
|
const message = "TODO planner did not return assistant text.";
|
|
@@ -1612,9 +2108,10 @@ async function runTodoPlannerPrompt(options: {
|
|
|
1612
2108
|
}
|
|
1613
2109
|
|
|
1614
2110
|
function plannerPromptDiagnostic(
|
|
1615
|
-
kind: Extract<PlannerDiagnosticKind, "timeout" | "abort" | "failure">,
|
|
2111
|
+
kind: Extract<PlannerDiagnosticKind, "timeout" | "cancelled" | "abort" | "failure">,
|
|
1616
2112
|
message: string,
|
|
1617
2113
|
promptResult: {
|
|
2114
|
+
outputObserved: boolean;
|
|
1618
2115
|
diagnostics: string[];
|
|
1619
2116
|
sessionFile?: string;
|
|
1620
2117
|
sessionId?: string;
|
|
@@ -1623,12 +2120,25 @@ function plannerPromptDiagnostic(
|
|
|
1623
2120
|
return {
|
|
1624
2121
|
kind,
|
|
1625
2122
|
message,
|
|
2123
|
+
partialOutputObserved: promptResult.outputObserved,
|
|
1626
2124
|
diagnostics: promptResult.diagnostics,
|
|
1627
2125
|
sessionFile: promptResult.sessionFile,
|
|
1628
2126
|
sessionId: promptResult.sessionId,
|
|
1629
2127
|
};
|
|
1630
2128
|
}
|
|
1631
2129
|
|
|
2130
|
+
function isSafeCompletedTodoPlannerOutput(text: string): boolean {
|
|
2131
|
+
if (!text.trim()) {
|
|
2132
|
+
return false;
|
|
2133
|
+
}
|
|
2134
|
+
try {
|
|
2135
|
+
extractAndValidateTodoMarkdown(text);
|
|
2136
|
+
return true;
|
|
2137
|
+
} catch {
|
|
2138
|
+
return false;
|
|
2139
|
+
}
|
|
2140
|
+
}
|
|
2141
|
+
|
|
1632
2142
|
function buildTodoPlanningShutdownMessage(): string {
|
|
1633
2143
|
return `Pi Long Task notice: TODO planning has reached its time budget.
|
|
1634
2144
|
Return the best valid Pi Long Task TODO markdown you can produce now, or stop if that is not possible.`;
|
|
@@ -1641,8 +2151,13 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1641
2151
|
const parsedWorkerConfig = parseWorkerRuntimeConfig(options.inputText ?? "");
|
|
1642
2152
|
const configuredAttempts = options.maxAttemptsPerTask ?? parsedWorkerConfig.maxAttemptsPerTask;
|
|
1643
2153
|
const configuredTaskTimeoutMs = options.taskTimeoutMs ?? parsedWorkerConfig.taskTimeoutMs;
|
|
1644
|
-
const configuredTodoTimeoutMs = options.todoTimeoutMs;
|
|
1645
|
-
const configuredTodoGracefulShutdownMs = options.todoGracefulShutdownMs;
|
|
2154
|
+
const configuredTodoTimeoutMs = options.todoTimeoutMs ?? parsedWorkerConfig.todoTimeoutMs;
|
|
2155
|
+
const configuredTodoGracefulShutdownMs = options.todoGracefulShutdownMs ?? parsedWorkerConfig.todoGracefulShutdownMs;
|
|
2156
|
+
const plannerBudget = resolvePlannerBudget({
|
|
2157
|
+
inputText: coordinatorInputText(options),
|
|
2158
|
+
explicitTimeoutMs: configuredTodoTimeoutMs,
|
|
2159
|
+
defaultTimeoutMs: DEFAULT_COORDINATOR_OPTIONS.todoTimeoutMs,
|
|
2160
|
+
});
|
|
1646
2161
|
const configuredMaxBashTimeoutMs = options.maxBashTimeoutMs ?? parsedWorkerConfig.maxBashTimeoutMs;
|
|
1647
2162
|
const workerModelName = options.workerModelName ?? parsedWorkerConfig.modelName;
|
|
1648
2163
|
const workerModel = workerModelName ? undefined : options.workerModel;
|
|
@@ -1652,6 +2167,11 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1652
2167
|
contextThresholdPercent:
|
|
1653
2168
|
options.workerSessionReuseContextThresholdPercent ?? parsedWorkerConfig.workerSessionReuseContextThresholdPercent,
|
|
1654
2169
|
});
|
|
2170
|
+
const networkRecovery = resolveNetworkRecoveryConfig({
|
|
2171
|
+
...parsedWorkerConfig.networkRecovery,
|
|
2172
|
+
...options.networkRecovery,
|
|
2173
|
+
});
|
|
2174
|
+
const capabilityWarnings = capabilityWarningsForRequest(options.inputText, options.goal);
|
|
1655
2175
|
|
|
1656
2176
|
return {
|
|
1657
2177
|
cwd,
|
|
@@ -1661,11 +2181,12 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1661
2181
|
taskResultPath: path.join(runDir, "TASK_RESULT.md"),
|
|
1662
2182
|
maxAttemptsPerTask: positiveInteger(configuredAttempts, DEFAULT_COORDINATOR_OPTIONS.maxAttemptsPerTask),
|
|
1663
2183
|
taskTimeoutSeconds: positiveMilliseconds(configuredTaskTimeoutMs, DEFAULT_COORDINATOR_OPTIONS.taskTimeoutMs) / 1000,
|
|
1664
|
-
todoTimeoutMs:
|
|
1665
|
-
todoGracefulShutdownMs:
|
|
2184
|
+
todoTimeoutMs: plannerBudget.timeoutMs,
|
|
2185
|
+
todoGracefulShutdownMs: resolvePlannerGracefulShutdownMs(
|
|
1666
2186
|
configuredTodoGracefulShutdownMs,
|
|
1667
2187
|
DEFAULT_COORDINATOR_OPTIONS.todoGracefulShutdownMs,
|
|
1668
2188
|
),
|
|
2189
|
+
plannerBudget,
|
|
1669
2190
|
maxBashTimeoutSeconds:
|
|
1670
2191
|
positiveMilliseconds(configuredMaxBashTimeoutMs, DEFAULT_COORDINATOR_OPTIONS.maxBashTimeoutMs) / 1000,
|
|
1671
2192
|
workerModel,
|
|
@@ -1675,6 +2196,7 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1675
2196
|
todoThinking: options.todoThinking ?? DEFAULT_COORDINATOR_OPTIONS.todoThinking,
|
|
1676
2197
|
workerSessionReuse: workerSessionReuseConfig.enabled,
|
|
1677
2198
|
workerSessionReuseContextThresholdPercent: workerSessionReuseConfig.contextThresholdPercent,
|
|
2199
|
+
networkRecovery,
|
|
1678
2200
|
workerRunner: options.workerRunner ?? runWorkerTask,
|
|
1679
2201
|
useRetainedWorkerLifecycle: options.workerRunner === undefined,
|
|
1680
2202
|
todoPlanner: options.todoPlanner ?? runTodoPlanner,
|
|
@@ -1688,18 +2210,41 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1688
2210
|
workerTextByWorker: new Map(),
|
|
1689
2211
|
workerTextPublishedLengthByWorker: new Map(),
|
|
1690
2212
|
plannerDiagnostics: [],
|
|
2213
|
+
capabilityWarnings,
|
|
1691
2214
|
workerSessionMetrics: createWorkerSessionMetrics(),
|
|
1692
2215
|
steeringQueue: options.steeringQueue,
|
|
1693
2216
|
onPlanRevisionAccepted: options.onPlanRevisionAccepted,
|
|
2217
|
+
onNetworkRecovery: options.onNetworkRecovery,
|
|
2218
|
+
progressClosed: false,
|
|
2219
|
+
networkRecoverySequence: 0,
|
|
2220
|
+
activeNetworkRecoveries: new Map(),
|
|
1694
2221
|
};
|
|
1695
2222
|
}
|
|
1696
2223
|
|
|
2224
|
+
function emitPlannerProgress(runtime: RuntimeOptions, event: Readonly<PlannerProgressEvent>): void {
|
|
2225
|
+
emitProgress(runtime, event.message, {
|
|
2226
|
+
phase: "planning",
|
|
2227
|
+
activeStatus: event.message,
|
|
2228
|
+
plannerBudget: event.budget,
|
|
2229
|
+
plannerProgressState: event.state,
|
|
2230
|
+
plannerElapsedMs: event.elapsedMs,
|
|
2231
|
+
plannerRemainingMs: event.remainingMs,
|
|
2232
|
+
plannerGracePeriodMs: event.gracePeriodMs,
|
|
2233
|
+
...(event.graceRemainingMs === undefined ? {} : { plannerGraceRemainingMs: event.graceRemainingMs }),
|
|
2234
|
+
});
|
|
2235
|
+
}
|
|
2236
|
+
|
|
2237
|
+
function notifyPlannerProgress(handler: PlannerProgressHandler | undefined, event: PlannerProgressEvent): void {
|
|
2238
|
+
handler?.(event);
|
|
2239
|
+
}
|
|
2240
|
+
|
|
1697
2241
|
function emitProgress(
|
|
1698
2242
|
runtime: RuntimeOptions,
|
|
1699
2243
|
message: string,
|
|
1700
2244
|
update: Omit<CoordinatorProgressUpdate, "message" | "runId" | "todoPath" | "resultPath" | "workerCostTotal">,
|
|
1701
2245
|
): void {
|
|
1702
|
-
runtime.
|
|
2246
|
+
if (runtime.progressClosed) return;
|
|
2247
|
+
const progress: CoordinatorProgressUpdate = {
|
|
1703
2248
|
message,
|
|
1704
2249
|
runId: runtime.runId,
|
|
1705
2250
|
todoPath: runtime.todoPath,
|
|
@@ -1707,16 +2252,193 @@ function emitProgress(
|
|
|
1707
2252
|
workerCostTotal: runtime.workerCostState.total,
|
|
1708
2253
|
...update,
|
|
1709
2254
|
goal: runtime.goal,
|
|
2255
|
+
};
|
|
2256
|
+
runtime.lastProgress = progress;
|
|
2257
|
+
const activeRecovery = latestNetworkRecovery(runtime.activeNetworkRecoveries);
|
|
2258
|
+
if (activeRecovery) {
|
|
2259
|
+
publishNetworkRecoveryProgress(runtime, activeRecovery.event, activeRecovery.operation);
|
|
2260
|
+
} else {
|
|
2261
|
+
runtime.onProgress?.(progress);
|
|
2262
|
+
}
|
|
2263
|
+
}
|
|
2264
|
+
|
|
2265
|
+
/**
|
|
2266
|
+
* Bridge one recovery lifecycle into coordinator progress without replacing the
|
|
2267
|
+
* last stable task/phase update. A recovered operation restores whichever
|
|
2268
|
+
* ordinary status is current; terminal failures are left for the normal final
|
|
2269
|
+
* status path. Operation IDs prevent an older concurrent recovery from
|
|
2270
|
+
* repainting a newer outage or completion.
|
|
2271
|
+
*/
|
|
2272
|
+
function createNetworkRecoveryProgressHandler(
|
|
2273
|
+
runtime: RuntimeOptions,
|
|
2274
|
+
operation: "planner" | "worker" = "worker",
|
|
2275
|
+
): (event: NetworkRecoveryEvent) => void {
|
|
2276
|
+
const operationId = ++runtime.networkRecoverySequence;
|
|
2277
|
+
let cleaned = false;
|
|
2278
|
+
|
|
2279
|
+
return (event) => {
|
|
2280
|
+
runtime.onNetworkRecovery?.(event);
|
|
2281
|
+
if (cleaned || runtime.progressClosed) return;
|
|
2282
|
+
|
|
2283
|
+
if (event.type === "cleanup") {
|
|
2284
|
+
cleaned = true;
|
|
2285
|
+
runtime.activeNetworkRecoveries.delete(operationId);
|
|
2286
|
+
return;
|
|
2287
|
+
}
|
|
2288
|
+
|
|
2289
|
+
if (isTerminalNetworkRecoveryEvent(event.type)) {
|
|
2290
|
+
runtime.activeNetworkRecoveries.delete(operationId);
|
|
2291
|
+
if (event.type === "recovered") {
|
|
2292
|
+
const active = latestNetworkRecovery(runtime.activeNetworkRecoveries);
|
|
2293
|
+
if (active) {
|
|
2294
|
+
publishNetworkRecoveryProgress(runtime, active.event, active.operation);
|
|
2295
|
+
} else if (runtime.lastProgress) {
|
|
2296
|
+
runtime.onProgress?.({ ...runtime.lastProgress, workerCostTotal: runtime.workerCostState.total });
|
|
2297
|
+
}
|
|
2298
|
+
}
|
|
2299
|
+
return;
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
runtime.activeNetworkRecoveries.set(operationId, { event, operation });
|
|
2303
|
+
if (operationId === latestNetworkRecoveryId(runtime.activeNetworkRecoveries)) {
|
|
2304
|
+
publishNetworkRecoveryProgress(runtime, event, operation);
|
|
2305
|
+
}
|
|
2306
|
+
};
|
|
2307
|
+
}
|
|
2308
|
+
|
|
2309
|
+
function publishNetworkRecoveryProgress(
|
|
2310
|
+
runtime: RuntimeOptions,
|
|
2311
|
+
event: NetworkRecoveryEvent,
|
|
2312
|
+
operation: "planner" | "worker" = "worker",
|
|
2313
|
+
): void {
|
|
2314
|
+
if (runtime.progressClosed) return;
|
|
2315
|
+
const stable = runtime.lastProgress;
|
|
2316
|
+
const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
|
|
2317
|
+
const recoveryStatus = formatNetworkRecoveryStatus(event);
|
|
2318
|
+
const message =
|
|
2319
|
+
operation === "planner"
|
|
2320
|
+
? `TODO planner network recovery: ${recoveryStatus} The ${formatFriendlyDuration(runtime.todoTimeoutMs)} planning deadline remains unchanged for each provider attempt.`
|
|
2321
|
+
: recoveryStatus;
|
|
2322
|
+
runtime.onProgress?.({
|
|
2323
|
+
message,
|
|
2324
|
+
phase: "network_wait",
|
|
2325
|
+
runId: runtime.runId,
|
|
2326
|
+
todoPath: runtime.todoPath,
|
|
2327
|
+
resultPath: runtime.taskResultPath,
|
|
2328
|
+
workerCostTotal: runtime.workerCostState.total,
|
|
2329
|
+
goal: runtime.goal,
|
|
2330
|
+
taskId: stable?.taskId,
|
|
2331
|
+
title: stable?.title,
|
|
2332
|
+
attempt: stable?.attempt,
|
|
2333
|
+
totalTasks: stable?.totalTasks,
|
|
2334
|
+
currentTask: stable?.currentTask,
|
|
2335
|
+
subtasks: stable?.subtasks,
|
|
2336
|
+
taskProgress: stable?.taskProgress,
|
|
2337
|
+
activeStatus: message,
|
|
2338
|
+
networkRecoveryEvent: event.type,
|
|
2339
|
+
networkRetryCount: event.state.retryCount,
|
|
2340
|
+
networkOutageElapsedMs: event.state.elapsedMs,
|
|
2341
|
+
networkNextRetryAtMs: event.state.nextRetryAtMs,
|
|
2342
|
+
networkNextRetryInMs:
|
|
2343
|
+
event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
|
|
2344
|
+
networkFailureReason: event.state.lastFailure.reason,
|
|
2345
|
+
networkOperation: operation,
|
|
2346
|
+
...(operation === "planner" ? { plannerDeadlinePolicy: "per_attempt_excludes_network_wait" as const } : {}),
|
|
1710
2347
|
});
|
|
1711
2348
|
}
|
|
1712
2349
|
|
|
2350
|
+
function latestNetworkRecovery(
|
|
2351
|
+
recoveries: ReadonlyMap<number, ActiveNetworkRecovery>,
|
|
2352
|
+
): ActiveNetworkRecovery | undefined {
|
|
2353
|
+
const id = latestNetworkRecoveryId(recoveries);
|
|
2354
|
+
return id === undefined ? undefined : recoveries.get(id);
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
function latestNetworkRecoveryId(recoveries: ReadonlyMap<number, ActiveNetworkRecovery>): number | undefined {
|
|
2358
|
+
let latest: number | undefined;
|
|
2359
|
+
for (const id of recoveries.keys()) {
|
|
2360
|
+
if (latest === undefined || id > latest) latest = id;
|
|
2361
|
+
}
|
|
2362
|
+
return latest;
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
function isTerminalNetworkRecoveryEvent(type: NetworkRecoveryEventType): boolean {
|
|
2366
|
+
return type === "recovered" || type === "failed" || type === "cancelled" || type === "outage_expired";
|
|
2367
|
+
}
|
|
2368
|
+
|
|
2369
|
+
function plannerNetworkDiagnostic(
|
|
2370
|
+
event: NetworkRecoveryEvent,
|
|
2371
|
+
timeoutMs: number | undefined,
|
|
2372
|
+
partialOutputObserved: boolean | undefined,
|
|
2373
|
+
): PlannerDiagnostic {
|
|
2374
|
+
const recovered = event.type === "recovered";
|
|
2375
|
+
const budget = formatFriendlyDuration(timeoutMs ?? DEFAULT_COORDINATOR_OPTIONS.todoTimeoutMs);
|
|
2376
|
+
return {
|
|
2377
|
+
kind: "network_recovery",
|
|
2378
|
+
message: recovered
|
|
2379
|
+
? `TODO planner network recovery succeeded after ${formatFriendlyDuration(event.state.elapsedMs)}; planning continues with its unchanged ${budget} per-attempt deadline.`
|
|
2380
|
+
: `TODO planner network recovery started (${event.state.lastFailure.reason}); its outage clock is separate and the ${budget} per-attempt planning deadline remains unchanged.`,
|
|
2381
|
+
partialOutputObserved,
|
|
2382
|
+
networkRecoveryEvent: event.type,
|
|
2383
|
+
networkFailureReason: event.state.lastFailure.reason,
|
|
2384
|
+
networkRetryCount: event.state.retryCount,
|
|
2385
|
+
networkOutageElapsedMs: event.state.elapsedMs,
|
|
2386
|
+
};
|
|
2387
|
+
}
|
|
2388
|
+
|
|
2389
|
+
function latestPlannerPartialOutput(diagnostics: readonly PlannerDiagnostic[]): boolean | undefined {
|
|
2390
|
+
return [...diagnostics].reverse().find((diagnostic) => diagnostic.partialOutputObserved !== undefined)
|
|
2391
|
+
?.partialOutputObserved;
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2394
|
+
function hasTerminalPlannerDiagnostic(diagnostics: readonly PlannerDiagnostic[]): boolean {
|
|
2395
|
+
const kind = diagnostics.at(-1)?.kind;
|
|
2396
|
+
return kind !== undefined && ["timeout", "cancelled", "abort", "network_failure", "failure"].includes(kind);
|
|
2397
|
+
}
|
|
2398
|
+
|
|
2399
|
+
function hasTerminalPlannerDiagnosticAfterLatestRecovery(diagnostics: readonly PlannerDiagnostic[]): boolean {
|
|
2400
|
+
let recoveryIndex = -1;
|
|
2401
|
+
for (let index = diagnostics.length - 1; index >= 0; index -= 1) {
|
|
2402
|
+
if (diagnostics[index]?.kind === "network_recovery") {
|
|
2403
|
+
recoveryIndex = index;
|
|
2404
|
+
break;
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
return diagnostics
|
|
2408
|
+
.slice(recoveryIndex + 1)
|
|
2409
|
+
.some((diagnostic) => ["timeout", "cancelled", "abort", "network_failure"].includes(diagnostic.kind));
|
|
2410
|
+
}
|
|
2411
|
+
|
|
2412
|
+
function recordPlannerCancellation(runtime: RuntimeOptions, signal: AbortSignal | undefined, cause: unknown): void {
|
|
2413
|
+
if (runtime.plannerDiagnostics.some((diagnostic) => diagnostic.kind === "cancelled")) return;
|
|
2414
|
+
recordPlannerDiagnostic(runtime, {
|
|
2415
|
+
kind: "cancelled",
|
|
2416
|
+
message: `TODO planning cancelled: ${signal?.aborted ? abortSignalReason(signal) : errorMessage(cause)}`,
|
|
2417
|
+
partialOutputObserved: latestPlannerPartialOutput(runtime.plannerDiagnostics),
|
|
2418
|
+
});
|
|
2419
|
+
}
|
|
2420
|
+
|
|
2421
|
+
function abortSignalReason(signal: AbortSignal): string {
|
|
2422
|
+
return signal.reason === undefined ? "cancelled by caller" : errorMessage(signal.reason);
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
function plannerCancellationError(signal: AbortSignal | undefined, cause: unknown): TodoGenerationError {
|
|
2426
|
+
const reason = signal?.aborted ? abortSignalReason(signal) : errorMessage(cause);
|
|
2427
|
+
return new TodoGenerationError(`TODO planning cancelled: ${reason}`, { cause });
|
|
2428
|
+
}
|
|
2429
|
+
|
|
1713
2430
|
function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDiagnostic): void {
|
|
1714
2431
|
const normalized: PlannerDiagnostic = {
|
|
1715
2432
|
kind: diagnostic.kind,
|
|
1716
2433
|
message: diagnostic.message,
|
|
2434
|
+
partialOutputObserved: diagnostic.partialOutputObserved,
|
|
1717
2435
|
diagnostics: diagnostic.diagnostics?.filter(Boolean),
|
|
1718
2436
|
sessionFile: diagnostic.sessionFile,
|
|
1719
2437
|
sessionId: diagnostic.sessionId,
|
|
2438
|
+
networkRecoveryEvent: diagnostic.networkRecoveryEvent,
|
|
2439
|
+
networkFailureReason: diagnostic.networkFailureReason,
|
|
2440
|
+
networkRetryCount: diagnostic.networkRetryCount,
|
|
2441
|
+
networkOutageElapsedMs: diagnostic.networkOutageElapsedMs,
|
|
1720
2442
|
};
|
|
1721
2443
|
const last = runtime.plannerDiagnostics.at(-1);
|
|
1722
2444
|
if (last?.kind === normalized.kind && last.message === normalized.message) {
|
|
@@ -1726,11 +2448,22 @@ function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDia
|
|
|
1726
2448
|
emitProgress(runtime, normalized.message, {
|
|
1727
2449
|
phase: "planning",
|
|
1728
2450
|
status: normalized.kind,
|
|
1729
|
-
isError:
|
|
2451
|
+
isError: !["repair_attempt", "network_recovery"].includes(normalized.kind),
|
|
1730
2452
|
plannerDiagnostic: normalized.kind,
|
|
1731
2453
|
plannerDiagnostics: normalized.diagnostics,
|
|
2454
|
+
plannerPartialOutputObserved: normalized.partialOutputObserved,
|
|
1732
2455
|
plannerSessionFile: normalized.sessionFile,
|
|
1733
2456
|
plannerSessionId: normalized.sessionId,
|
|
2457
|
+
networkRecoveryEvent: normalized.networkRecoveryEvent,
|
|
2458
|
+
networkRetryCount: normalized.networkRetryCount,
|
|
2459
|
+
networkOutageElapsedMs: normalized.networkOutageElapsedMs,
|
|
2460
|
+
networkFailureReason: normalized.networkFailureReason,
|
|
2461
|
+
networkOperation:
|
|
2462
|
+
normalized.kind === "network_recovery" || normalized.kind === "network_failure" ? "planner" : undefined,
|
|
2463
|
+
plannerDeadlinePolicy:
|
|
2464
|
+
normalized.kind === "network_recovery" || normalized.kind === "network_failure"
|
|
2465
|
+
? "per_attempt_excludes_network_wait"
|
|
2466
|
+
: undefined,
|
|
1734
2467
|
taskProgress: buildTaskProgressModel({ tasks: [] }),
|
|
1735
2468
|
});
|
|
1736
2469
|
}
|
|
@@ -2203,8 +2936,11 @@ function outcomeProgressItemStatus(
|
|
|
2203
2936
|
return "failed";
|
|
2204
2937
|
}
|
|
2205
2938
|
|
|
2206
|
-
function initialTaskResultMarkdown(runId: string): string {
|
|
2207
|
-
|
|
2939
|
+
function initialTaskResultMarkdown(runId: string, capabilityWarnings: readonly WorkerCapabilityWarning[] = []): string {
|
|
2940
|
+
const warningBlock = capabilityWarnings.length
|
|
2941
|
+
? `\n\n## Worker capability warnings\n\n${capabilityWarnings.map((warning) => `- ${warning.message}`).join("\n")}`
|
|
2942
|
+
: "";
|
|
2943
|
+
return `# Pi Long Task TASK_RESULT\n\nRun: ${runId}${warningBlock}\n`;
|
|
2208
2944
|
}
|
|
2209
2945
|
|
|
2210
2946
|
async function appendFailureNote(
|
|
@@ -2217,6 +2953,21 @@ async function appendFailureNote(
|
|
|
2217
2953
|
lines.push("", "### Planner diagnostics");
|
|
2218
2954
|
for (const diagnostic of plannerDiagnostics) {
|
|
2219
2955
|
lines.push("", `- ${diagnostic.kind}: ${diagnostic.message}`);
|
|
2956
|
+
if (diagnostic.partialOutputObserved !== undefined) {
|
|
2957
|
+
lines.push(` - Partial output observed: ${diagnostic.partialOutputObserved ? "yes" : "no"}`);
|
|
2958
|
+
}
|
|
2959
|
+
if (diagnostic.networkRecoveryEvent) {
|
|
2960
|
+
lines.push(` - Network recovery event: ${diagnostic.networkRecoveryEvent}`);
|
|
2961
|
+
}
|
|
2962
|
+
if (diagnostic.networkFailureReason) {
|
|
2963
|
+
lines.push(` - Network failure reason: ${diagnostic.networkFailureReason}`);
|
|
2964
|
+
}
|
|
2965
|
+
if (diagnostic.networkRetryCount !== undefined) {
|
|
2966
|
+
lines.push(` - Network retry count: ${diagnostic.networkRetryCount}`);
|
|
2967
|
+
}
|
|
2968
|
+
if (diagnostic.networkOutageElapsedMs !== undefined) {
|
|
2969
|
+
lines.push(` - Network outage elapsed: ${formatFriendlyDuration(diagnostic.networkOutageElapsedMs)}`);
|
|
2970
|
+
}
|
|
2220
2971
|
if (diagnostic.sessionId) {
|
|
2221
2972
|
lines.push(` - Session ID: ${diagnostic.sessionId}`);
|
|
2222
2973
|
}
|
|
@@ -2243,6 +2994,43 @@ async function appendCommitNote(pathname: string, result: CommitAfterSessionResu
|
|
|
2243
2994
|
await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
|
|
2244
2995
|
}
|
|
2245
2996
|
|
|
2997
|
+
async function appendNetworkInterruptionEvidence(
|
|
2998
|
+
pathname: string,
|
|
2999
|
+
outcome: SessionOutcome,
|
|
3000
|
+
networkRetry: number,
|
|
3001
|
+
): Promise<void> {
|
|
3002
|
+
const summary = extractResultSummary(outcome.assistantText || "").trim() || "TASK_RESULT:\nstatus: unknown";
|
|
3003
|
+
const lines = [
|
|
3004
|
+
"",
|
|
3005
|
+
`## TODO ${outcome.task.taskId} — ${outcome.task.title} (ordinary attempt ${outcome.attempt}, network interruption ${networkRetry})`,
|
|
3006
|
+
"",
|
|
3007
|
+
"Disposition: transient provider/transport failure; this is durable evidence, not an ordinary task attempt.",
|
|
3008
|
+
`Started: ${outcome.startedAt}`,
|
|
3009
|
+
`Ended: ${outcome.endedAt}`,
|
|
3010
|
+
`Worker error: ${outcome.error ?? "transient provider or transport failure"}`,
|
|
3011
|
+
];
|
|
3012
|
+
if (outcome.sessionId) lines.push(`Session ID: ${outcome.sessionId}`);
|
|
3013
|
+
if (outcome.sessionFile) lines.push(`Session file: ${outcome.sessionFile}`);
|
|
3014
|
+
if (outcome.workerCostSource || outcome.workerCostTotal > 0) {
|
|
3015
|
+
lines.push(`Worker cost: ${outcome.workerCostTotal} (${outcome.workerCostSource ?? "unavailable"})`);
|
|
3016
|
+
}
|
|
3017
|
+
if (outcome.workerUsage) {
|
|
3018
|
+
lines.push(
|
|
3019
|
+
`Worker token usage: input=${outcome.workerUsage.input}, output=${outcome.workerUsage.output}, cacheRead=${outcome.workerUsage.cacheRead}, cacheWrite=${outcome.workerUsage.cacheWrite}, total=${outcome.workerUsage.total}`,
|
|
3020
|
+
);
|
|
3021
|
+
}
|
|
3022
|
+
lines.push(
|
|
3023
|
+
"",
|
|
3024
|
+
"Safety: the replacement session must inspect the working tree and this evidence before continuing; completed side effects must not be blindly replayed.",
|
|
3025
|
+
"",
|
|
3026
|
+
"```text",
|
|
3027
|
+
summary,
|
|
3028
|
+
"```",
|
|
3029
|
+
"",
|
|
3030
|
+
);
|
|
3031
|
+
await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
|
|
3032
|
+
}
|
|
3033
|
+
|
|
2246
3034
|
async function appendTaskResult(
|
|
2247
3035
|
pathname: string,
|
|
2248
3036
|
task: Task,
|
|
@@ -2378,6 +3166,17 @@ function coordinatorInputText(options: RunCoordinatorOptions): string {
|
|
|
2378
3166
|
return normalizeOptionalText(options.inputText) ?? normalizeOptionalText(options.goal) ?? "";
|
|
2379
3167
|
}
|
|
2380
3168
|
|
|
3169
|
+
function capabilityWarningsForRequest(inputText?: string, goal?: string): WorkerCapabilityWarning[] {
|
|
3170
|
+
const requestText = [normalizeOptionalText(inputText), normalizeOptionalText(goal)]
|
|
3171
|
+
.filter((item): item is string => Boolean(item))
|
|
3172
|
+
.filter((item, index, all) => all.indexOf(item) === index)
|
|
3173
|
+
.join("\n");
|
|
3174
|
+
return detectUnavailableWorkerCapabilities(requestText, {
|
|
3175
|
+
tools: DEFAULT_WORKER_TOOLS,
|
|
3176
|
+
extensionsEnabled: false,
|
|
3177
|
+
});
|
|
3178
|
+
}
|
|
3179
|
+
|
|
2381
3180
|
function positiveInteger(value: number | undefined, fallback: number): number {
|
|
2382
3181
|
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
|
|
2383
3182
|
return Math.floor(value);
|