pi-long-task 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +85 -2
- package/package.json +1 -1
- package/src/coordinator.ts +450 -34
- package/src/goal_discovery.ts +2 -0
- package/src/goal_loop.ts +76 -0
- package/src/goal_orchestrator.ts +87 -1
- package/src/goal_review.ts +206 -15
- package/src/goal_todo_execution.ts +3 -0
- package/src/goal_todo_generation.ts +96 -3
- package/src/index.ts +2 -0
- package/src/network_failure.ts +574 -0
- package/src/network_recovery.ts +395 -0
- package/src/network_recovery_config.ts +89 -0
- package/src/render.ts +2 -0
- package/src/session_guard.ts +8 -1
- package/src/todo_generator.ts +2 -2
- package/src/types.ts +32 -0
- package/src/worker_config.ts +74 -0
- package/src/worker_session.ts +33 -1
package/src/coordinator.ts
CHANGED
|
@@ -10,6 +10,18 @@ import type {
|
|
|
10
10
|
} from "./types.ts";
|
|
11
11
|
import { commitAfterSession, gitDirtyPaths, shouldCommitOutcome, type CommitAfterSessionResult } from "./git.ts";
|
|
12
12
|
import { formatCoordinatorResultMessage } from "./render.ts";
|
|
13
|
+
import { classifyNetworkFailure } from "./network_failure.ts";
|
|
14
|
+
import {
|
|
15
|
+
formatNetworkRecoveryStatus,
|
|
16
|
+
recoverNetworkOperation,
|
|
17
|
+
type NetworkRecoveryEvent,
|
|
18
|
+
type NetworkRecoveryEventType,
|
|
19
|
+
} from "./network_recovery.ts";
|
|
20
|
+
import {
|
|
21
|
+
DEFAULT_NETWORK_RECOVERY_CONFIG,
|
|
22
|
+
resolveNetworkRecoveryConfig,
|
|
23
|
+
type NetworkRecoveryConfig,
|
|
24
|
+
} from "./network_recovery_config.ts";
|
|
13
25
|
import { extractResultSummary, hasCompleteTaskResult } from "./result_writer.ts";
|
|
14
26
|
import { runGuardedSessionPrompt } from "./session_guard.ts";
|
|
15
27
|
import {
|
|
@@ -79,6 +91,7 @@ export const DEFAULT_COORDINATOR_OPTIONS = {
|
|
|
79
91
|
todoThinking: "xhigh",
|
|
80
92
|
workerSessionReuse: DEFAULT_WORKER_SESSION_REUSE_ENABLED,
|
|
81
93
|
workerSessionReuseContextThresholdPercent: DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
|
|
94
|
+
networkRecovery: DEFAULT_NETWORK_RECOVERY_CONFIG,
|
|
82
95
|
} as const;
|
|
83
96
|
|
|
84
97
|
export type WorkerRunner = (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
|
|
@@ -88,6 +101,7 @@ export type CoordinatorProgressPhase =
|
|
|
88
101
|
| "task_start"
|
|
89
102
|
| "worker_session"
|
|
90
103
|
| "worker_tool"
|
|
104
|
+
| "network_wait"
|
|
91
105
|
| "task_done"
|
|
92
106
|
| "task_blocked"
|
|
93
107
|
| "task_failed"
|
|
@@ -150,6 +164,12 @@ export interface CoordinatorProgressUpdate {
|
|
|
150
164
|
workerSessionReason?: string;
|
|
151
165
|
workerSessionContextUsagePercent?: number;
|
|
152
166
|
workerSessionContextThresholdPercent?: number;
|
|
167
|
+
networkRecoveryEvent?: NetworkRecoveryEventType;
|
|
168
|
+
networkRetryCount?: number;
|
|
169
|
+
networkOutageElapsedMs?: number;
|
|
170
|
+
networkNextRetryAtMs?: number;
|
|
171
|
+
networkNextRetryInMs?: number;
|
|
172
|
+
networkFailureReason?: string;
|
|
153
173
|
}
|
|
154
174
|
|
|
155
175
|
export type CoordinatorProgressHandler = (update: CoordinatorProgressUpdate) => void;
|
|
@@ -182,6 +202,8 @@ export interface RunCoordinatorOptions extends PiLongTaskInput {
|
|
|
182
202
|
steeringQueue?: SerializedSteeringQueue;
|
|
183
203
|
/** Runs after rebase/validation and immediately before the revision is atomically persisted. */
|
|
184
204
|
onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
|
|
205
|
+
/** Receives coordinator-level outage lifecycle events for parent orchestrators and status integrations. */
|
|
206
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
185
207
|
}
|
|
186
208
|
|
|
187
209
|
export interface TodoPlannerOptions {
|
|
@@ -200,6 +222,8 @@ export interface TodoPlannerOptions {
|
|
|
200
222
|
plannerPrompt?: string;
|
|
201
223
|
/** Structured revision context supplied alongside plannerPrompt. */
|
|
202
224
|
planRevision?: Readonly<PlanRevisionRequest>;
|
|
225
|
+
/** Normalized coordinator recovery policy; network wait is excluded from operation timeouts. */
|
|
226
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
203
227
|
}
|
|
204
228
|
|
|
205
229
|
export interface TaskAttemptSummary {
|
|
@@ -280,6 +304,7 @@ interface RuntimeOptions {
|
|
|
280
304
|
todoThinking: string;
|
|
281
305
|
workerSessionReuse: boolean;
|
|
282
306
|
workerSessionReuseContextThresholdPercent: number;
|
|
307
|
+
networkRecovery: NetworkRecoveryConfig;
|
|
283
308
|
todoTimeoutMs: number;
|
|
284
309
|
todoGracefulShutdownMs: number;
|
|
285
310
|
workerRunner: WorkerRunner;
|
|
@@ -298,6 +323,11 @@ interface RuntimeOptions {
|
|
|
298
323
|
workerSessionMetrics: WorkerSessionMetrics;
|
|
299
324
|
steeringQueue?: SerializedSteeringQueue;
|
|
300
325
|
onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
|
|
326
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
327
|
+
lastProgress?: CoordinatorProgressUpdate;
|
|
328
|
+
progressClosed: boolean;
|
|
329
|
+
networkRecoverySequence: number;
|
|
330
|
+
activeNetworkRecoveries: Map<number, NetworkRecoveryEvent>;
|
|
301
331
|
}
|
|
302
332
|
|
|
303
333
|
type RetainedWorkerReuseScope = "sequential_task" | "partial_continuation";
|
|
@@ -733,6 +763,176 @@ function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortS
|
|
|
733
763
|
return AbortSignal.any(available);
|
|
734
764
|
}
|
|
735
765
|
|
|
766
|
+
/** Retains the full worker result while exposing its provider error to the shared classifier. */
|
|
767
|
+
class WorkerNetworkFailure extends Error {
|
|
768
|
+
readonly outcome: SessionOutcome;
|
|
769
|
+
|
|
770
|
+
constructor(outcome: SessionOutcome) {
|
|
771
|
+
const message = outcome.error ?? "worker network operation failed";
|
|
772
|
+
super(message, { cause: workerFailureValue(outcome) });
|
|
773
|
+
this.name = "WorkerNetworkFailure";
|
|
774
|
+
this.outcome = outcome;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
interface WorkerRecoveryExecutionOptions {
|
|
779
|
+
workerOptions: RunWorkerTaskOptions;
|
|
780
|
+
run: (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
|
|
781
|
+
taskResultPath: string;
|
|
782
|
+
networkRecovery: Readonly<NetworkRecoveryConfig>;
|
|
783
|
+
signal?: AbortSignal;
|
|
784
|
+
onInterruption?: (outcome: SessionOutcome) => void;
|
|
785
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* Run one ordinary worker attempt, replacing only transport-failed sessions.
|
|
790
|
+
* Pi owns bounded request retries inside session.prompt(); therefore an outcome
|
|
791
|
+
* reaches this boundary only after those retries have settled. Coordinator
|
|
792
|
+
* probes retain the same task/attempt and always receive a recovery prompt.
|
|
793
|
+
*/
|
|
794
|
+
async function runWorkerAttemptWithNetworkRecovery(options: WorkerRecoveryExecutionOptions): Promise<SessionOutcome> {
|
|
795
|
+
const interrupted: SessionOutcome[] = [];
|
|
796
|
+
const thrownErrors = new Map<SessionOutcome, unknown>();
|
|
797
|
+
const execute = async (workerOptions: RunWorkerTaskOptions): Promise<SessionOutcome> => {
|
|
798
|
+
try {
|
|
799
|
+
return await options.run(workerOptions);
|
|
800
|
+
} catch (error) {
|
|
801
|
+
const outcome = buildWorkerSessionCreationFailureOutcome(workerOptions, error);
|
|
802
|
+
thrownErrors.set(outcome, error);
|
|
803
|
+
return outcome;
|
|
804
|
+
}
|
|
805
|
+
};
|
|
806
|
+
const recordRecoverableInterruption = async (outcome: SessionOutcome): Promise<void> => {
|
|
807
|
+
await appendNetworkInterruptionEvidence(options.taskResultPath, outcome, interrupted.length + 1);
|
|
808
|
+
interrupted.push(outcome);
|
|
809
|
+
options.onInterruption?.(outcome);
|
|
810
|
+
};
|
|
811
|
+
|
|
812
|
+
const initial = await execute(options.workerOptions);
|
|
813
|
+
const initialFailure = workerFailureValue(initial);
|
|
814
|
+
const initialClassification = initialFailure === undefined ? undefined : classifyNetworkFailure(initialFailure);
|
|
815
|
+
if (initialClassification && isFailFastWorkerFailure(initialClassification.reason)) {
|
|
816
|
+
throw new WorkerNetworkFailure(initial);
|
|
817
|
+
}
|
|
818
|
+
if (!options.networkRecovery.enabled || !initialClassification?.recoverable) {
|
|
819
|
+
if (thrownErrors.has(initial)) throw thrownErrors.get(initial);
|
|
820
|
+
return initial;
|
|
821
|
+
}
|
|
822
|
+
await recordRecoverableInterruption(initial);
|
|
823
|
+
|
|
824
|
+
try {
|
|
825
|
+
const recovered = await recoverNetworkOperation({
|
|
826
|
+
initialFailure: new WorkerNetworkFailure(initial),
|
|
827
|
+
config: options.networkRecovery,
|
|
828
|
+
signal: options.signal,
|
|
829
|
+
onEvent: options.onNetworkRecovery,
|
|
830
|
+
retry: async ({ retryCount, signal }) => {
|
|
831
|
+
const previous = interrupted.at(-1)!;
|
|
832
|
+
const resumed = await execute({
|
|
833
|
+
...options.workerOptions,
|
|
834
|
+
// The recovery signal includes both run cancellation and the outage
|
|
835
|
+
// deadline without replacing the assignment/steering cancellation.
|
|
836
|
+
abortSignal: combineAbortSignals(options.workerOptions.abortSignal, signal),
|
|
837
|
+
networkRecoveryContext: {
|
|
838
|
+
retryCount,
|
|
839
|
+
durableEvidencePath: options.taskResultPath,
|
|
840
|
+
priorSessionId: previous.sessionId,
|
|
841
|
+
failure: previous.error ?? "transient provider or transport failure",
|
|
842
|
+
},
|
|
843
|
+
});
|
|
844
|
+
const resumedFailure = workerFailureValue(resumed);
|
|
845
|
+
const classification = resumedFailure === undefined ? undefined : classifyNetworkFailure(resumedFailure);
|
|
846
|
+
if (classification?.recoverable) {
|
|
847
|
+
await recordRecoverableInterruption(resumed);
|
|
848
|
+
}
|
|
849
|
+
if (resumed.error) {
|
|
850
|
+
if (
|
|
851
|
+
thrownErrors.has(resumed) &&
|
|
852
|
+
!classification?.recoverable &&
|
|
853
|
+
!isFailFastWorkerFailure(classification!.reason)
|
|
854
|
+
) {
|
|
855
|
+
throw thrownErrors.get(resumed);
|
|
856
|
+
}
|
|
857
|
+
throw new WorkerNetworkFailure(resumed);
|
|
858
|
+
}
|
|
859
|
+
return resumed;
|
|
860
|
+
},
|
|
861
|
+
});
|
|
862
|
+
return mergeWorkerRecoveryOutcomes(interrupted, recovered.value);
|
|
863
|
+
} catch (error) {
|
|
864
|
+
// If connectivity recovered but the fresh session failed deterministically,
|
|
865
|
+
// hand its outcome back to the ordinary worker failure path immediately.
|
|
866
|
+
if (error instanceof WorkerNetworkFailure) {
|
|
867
|
+
const merged = mergeWorkerRecoveryOutcomes(interrupted, error.outcome);
|
|
868
|
+
const mergedFailure = workerFailureValue(merged);
|
|
869
|
+
const classification = mergedFailure === undefined ? undefined : classifyNetworkFailure(mergedFailure);
|
|
870
|
+
if (classification && isFailFastWorkerFailure(classification.reason)) {
|
|
871
|
+
throw new WorkerNetworkFailure(merged);
|
|
872
|
+
}
|
|
873
|
+
return merged;
|
|
874
|
+
}
|
|
875
|
+
throw error;
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
function workerFailureValue(outcome: SessionOutcome): unknown {
|
|
880
|
+
return outcome.failure ?? outcome.error;
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
function isFailFastWorkerFailure(reason: ReturnType<typeof classifyNetworkFailure>["reason"]): boolean {
|
|
884
|
+
return [
|
|
885
|
+
"authentication",
|
|
886
|
+
"authorization",
|
|
887
|
+
"billing",
|
|
888
|
+
"quota_exhausted",
|
|
889
|
+
"invalid_model",
|
|
890
|
+
"invalid_request",
|
|
891
|
+
"http_client_error",
|
|
892
|
+
"non_retryable_server_error",
|
|
893
|
+
].includes(reason);
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
function mergeWorkerRecoveryOutcomes(interrupted: readonly SessionOutcome[], final: SessionOutcome): SessionOutcome {
|
|
897
|
+
if (interrupted.length === 0) return final;
|
|
898
|
+
const usage = addWorkerUsage([...interrupted.map((item) => item.workerUsage), final.workerUsage]);
|
|
899
|
+
return {
|
|
900
|
+
...final,
|
|
901
|
+
startedAt: interrupted[0].startedAt,
|
|
902
|
+
contextObservations: [
|
|
903
|
+
...interrupted.flatMap((item, index) => [
|
|
904
|
+
`network interruption ${index + 1}: ${item.error ?? "transient provider or transport failure"}`,
|
|
905
|
+
...item.contextObservations,
|
|
906
|
+
]),
|
|
907
|
+
...final.contextObservations,
|
|
908
|
+
],
|
|
909
|
+
compactionEvents: [...interrupted.flatMap((item) => item.compactionEvents), ...final.compactionEvents],
|
|
910
|
+
events: [...interrupted.flatMap((item) => item.events), ...final.events],
|
|
911
|
+
workerCostTotal: [...interrupted, final].reduce((total, item) => total + item.workerCostTotal, 0),
|
|
912
|
+
workerCostSource: "network_recovery_aggregate",
|
|
913
|
+
workerUsage: usage,
|
|
914
|
+
sessionDiagnostics: [
|
|
915
|
+
...interrupted.flatMap((item) => item.sessionDiagnostics ?? []),
|
|
916
|
+
...(final.sessionDiagnostics ?? []),
|
|
917
|
+
],
|
|
918
|
+
};
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
function addWorkerUsage(values: Array<WorkerUsageTotals | undefined>): WorkerUsageTotals | undefined {
|
|
922
|
+
const available = values.filter((value): value is WorkerUsageTotals => Boolean(value));
|
|
923
|
+
if (available.length === 0) return undefined;
|
|
924
|
+
return available.reduce<WorkerUsageTotals>(
|
|
925
|
+
(total, value) => ({
|
|
926
|
+
input: total.input + value.input,
|
|
927
|
+
output: total.output + value.output,
|
|
928
|
+
cacheRead: total.cacheRead + value.cacheRead,
|
|
929
|
+
cacheWrite: total.cacheWrite + value.cacheWrite,
|
|
930
|
+
total: total.total + value.total,
|
|
931
|
+
}),
|
|
932
|
+
{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
933
|
+
);
|
|
934
|
+
}
|
|
935
|
+
|
|
736
936
|
export function workerSessionHealthForOutcome(
|
|
737
937
|
outcome: Pick<SessionOutcome, "timedOut" | "aborted" | "error" | "assistantText">,
|
|
738
938
|
cancelled = false,
|
|
@@ -949,6 +1149,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
949
1149
|
thinkingLevel: runtime.taskThinking,
|
|
950
1150
|
abortSignal: combineAbortSignals(runtime.abortSignal, assignmentController.signal),
|
|
951
1151
|
sessionFactory: runtime.workerSessionFactory,
|
|
1152
|
+
networkRecovery: runtime.networkRecovery,
|
|
952
1153
|
now: runtime.now,
|
|
953
1154
|
onEvent: (event) => {
|
|
954
1155
|
if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
|
|
@@ -966,18 +1167,39 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
966
1167
|
},
|
|
967
1168
|
};
|
|
968
1169
|
let outcome: SessionOutcome;
|
|
1170
|
+
const networkInterruptedOutcomes: SessionOutcome[] = [];
|
|
969
1171
|
try {
|
|
970
|
-
outcome =
|
|
971
|
-
|
|
972
|
-
:
|
|
1172
|
+
outcome = await runWorkerAttemptWithNetworkRecovery({
|
|
1173
|
+
workerOptions,
|
|
1174
|
+
run: (resumedOptions) =>
|
|
1175
|
+
workerSessionOwner
|
|
1176
|
+
? workerSessionOwner.run(resumedOptions, assignmentIdentity)
|
|
1177
|
+
: runtime.workerRunner(resumedOptions),
|
|
1178
|
+
taskResultPath: runtime.taskResultPath,
|
|
1179
|
+
networkRecovery: runtime.networkRecovery,
|
|
1180
|
+
signal: workerOptions.abortSignal,
|
|
1181
|
+
onInterruption: (interrupted) => networkInterruptedOutcomes.push(interrupted),
|
|
1182
|
+
onNetworkRecovery: createNetworkRecoveryProgressHandler(runtime),
|
|
1183
|
+
});
|
|
973
1184
|
} catch (error) {
|
|
974
1185
|
if (!assignmentState.obsolete) {
|
|
1186
|
+
const terminalOutcome = error instanceof WorkerNetworkFailure ? error.outcome : undefined;
|
|
1187
|
+
if (terminalOutcome || networkInterruptedOutcomes.length > 0) {
|
|
1188
|
+
finalizeWorkerCost(runtime.workerCostState, accountingWorker, {
|
|
1189
|
+
workerCostTotal:
|
|
1190
|
+
terminalOutcome?.workerCostTotal ??
|
|
1191
|
+
networkInterruptedOutcomes.reduce((total, interrupted) => total + interrupted.workerCostTotal, 0),
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
975
1194
|
throw error;
|
|
976
1195
|
}
|
|
977
1196
|
// A cancellation-aware custom runner may reject instead of returning
|
|
978
1197
|
// an aborted outcome. Preserve historical evidence, but never let that
|
|
979
1198
|
// obsolete rejection terminate or update the replacement assignment.
|
|
980
|
-
outcome =
|
|
1199
|
+
outcome = mergeWorkerRecoveryOutcomes(
|
|
1200
|
+
networkInterruptedOutcomes,
|
|
1201
|
+
buildWorkerSessionCreationFailureOutcome(workerOptions, error),
|
|
1202
|
+
);
|
|
981
1203
|
outcome.aborted = true;
|
|
982
1204
|
}
|
|
983
1205
|
finalizeWorkerCost(runtime.workerCostState, accountingWorker, outcome);
|
|
@@ -1248,6 +1470,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
1248
1470
|
});
|
|
1249
1471
|
return result;
|
|
1250
1472
|
} finally {
|
|
1473
|
+
runtime.progressClosed = true;
|
|
1474
|
+
runtime.activeNetworkRecoveries.clear();
|
|
1251
1475
|
removeSteeringProcessor?.();
|
|
1252
1476
|
await workerSessionOwner?.dispose();
|
|
1253
1477
|
}
|
|
@@ -1319,19 +1543,59 @@ async function extractTodoMarkdownWithOneRepair(
|
|
|
1319
1543
|
}
|
|
1320
1544
|
|
|
1321
1545
|
async function requestTodoPlan(inputText: string, runtime: RuntimeOptions): Promise<string> {
|
|
1322
|
-
return
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1546
|
+
return runPlannerOperationWithNetworkRecovery(
|
|
1547
|
+
{
|
|
1548
|
+
inputText,
|
|
1549
|
+
cwd: runtime.cwd,
|
|
1550
|
+
runDir: runtime.runDir,
|
|
1551
|
+
thinkingLevel: runtime.todoThinking,
|
|
1552
|
+
model: runtime.workerModel,
|
|
1553
|
+
abortSignal: runtime.abortSignal,
|
|
1554
|
+
timeoutMs: runtime.todoTimeoutMs,
|
|
1555
|
+
gracefulShutdownMs: runtime.todoGracefulShutdownMs,
|
|
1556
|
+
sessionFactory: runtime.todoSessionFactory,
|
|
1557
|
+
networkRecovery: runtime.networkRecovery,
|
|
1558
|
+
onDiagnostic: (diagnostic) => recordPlannerDiagnostic(runtime, diagnostic),
|
|
1559
|
+
goal: runtime.goal,
|
|
1560
|
+
},
|
|
1561
|
+
runtime,
|
|
1562
|
+
);
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
/**
|
|
1566
|
+
* Retry a side-effect-free planner request only after its provider boundary has
|
|
1567
|
+
* failed. The default planner disables tools and disposes every session before
|
|
1568
|
+
* rejecting, so each retry rotates unsafe conversation state while replaying
|
|
1569
|
+
* only the complete immutable planning context. Recovery owns no planner
|
|
1570
|
+
* repair/attempt counter, and each fresh call retains the planner timeout;
|
|
1571
|
+
* backoff remains governed solely by the separate outage deadline.
|
|
1572
|
+
*/
|
|
1573
|
+
async function runPlannerOperationWithNetworkRecovery(
|
|
1574
|
+
plannerOptions: TodoPlannerOptions,
|
|
1575
|
+
runtime: RuntimeOptions,
|
|
1576
|
+
): Promise<string> {
|
|
1577
|
+
const run = (recoverySignal?: AbortSignal) =>
|
|
1578
|
+
runtime.todoPlanner({
|
|
1579
|
+
...plannerOptions,
|
|
1580
|
+
abortSignal: combineAbortSignals(plannerOptions.abortSignal, recoverySignal),
|
|
1581
|
+
});
|
|
1582
|
+
|
|
1583
|
+
try {
|
|
1584
|
+
return await run();
|
|
1585
|
+
} catch (initialFailure) {
|
|
1586
|
+
const classification = classifyNetworkFailure(initialFailure);
|
|
1587
|
+
if (!runtime.networkRecovery.enabled || !classification.recoverable) {
|
|
1588
|
+
throw initialFailure;
|
|
1589
|
+
}
|
|
1590
|
+
const recovered = await recoverNetworkOperation({
|
|
1591
|
+
initialFailure,
|
|
1592
|
+
config: runtime.networkRecovery,
|
|
1593
|
+
signal: plannerOptions.abortSignal,
|
|
1594
|
+
onEvent: createNetworkRecoveryProgressHandler(runtime),
|
|
1595
|
+
retry: ({ signal }) => run(signal),
|
|
1596
|
+
});
|
|
1597
|
+
return recovered.value;
|
|
1598
|
+
}
|
|
1335
1599
|
}
|
|
1336
1600
|
|
|
1337
1601
|
async function generateSteeringPlanRevision(options: {
|
|
@@ -1364,21 +1628,25 @@ async function generateSteeringPlanRevision(options: {
|
|
|
1364
1628
|
}
|
|
1365
1629
|
: undefined,
|
|
1366
1630
|
planner: ({ prompt, request }) =>
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1631
|
+
runPlannerOperationWithNetworkRecovery(
|
|
1632
|
+
{
|
|
1633
|
+
inputText: prompt,
|
|
1634
|
+
plannerPrompt: prompt,
|
|
1635
|
+
planRevision: request,
|
|
1636
|
+
cwd: options.runtime.cwd,
|
|
1637
|
+
runDir: options.runtime.runDir,
|
|
1638
|
+
thinkingLevel: options.runtime.todoThinking,
|
|
1639
|
+
model: options.runtime.workerModel,
|
|
1640
|
+
abortSignal: options.runtime.abortSignal,
|
|
1641
|
+
timeoutMs: options.runtime.todoTimeoutMs,
|
|
1642
|
+
gracefulShutdownMs: options.runtime.todoGracefulShutdownMs,
|
|
1643
|
+
sessionFactory: options.runtime.todoSessionFactory,
|
|
1644
|
+
networkRecovery: options.runtime.networkRecovery,
|
|
1645
|
+
onDiagnostic: (diagnostic) => recordPlannerDiagnostic(options.runtime, diagnostic),
|
|
1646
|
+
goal: options.runtime.goal,
|
|
1647
|
+
},
|
|
1648
|
+
options.runtime,
|
|
1649
|
+
),
|
|
1382
1650
|
});
|
|
1383
1651
|
}
|
|
1384
1652
|
|
|
@@ -1601,7 +1869,10 @@ async function runTodoPlannerPrompt(options: {
|
|
|
1601
1869
|
if (promptResult.error) {
|
|
1602
1870
|
const message = `TODO planner failed: ${promptResult.error}`;
|
|
1603
1871
|
options.onDiagnostic?.(plannerPromptDiagnostic("failure", message, promptResult));
|
|
1604
|
-
throw new TodoGenerationError(
|
|
1872
|
+
throw new TodoGenerationError(
|
|
1873
|
+
message,
|
|
1874
|
+
promptResult.failure === undefined ? undefined : { cause: promptResult.failure },
|
|
1875
|
+
);
|
|
1605
1876
|
}
|
|
1606
1877
|
if (!promptResult.assistantText) {
|
|
1607
1878
|
const message = "TODO planner did not return assistant text.";
|
|
@@ -1652,6 +1923,10 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1652
1923
|
contextThresholdPercent:
|
|
1653
1924
|
options.workerSessionReuseContextThresholdPercent ?? parsedWorkerConfig.workerSessionReuseContextThresholdPercent,
|
|
1654
1925
|
});
|
|
1926
|
+
const networkRecovery = resolveNetworkRecoveryConfig({
|
|
1927
|
+
...parsedWorkerConfig.networkRecovery,
|
|
1928
|
+
...options.networkRecovery,
|
|
1929
|
+
});
|
|
1655
1930
|
|
|
1656
1931
|
return {
|
|
1657
1932
|
cwd,
|
|
@@ -1675,6 +1950,7 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1675
1950
|
todoThinking: options.todoThinking ?? DEFAULT_COORDINATOR_OPTIONS.todoThinking,
|
|
1676
1951
|
workerSessionReuse: workerSessionReuseConfig.enabled,
|
|
1677
1952
|
workerSessionReuseContextThresholdPercent: workerSessionReuseConfig.contextThresholdPercent,
|
|
1953
|
+
networkRecovery,
|
|
1678
1954
|
workerRunner: options.workerRunner ?? runWorkerTask,
|
|
1679
1955
|
useRetainedWorkerLifecycle: options.workerRunner === undefined,
|
|
1680
1956
|
todoPlanner: options.todoPlanner ?? runTodoPlanner,
|
|
@@ -1691,6 +1967,10 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1691
1967
|
workerSessionMetrics: createWorkerSessionMetrics(),
|
|
1692
1968
|
steeringQueue: options.steeringQueue,
|
|
1693
1969
|
onPlanRevisionAccepted: options.onPlanRevisionAccepted,
|
|
1970
|
+
onNetworkRecovery: options.onNetworkRecovery,
|
|
1971
|
+
progressClosed: false,
|
|
1972
|
+
networkRecoverySequence: 0,
|
|
1973
|
+
activeNetworkRecoveries: new Map(),
|
|
1694
1974
|
};
|
|
1695
1975
|
}
|
|
1696
1976
|
|
|
@@ -1699,7 +1979,8 @@ function emitProgress(
|
|
|
1699
1979
|
message: string,
|
|
1700
1980
|
update: Omit<CoordinatorProgressUpdate, "message" | "runId" | "todoPath" | "resultPath" | "workerCostTotal">,
|
|
1701
1981
|
): void {
|
|
1702
|
-
runtime.
|
|
1982
|
+
if (runtime.progressClosed) return;
|
|
1983
|
+
const progress: CoordinatorProgressUpdate = {
|
|
1703
1984
|
message,
|
|
1704
1985
|
runId: runtime.runId,
|
|
1705
1986
|
todoPath: runtime.todoPath,
|
|
@@ -1707,9 +1988,107 @@ function emitProgress(
|
|
|
1707
1988
|
workerCostTotal: runtime.workerCostState.total,
|
|
1708
1989
|
...update,
|
|
1709
1990
|
goal: runtime.goal,
|
|
1991
|
+
};
|
|
1992
|
+
runtime.lastProgress = progress;
|
|
1993
|
+
const activeRecovery = latestNetworkRecovery(runtime.activeNetworkRecoveries);
|
|
1994
|
+
if (activeRecovery) {
|
|
1995
|
+
publishNetworkRecoveryProgress(runtime, activeRecovery);
|
|
1996
|
+
} else {
|
|
1997
|
+
runtime.onProgress?.(progress);
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
/**
|
|
2002
|
+
* Bridge one recovery lifecycle into coordinator progress without replacing the
|
|
2003
|
+
* last stable task/phase update. A recovered operation restores whichever
|
|
2004
|
+
* ordinary status is current; terminal failures are left for the normal final
|
|
2005
|
+
* status path. Operation IDs prevent an older concurrent recovery from
|
|
2006
|
+
* repainting a newer outage or completion.
|
|
2007
|
+
*/
|
|
2008
|
+
function createNetworkRecoveryProgressHandler(runtime: RuntimeOptions): (event: NetworkRecoveryEvent) => void {
|
|
2009
|
+
const operationId = ++runtime.networkRecoverySequence;
|
|
2010
|
+
let cleaned = false;
|
|
2011
|
+
|
|
2012
|
+
return (event) => {
|
|
2013
|
+
runtime.onNetworkRecovery?.(event);
|
|
2014
|
+
if (cleaned || runtime.progressClosed) return;
|
|
2015
|
+
|
|
2016
|
+
if (event.type === "cleanup") {
|
|
2017
|
+
cleaned = true;
|
|
2018
|
+
runtime.activeNetworkRecoveries.delete(operationId);
|
|
2019
|
+
return;
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
if (isTerminalNetworkRecoveryEvent(event.type)) {
|
|
2023
|
+
runtime.activeNetworkRecoveries.delete(operationId);
|
|
2024
|
+
if (event.type === "recovered") {
|
|
2025
|
+
const active = latestNetworkRecovery(runtime.activeNetworkRecoveries);
|
|
2026
|
+
if (active) {
|
|
2027
|
+
publishNetworkRecoveryProgress(runtime, active);
|
|
2028
|
+
} else if (runtime.lastProgress) {
|
|
2029
|
+
runtime.onProgress?.({ ...runtime.lastProgress, workerCostTotal: runtime.workerCostState.total });
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
2032
|
+
return;
|
|
2033
|
+
}
|
|
2034
|
+
|
|
2035
|
+
runtime.activeNetworkRecoveries.set(operationId, event);
|
|
2036
|
+
if (operationId === latestNetworkRecoveryId(runtime.activeNetworkRecoveries)) {
|
|
2037
|
+
publishNetworkRecoveryProgress(runtime, event);
|
|
2038
|
+
}
|
|
2039
|
+
};
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
function publishNetworkRecoveryProgress(runtime: RuntimeOptions, event: NetworkRecoveryEvent): void {
|
|
2043
|
+
if (runtime.progressClosed) return;
|
|
2044
|
+
const stable = runtime.lastProgress;
|
|
2045
|
+
const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
|
|
2046
|
+
const message = formatNetworkRecoveryStatus(event);
|
|
2047
|
+
runtime.onProgress?.({
|
|
2048
|
+
message,
|
|
2049
|
+
phase: "network_wait",
|
|
2050
|
+
runId: runtime.runId,
|
|
2051
|
+
todoPath: runtime.todoPath,
|
|
2052
|
+
resultPath: runtime.taskResultPath,
|
|
2053
|
+
workerCostTotal: runtime.workerCostState.total,
|
|
2054
|
+
goal: runtime.goal,
|
|
2055
|
+
taskId: stable?.taskId,
|
|
2056
|
+
title: stable?.title,
|
|
2057
|
+
attempt: stable?.attempt,
|
|
2058
|
+
totalTasks: stable?.totalTasks,
|
|
2059
|
+
currentTask: stable?.currentTask,
|
|
2060
|
+
subtasks: stable?.subtasks,
|
|
2061
|
+
taskProgress: stable?.taskProgress,
|
|
2062
|
+
activeStatus: message,
|
|
2063
|
+
networkRecoveryEvent: event.type,
|
|
2064
|
+
networkRetryCount: event.state.retryCount,
|
|
2065
|
+
networkOutageElapsedMs: event.state.elapsedMs,
|
|
2066
|
+
networkNextRetryAtMs: event.state.nextRetryAtMs,
|
|
2067
|
+
networkNextRetryInMs:
|
|
2068
|
+
event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
|
|
2069
|
+
networkFailureReason: event.state.lastFailure.reason,
|
|
1710
2070
|
});
|
|
1711
2071
|
}
|
|
1712
2072
|
|
|
2073
|
+
function latestNetworkRecovery(
|
|
2074
|
+
recoveries: ReadonlyMap<number, NetworkRecoveryEvent>,
|
|
2075
|
+
): NetworkRecoveryEvent | undefined {
|
|
2076
|
+
const id = latestNetworkRecoveryId(recoveries);
|
|
2077
|
+
return id === undefined ? undefined : recoveries.get(id);
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
function latestNetworkRecoveryId(recoveries: ReadonlyMap<number, NetworkRecoveryEvent>): number | undefined {
|
|
2081
|
+
let latest: number | undefined;
|
|
2082
|
+
for (const id of recoveries.keys()) {
|
|
2083
|
+
if (latest === undefined || id > latest) latest = id;
|
|
2084
|
+
}
|
|
2085
|
+
return latest;
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
function isTerminalNetworkRecoveryEvent(type: NetworkRecoveryEventType): boolean {
|
|
2089
|
+
return type === "recovered" || type === "failed" || type === "cancelled" || type === "outage_expired";
|
|
2090
|
+
}
|
|
2091
|
+
|
|
1713
2092
|
function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDiagnostic): void {
|
|
1714
2093
|
const normalized: PlannerDiagnostic = {
|
|
1715
2094
|
kind: diagnostic.kind,
|
|
@@ -2243,6 +2622,43 @@ async function appendCommitNote(pathname: string, result: CommitAfterSessionResu
|
|
|
2243
2622
|
await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
|
|
2244
2623
|
}
|
|
2245
2624
|
|
|
2625
|
+
async function appendNetworkInterruptionEvidence(
|
|
2626
|
+
pathname: string,
|
|
2627
|
+
outcome: SessionOutcome,
|
|
2628
|
+
networkRetry: number,
|
|
2629
|
+
): Promise<void> {
|
|
2630
|
+
const summary = extractResultSummary(outcome.assistantText || "").trim() || "TASK_RESULT:\nstatus: unknown";
|
|
2631
|
+
const lines = [
|
|
2632
|
+
"",
|
|
2633
|
+
`## TODO ${outcome.task.taskId} — ${outcome.task.title} (ordinary attempt ${outcome.attempt}, network interruption ${networkRetry})`,
|
|
2634
|
+
"",
|
|
2635
|
+
"Disposition: transient provider/transport failure; this is durable evidence, not an ordinary task attempt.",
|
|
2636
|
+
`Started: ${outcome.startedAt}`,
|
|
2637
|
+
`Ended: ${outcome.endedAt}`,
|
|
2638
|
+
`Worker error: ${outcome.error ?? "transient provider or transport failure"}`,
|
|
2639
|
+
];
|
|
2640
|
+
if (outcome.sessionId) lines.push(`Session ID: ${outcome.sessionId}`);
|
|
2641
|
+
if (outcome.sessionFile) lines.push(`Session file: ${outcome.sessionFile}`);
|
|
2642
|
+
if (outcome.workerCostSource || outcome.workerCostTotal > 0) {
|
|
2643
|
+
lines.push(`Worker cost: ${outcome.workerCostTotal} (${outcome.workerCostSource ?? "unavailable"})`);
|
|
2644
|
+
}
|
|
2645
|
+
if (outcome.workerUsage) {
|
|
2646
|
+
lines.push(
|
|
2647
|
+
`Worker token usage: input=${outcome.workerUsage.input}, output=${outcome.workerUsage.output}, cacheRead=${outcome.workerUsage.cacheRead}, cacheWrite=${outcome.workerUsage.cacheWrite}, total=${outcome.workerUsage.total}`,
|
|
2648
|
+
);
|
|
2649
|
+
}
|
|
2650
|
+
lines.push(
|
|
2651
|
+
"",
|
|
2652
|
+
"Safety: the replacement session must inspect the working tree and this evidence before continuing; completed side effects must not be blindly replayed.",
|
|
2653
|
+
"",
|
|
2654
|
+
"```text",
|
|
2655
|
+
summary,
|
|
2656
|
+
"```",
|
|
2657
|
+
"",
|
|
2658
|
+
);
|
|
2659
|
+
await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2246
2662
|
async function appendTaskResult(
|
|
2247
2663
|
pathname: string,
|
|
2248
2664
|
task: Task,
|
package/src/goal_discovery.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
} from "./goal_spec.ts";
|
|
14
14
|
import type { GoalLoopState } from "./goal_loop.ts";
|
|
15
15
|
import type { GoalStateStore } from "./goal_state.ts";
|
|
16
|
+
import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
|
|
16
17
|
|
|
17
18
|
export type GoalDiscoveryEntrypoint = "pi_goal_task" | "pi_long_task";
|
|
18
19
|
export type GoalDiscoveryRoute = "discovery" | "direct";
|
|
@@ -41,6 +42,7 @@ export interface GoalDiscoveryRunnerOptions {
|
|
|
41
42
|
model?: unknown;
|
|
42
43
|
modelName?: string;
|
|
43
44
|
thinkingLevel?: string;
|
|
45
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
44
46
|
now: () => Date;
|
|
45
47
|
}
|
|
46
48
|
|