pi-long-task 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +153 -3
- package/package.json +1 -1
- package/src/coordinator.ts +861 -62
- package/src/goal_discovery.ts +2 -0
- package/src/goal_loop.ts +76 -0
- package/src/goal_orchestrator.ts +98 -1
- package/src/goal_review.ts +206 -15
- package/src/goal_todo_execution.ts +7 -0
- package/src/goal_todo_generation.ts +108 -13
- package/src/index.ts +15 -1
- package/src/network_failure.ts +574 -0
- package/src/network_recovery.ts +395 -0
- package/src/network_recovery_config.ts +89 -0
- package/src/planner_config.ts +214 -0
- package/src/planner_progress.ts +156 -0
- package/src/render.ts +38 -0
- package/src/session_guard.ts +120 -7
- package/src/todo_generator.ts +84 -7
- package/src/types.ts +68 -0
- package/src/worker_capabilities.ts +103 -0
- package/src/worker_config.ts +148 -7
- package/src/worker_session.ts +33 -1
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
import { classifyNetworkFailure, type NetworkFailureClassification } from "./network_failure.ts";
|
|
2
|
+
import type { NetworkRecoveryConfig } from "./network_recovery_config.ts";
|
|
3
|
+
|
|
4
|
+
/** One-based network retry number. This counter is deliberately independent of TODO/session attempts. */
|
|
5
|
+
export interface NetworkRetryContext {
|
|
6
|
+
readonly retryCount: number;
|
|
7
|
+
readonly outageStartedAtMs: number;
|
|
8
|
+
readonly elapsedMs: number;
|
|
9
|
+
readonly networkWaitMs: number;
|
|
10
|
+
readonly signal?: AbortSignal;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface NetworkOutageState extends Omit<NetworkRetryContext, "signal"> {
|
|
14
|
+
readonly nextRetryAtMs?: number;
|
|
15
|
+
readonly lastDelayMs?: number;
|
|
16
|
+
readonly outageExpiresAtMs?: number;
|
|
17
|
+
readonly lastFailure: NetworkFailureClassification;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export type NetworkRecoveryEventType =
|
|
21
|
+
| "outage_started"
|
|
22
|
+
| "retry_scheduled"
|
|
23
|
+
| "retry_started"
|
|
24
|
+
| "retry_failed"
|
|
25
|
+
| "expiry_scheduled"
|
|
26
|
+
| "outage_expired"
|
|
27
|
+
| "recovered"
|
|
28
|
+
| "failed"
|
|
29
|
+
| "cancelled"
|
|
30
|
+
| "cleanup";
|
|
31
|
+
|
|
32
|
+
export interface NetworkRecoveryEvent {
|
|
33
|
+
readonly type: NetworkRecoveryEventType;
|
|
34
|
+
readonly state: NetworkOutageState;
|
|
35
|
+
/** Present when this event was caused by a newly observed failure. */
|
|
36
|
+
readonly classification?: NetworkFailureClassification;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type NetworkRecoveryJitter = (cappedDelayMs: number, random: () => number, retryCount: number) => number;
|
|
40
|
+
|
|
41
|
+
export type NetworkRecoverySleep = (delayMs: number, signal?: AbortSignal) => Promise<void>;
|
|
42
|
+
|
|
43
|
+
export interface RecoverNetworkOperationOptions<T> {
|
|
44
|
+
/** The transient error observed after Pi's own bounded request retries were exhausted. */
|
|
45
|
+
readonly initialFailure: unknown;
|
|
46
|
+
readonly config: Readonly<NetworkRecoveryConfig>;
|
|
47
|
+
/** A fresh coordinator-level retry. It receives only a network retry count, never an ordinary task attempt. */
|
|
48
|
+
readonly retry: (context: NetworkRetryContext) => Promise<T>;
|
|
49
|
+
readonly signal?: AbortSignal;
|
|
50
|
+
readonly now?: () => number;
|
|
51
|
+
readonly sleep?: NetworkRecoverySleep;
|
|
52
|
+
/** Injectable outage-deadline timer, separate from backoff sleep for deterministic tests. */
|
|
53
|
+
readonly deadlineSleep?: NetworkRecoverySleep;
|
|
54
|
+
readonly random?: () => number;
|
|
55
|
+
readonly jitter?: NetworkRecoveryJitter;
|
|
56
|
+
readonly classify?: (error: unknown) => NetworkFailureClassification;
|
|
57
|
+
readonly onEvent?: (event: NetworkRecoveryEvent) => void;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface NetworkRecoveryOutcome<T> {
|
|
61
|
+
readonly value: T;
|
|
62
|
+
readonly outage: NetworkOutageState;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** User-facing recovery status derived only from an immutable lifecycle event. */
|
|
66
|
+
export function formatNetworkRecoveryStatus(event: NetworkRecoveryEvent): string {
|
|
67
|
+
const elapsed = formatRecoveryDuration(event.state.elapsedMs);
|
|
68
|
+
const retry = event.state.retryCount;
|
|
69
|
+
const prefix = "Waiting for connection…";
|
|
70
|
+
|
|
71
|
+
switch (event.type) {
|
|
72
|
+
case "outage_started":
|
|
73
|
+
return `${prefix} outage detected (elapsed ${elapsed}).`;
|
|
74
|
+
case "retry_scheduled": {
|
|
75
|
+
const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
|
|
76
|
+
const remainingMs = Math.max(0, (event.state.nextRetryAtMs ?? nowMs) - nowMs);
|
|
77
|
+
return `${prefix} retry ${retry} in ${formatRecoveryDuration(remainingMs)} (outage ${elapsed}).`;
|
|
78
|
+
}
|
|
79
|
+
case "retry_started":
|
|
80
|
+
return `${prefix} checking connection with retry ${retry} (outage ${elapsed}).`;
|
|
81
|
+
case "retry_failed":
|
|
82
|
+
return `${prefix} retry ${retry} failed (outage ${elapsed}).`;
|
|
83
|
+
case "expiry_scheduled":
|
|
84
|
+
return `${prefix} outage ${elapsed}; recovery window expires in ${formatRecoveryDuration(
|
|
85
|
+
event.state.lastDelayMs ?? 0,
|
|
86
|
+
)}.`;
|
|
87
|
+
default:
|
|
88
|
+
return `${prefix} outage ${elapsed}.`;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function formatRecoveryDuration(milliseconds: number): string {
|
|
93
|
+
const safeMs = Math.max(0, Math.round(milliseconds));
|
|
94
|
+
if (safeMs > 0 && safeMs < 1_000) return "less than 1 second";
|
|
95
|
+
const seconds = safeMs / 1_000;
|
|
96
|
+
return `${Number.isInteger(seconds) ? seconds : seconds.toFixed(1)} seconds`;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export class NetworkOutageExpiredError extends Error {
|
|
100
|
+
readonly outage: NetworkOutageState;
|
|
101
|
+
readonly lastFailure: unknown;
|
|
102
|
+
|
|
103
|
+
constructor(outage: NetworkOutageState) {
|
|
104
|
+
const maximumMs =
|
|
105
|
+
outage.outageExpiresAtMs === undefined ? outage.elapsedMs : outage.outageExpiresAtMs - outage.outageStartedAtMs;
|
|
106
|
+
super(`Network outage exceeded the configured maximum of ${maximumMs}ms.`, {
|
|
107
|
+
cause: outage.lastFailure.error,
|
|
108
|
+
});
|
|
109
|
+
this.name = "NetworkOutageExpiredError";
|
|
110
|
+
this.outage = outage;
|
|
111
|
+
this.lastFailure = outage.lastFailure.error;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export class NetworkRecoveryAbortedError extends Error {
|
|
116
|
+
readonly reason: unknown;
|
|
117
|
+
|
|
118
|
+
constructor(reason: unknown) {
|
|
119
|
+
super(abortMessage(reason), reason instanceof Error ? { cause: reason } : undefined);
|
|
120
|
+
this.name = "AbortError";
|
|
121
|
+
this.reason = reason;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Equal jitter in the inclusive range [ceil(cappedDelay / 2), cappedDelay].
|
|
127
|
+
* Equal jitter avoids zero-delay retry loops while still spreading probes.
|
|
128
|
+
*/
|
|
129
|
+
export const equalNetworkRetryJitter: NetworkRecoveryJitter = (cappedDelayMs, random) => {
|
|
130
|
+
const randomValue = random();
|
|
131
|
+
if (!Number.isFinite(randomValue) || randomValue < 0 || randomValue > 1) {
|
|
132
|
+
throw new RangeError("Network recovery random() must return a finite number between 0 and 1.");
|
|
133
|
+
}
|
|
134
|
+
const minimum = Math.ceil(cappedDelayMs / 2);
|
|
135
|
+
return Math.min(cappedDelayMs, minimum + Math.floor(randomValue * (cappedDelayMs - minimum + 1)));
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
/** Calculate a one-based capped exponential delay, then apply and cap jitter. */
|
|
139
|
+
export function networkRetryDelayMs(
|
|
140
|
+
retryCount: number,
|
|
141
|
+
config: Pick<NetworkRecoveryConfig, "baseDelayMs" | "maxDelayMs">,
|
|
142
|
+
options: {
|
|
143
|
+
random?: () => number;
|
|
144
|
+
jitter?: NetworkRecoveryJitter;
|
|
145
|
+
} = {},
|
|
146
|
+
): number {
|
|
147
|
+
if (!Number.isSafeInteger(retryCount) || retryCount < 1) {
|
|
148
|
+
throw new RangeError("Network recovery retryCount must be a positive safe integer.");
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
const exponent = retryCount - 1;
|
|
152
|
+
const exponential = exponent > 1023 ? Number.POSITIVE_INFINITY : config.baseDelayMs * 2 ** exponent;
|
|
153
|
+
const capped = Math.min(config.maxDelayMs, exponential);
|
|
154
|
+
const jittered = (options.jitter ?? equalNetworkRetryJitter)(capped, options.random ?? Math.random, retryCount);
|
|
155
|
+
if (!Number.isFinite(jittered)) {
|
|
156
|
+
throw new RangeError("Network recovery jitter must return a finite delay.");
|
|
157
|
+
}
|
|
158
|
+
return Math.max(0, Math.min(capped, Math.round(jittered)));
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Resume an operation after an already-observed provider/transport failure.
|
|
163
|
+
*
|
|
164
|
+
* The helper owns only its `retryCount`; callers must not increment ordinary
|
|
165
|
+
* task, planner, reviewer, or goal-loop counters while it is running. Every
|
|
166
|
+
* wait and retry is raced with the caller's AbortSignal so cancellation does
|
|
167
|
+
* not depend on an injected sleep or provider callback cooperating.
|
|
168
|
+
*/
|
|
169
|
+
export async function recoverNetworkOperation<T>(
|
|
170
|
+
options: RecoverNetworkOperationOptions<T>,
|
|
171
|
+
): Promise<NetworkRecoveryOutcome<T>> {
|
|
172
|
+
throwIfAborted(options.signal);
|
|
173
|
+
|
|
174
|
+
const classify = options.classify ?? classifyNetworkFailure;
|
|
175
|
+
let classification = classify(options.initialFailure);
|
|
176
|
+
if (!options.config.enabled || !classification.recoverable) {
|
|
177
|
+
throw options.initialFailure;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const now = options.now ?? Date.now;
|
|
181
|
+
const sleep = options.sleep ?? abortableSleep;
|
|
182
|
+
const deadlineSleep = options.deadlineSleep ?? abortableSleep;
|
|
183
|
+
const outageStartedAtMs = now();
|
|
184
|
+
const outageExpiresAtMs =
|
|
185
|
+
options.config.maxOutageMs === null ? undefined : outageStartedAtMs + options.config.maxOutageMs;
|
|
186
|
+
let retryCount = 0;
|
|
187
|
+
let networkWaitMs = 0;
|
|
188
|
+
let nextRetryAtMs: number | undefined;
|
|
189
|
+
let lastDelayMs: number | undefined;
|
|
190
|
+
let elapsedMs = 0;
|
|
191
|
+
let terminalEventEmitted = false;
|
|
192
|
+
|
|
193
|
+
const state = (): NetworkOutageState => {
|
|
194
|
+
elapsedMs = Math.max(elapsedMs, Math.max(0, now() - outageStartedAtMs));
|
|
195
|
+
return Object.freeze({
|
|
196
|
+
retryCount,
|
|
197
|
+
outageStartedAtMs,
|
|
198
|
+
elapsedMs,
|
|
199
|
+
networkWaitMs,
|
|
200
|
+
nextRetryAtMs,
|
|
201
|
+
lastDelayMs,
|
|
202
|
+
outageExpiresAtMs,
|
|
203
|
+
lastFailure: classification,
|
|
204
|
+
});
|
|
205
|
+
};
|
|
206
|
+
const emit = (type: NetworkRecoveryEventType, causedByFailure = false): NetworkOutageState => {
|
|
207
|
+
const snapshot = state();
|
|
208
|
+
options.onEvent?.({ type, state: snapshot, classification: causedByFailure ? classification : undefined });
|
|
209
|
+
return snapshot;
|
|
210
|
+
};
|
|
211
|
+
const expire = (): never => {
|
|
212
|
+
nextRetryAtMs = undefined;
|
|
213
|
+
terminalEventEmitted = true;
|
|
214
|
+
const snapshot = emit("outage_expired");
|
|
215
|
+
throw new NetworkOutageExpiredError(snapshot);
|
|
216
|
+
};
|
|
217
|
+
|
|
218
|
+
const performWait = async (delayMs: number): Promise<void> => {
|
|
219
|
+
const startedAtMs = now();
|
|
220
|
+
try {
|
|
221
|
+
await raceWithAbort(sleep(delayMs, options.signal), options.signal);
|
|
222
|
+
} finally {
|
|
223
|
+
networkWaitMs += Math.max(0, now() - startedAtMs);
|
|
224
|
+
}
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
const executeRetry = async (): Promise<T> => {
|
|
228
|
+
const retryStartedAtMs = now();
|
|
229
|
+
const remainingOutageMs =
|
|
230
|
+
outageExpiresAtMs === undefined ? undefined : Math.max(0, outageExpiresAtMs - retryStartedAtMs);
|
|
231
|
+
if (remainingOutageMs === 0) expire();
|
|
232
|
+
|
|
233
|
+
const deadlineController = remainingOutageMs === undefined ? undefined : new AbortController();
|
|
234
|
+
const retryController = deadlineController ? new AbortController() : undefined;
|
|
235
|
+
const retrySignal = combineAbortSignals(options.signal, retryController?.signal);
|
|
236
|
+
const operation = Promise.resolve().then(() =>
|
|
237
|
+
options.retry({
|
|
238
|
+
retryCount,
|
|
239
|
+
outageStartedAtMs,
|
|
240
|
+
elapsedMs: state().elapsedMs,
|
|
241
|
+
networkWaitMs,
|
|
242
|
+
signal: retrySignal,
|
|
243
|
+
}),
|
|
244
|
+
);
|
|
245
|
+
|
|
246
|
+
if (remainingOutageMs === undefined || !deadlineController || !retryController) {
|
|
247
|
+
return await raceWithAbort(operation, options.signal);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
const deadline = raceWithAbort(
|
|
251
|
+
deadlineSleep(remainingOutageMs, deadlineController.signal),
|
|
252
|
+
deadlineController.signal,
|
|
253
|
+
).then(() => {
|
|
254
|
+
// The operation may have won the race and cancelled this deadline.
|
|
255
|
+
// Never allow a non-cooperative injected timer to emit a stale expiry.
|
|
256
|
+
if (deadlineController.signal.aborted) {
|
|
257
|
+
throw abortedError(deadlineController.signal);
|
|
258
|
+
}
|
|
259
|
+
const expired = new NetworkOutageExpiredError(state());
|
|
260
|
+
retryController.abort(expired);
|
|
261
|
+
return expire();
|
|
262
|
+
});
|
|
263
|
+
try {
|
|
264
|
+
return await raceWithAbort(Promise.race([operation, deadline]), options.signal);
|
|
265
|
+
} finally {
|
|
266
|
+
deadlineController.abort("network retry completed");
|
|
267
|
+
}
|
|
268
|
+
};
|
|
269
|
+
|
|
270
|
+
try {
|
|
271
|
+
emit("outage_started", true);
|
|
272
|
+
while (true) {
|
|
273
|
+
throwIfAborted(options.signal);
|
|
274
|
+
const beforeDelay = state();
|
|
275
|
+
if (outageExpiresAtMs !== undefined && beforeDelay.elapsedMs > options.config.maxOutageMs!) {
|
|
276
|
+
expire();
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const nextRetryCount = retryCount + 1;
|
|
280
|
+
const desiredDelayMs = networkRetryDelayMs(nextRetryCount, options.config, {
|
|
281
|
+
random: options.random,
|
|
282
|
+
jitter: options.jitter,
|
|
283
|
+
});
|
|
284
|
+
const currentTimeMs = now();
|
|
285
|
+
const remainingOutageMs =
|
|
286
|
+
outageExpiresAtMs === undefined ? undefined : Math.max(0, outageExpiresAtMs - currentTimeMs);
|
|
287
|
+
|
|
288
|
+
if (remainingOutageMs !== undefined && desiredDelayMs >= remainingOutageMs) {
|
|
289
|
+
lastDelayMs = remainingOutageMs;
|
|
290
|
+
nextRetryAtMs = undefined;
|
|
291
|
+
emit("expiry_scheduled");
|
|
292
|
+
await performWait(remainingOutageMs);
|
|
293
|
+
expire();
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
retryCount = nextRetryCount;
|
|
297
|
+
lastDelayMs = desiredDelayMs;
|
|
298
|
+
nextRetryAtMs = currentTimeMs + desiredDelayMs;
|
|
299
|
+
emit("retry_scheduled");
|
|
300
|
+
await performWait(desiredDelayMs);
|
|
301
|
+
nextRetryAtMs = undefined;
|
|
302
|
+
|
|
303
|
+
const afterWait = state();
|
|
304
|
+
if (outageExpiresAtMs !== undefined && afterWait.elapsedMs > options.config.maxOutageMs!) {
|
|
305
|
+
expire();
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
emit("retry_started");
|
|
309
|
+
try {
|
|
310
|
+
const value = await executeRetry();
|
|
311
|
+
terminalEventEmitted = true;
|
|
312
|
+
return { value, outage: emit("recovered") };
|
|
313
|
+
} catch (error) {
|
|
314
|
+
if (options.signal?.aborted) throw abortedError(options.signal);
|
|
315
|
+
if (error instanceof NetworkOutageExpiredError) throw error;
|
|
316
|
+
classification = classify(error);
|
|
317
|
+
if (!classification.recoverable) {
|
|
318
|
+
terminalEventEmitted = true;
|
|
319
|
+
emit("failed", true);
|
|
320
|
+
throw error;
|
|
321
|
+
}
|
|
322
|
+
emit("retry_failed", true);
|
|
323
|
+
if (outageExpiresAtMs !== undefined && state().elapsedMs >= options.config.maxOutageMs!) {
|
|
324
|
+
expire();
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
} catch (error) {
|
|
329
|
+
if (options.signal?.aborted) {
|
|
330
|
+
if (!terminalEventEmitted) emit("cancelled");
|
|
331
|
+
throw abortedError(options.signal);
|
|
332
|
+
}
|
|
333
|
+
throw error;
|
|
334
|
+
} finally {
|
|
335
|
+
emit("cleanup");
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
export function abortableSleep(delayMs: number, signal?: AbortSignal): Promise<void> {
|
|
340
|
+
throwIfAborted(signal);
|
|
341
|
+
if (delayMs <= 0) return Promise.resolve();
|
|
342
|
+
|
|
343
|
+
return new Promise((resolve, reject) => {
|
|
344
|
+
const timer = setTimeout(finish, delayMs);
|
|
345
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
346
|
+
|
|
347
|
+
function finish(): void {
|
|
348
|
+
signal?.removeEventListener("abort", abort);
|
|
349
|
+
resolve();
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function abort(): void {
|
|
353
|
+
clearTimeout(timer);
|
|
354
|
+
signal?.removeEventListener("abort", abort);
|
|
355
|
+
reject(abortedError(signal));
|
|
356
|
+
}
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
async function raceWithAbort<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
361
|
+
throwIfAborted(signal);
|
|
362
|
+
if (!signal) return promise;
|
|
363
|
+
|
|
364
|
+
let abort: (() => void) | undefined;
|
|
365
|
+
const cancellation = new Promise<never>((_resolve, reject) => {
|
|
366
|
+
abort = () => reject(abortedError(signal));
|
|
367
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
368
|
+
});
|
|
369
|
+
try {
|
|
370
|
+
return await Promise.race([promise, cancellation]);
|
|
371
|
+
} finally {
|
|
372
|
+
if (abort) signal.removeEventListener("abort", abort);
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined {
|
|
377
|
+
const available = signals.filter((signal): signal is AbortSignal => Boolean(signal));
|
|
378
|
+
if (available.length === 0) return undefined;
|
|
379
|
+
if (available.length === 1) return available[0];
|
|
380
|
+
return AbortSignal.any(available);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function throwIfAborted(signal?: AbortSignal): void {
|
|
384
|
+
if (signal?.aborted) throw abortedError(signal);
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function abortedError(signal?: AbortSignal): NetworkRecoveryAbortedError {
|
|
388
|
+
return new NetworkRecoveryAbortedError(signal?.reason);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function abortMessage(reason: unknown): string {
|
|
392
|
+
if (reason instanceof Error && reason.message) return `Network recovery was aborted: ${reason.message}`;
|
|
393
|
+
if (typeof reason === "string" && reason) return `Network recovery was aborted: ${reason}`;
|
|
394
|
+
return "Network recovery was aborted.";
|
|
395
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
export const NETWORK_RECOVERY_TIMEOUT_POLICY = "exclude-network-wait" as const;
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* User-provided coordinator network recovery settings. `maxOutageMs: null` is
|
|
5
|
+
* the only unlimited mode: recovery continues until the run is cancelled.
|
|
6
|
+
*/
|
|
7
|
+
export interface NetworkRecoveryConfigInput {
|
|
8
|
+
enabled?: boolean;
|
|
9
|
+
baseDelayMs?: number;
|
|
10
|
+
maxDelayMs?: number;
|
|
11
|
+
maxOutageMs?: number | null;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Fully normalized settings shared by every coordinator provider operation.
|
|
16
|
+
*
|
|
17
|
+
* Timeout policy: time spent waiting for network recovery is accounted for
|
|
18
|
+
* separately and does not consume worker, TODO-planner, reviewer, goal-loop,
|
|
19
|
+
* or goal-iteration timeout budgets. It also does not consume their ordinary
|
|
20
|
+
* attempt/retry/iteration limits. The bounded maxOutageMs (or cancellation in
|
|
21
|
+
* unlimited mode) is the sole limit while connectivity is unavailable.
|
|
22
|
+
*/
|
|
23
|
+
export interface NetworkRecoveryConfig {
|
|
24
|
+
readonly enabled: boolean;
|
|
25
|
+
readonly baseDelayMs: number;
|
|
26
|
+
readonly maxDelayMs: number;
|
|
27
|
+
readonly maxOutageMs: number | null;
|
|
28
|
+
readonly timeoutPolicy: typeof NETWORK_RECOVERY_TIMEOUT_POLICY;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Disabled by default to preserve pre-recovery coordinator behavior. */
|
|
32
|
+
export const DEFAULT_NETWORK_RECOVERY_CONFIG: Readonly<NetworkRecoveryConfig> = Object.freeze({
|
|
33
|
+
enabled: false,
|
|
34
|
+
baseDelayMs: 1_000,
|
|
35
|
+
maxDelayMs: 30_000,
|
|
36
|
+
maxOutageMs: 5 * 60_000,
|
|
37
|
+
timeoutPolicy: NETWORK_RECOVERY_TIMEOUT_POLICY,
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
export class NetworkRecoveryConfigError extends Error {
|
|
41
|
+
constructor(message: string) {
|
|
42
|
+
super(message);
|
|
43
|
+
this.name = "NetworkRecoveryConfigError";
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function resolveNetworkRecoveryConfig(
|
|
48
|
+
input: NetworkRecoveryConfigInput | undefined = {},
|
|
49
|
+
): NetworkRecoveryConfig {
|
|
50
|
+
const enabled = input.enabled ?? DEFAULT_NETWORK_RECOVERY_CONFIG.enabled;
|
|
51
|
+
if (typeof enabled !== "boolean") {
|
|
52
|
+
throw new NetworkRecoveryConfigError("networkRecovery.enabled must be a boolean.");
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const baseDelayMs = input.baseDelayMs ?? DEFAULT_NETWORK_RECOVERY_CONFIG.baseDelayMs;
|
|
56
|
+
const maxDelayMs = input.maxDelayMs ?? DEFAULT_NETWORK_RECOVERY_CONFIG.maxDelayMs;
|
|
57
|
+
const maxOutageMs = input.maxOutageMs === undefined ? DEFAULT_NETWORK_RECOVERY_CONFIG.maxOutageMs : input.maxOutageMs;
|
|
58
|
+
|
|
59
|
+
assertPositiveMilliseconds("networkRecovery.baseDelayMs", baseDelayMs);
|
|
60
|
+
assertPositiveMilliseconds("networkRecovery.maxDelayMs", maxDelayMs);
|
|
61
|
+
if (maxDelayMs < baseDelayMs) {
|
|
62
|
+
throw new NetworkRecoveryConfigError(
|
|
63
|
+
"networkRecovery.maxDelayMs must be greater than or equal to networkRecovery.baseDelayMs.",
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (maxOutageMs !== null) {
|
|
68
|
+
assertPositiveMilliseconds("networkRecovery.maxOutageMs", maxOutageMs);
|
|
69
|
+
if (maxOutageMs < baseDelayMs) {
|
|
70
|
+
throw new NetworkRecoveryConfigError(
|
|
71
|
+
"networkRecovery.maxOutageMs must be null (unlimited) or greater than or equal to networkRecovery.baseDelayMs.",
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
return Object.freeze({
|
|
77
|
+
enabled,
|
|
78
|
+
baseDelayMs,
|
|
79
|
+
maxDelayMs,
|
|
80
|
+
maxOutageMs,
|
|
81
|
+
timeoutPolicy: NETWORK_RECOVERY_TIMEOUT_POLICY,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function assertPositiveMilliseconds(name: string, value: unknown): asserts value is number {
|
|
86
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) {
|
|
87
|
+
throw new NetworkRecoveryConfigError(`${name} must be a positive, finite, safe integer number of milliseconds.`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
export const MAX_PLANNER_DURATION_MS = 2_147_483_647;
|
|
2
|
+
|
|
3
|
+
/** Thinking levels accepted by the supported Pi SDK, in increasing reasoning-budget order. */
|
|
4
|
+
export const SUPPORTED_PLANNER_THINKING_LEVELS = ["minimal", "low", "medium", "high", "xhigh", "max"] as const;
|
|
5
|
+
export type PlannerThinkingLevel = (typeof SUPPORTED_PLANNER_THINKING_LEVELS)[number];
|
|
6
|
+
/**
|
|
7
|
+
* Planner-only quality/latency balance. `high` retains enough reasoning budget
|
|
8
|
+
* for dependency-aware complex plans without imposing `xhigh` latency on every
|
|
9
|
+
* ordinary request. Explicit caller values remain authoritative.
|
|
10
|
+
*/
|
|
11
|
+
export const DEFAULT_PLANNER_THINKING_LEVEL: PlannerThinkingLevel = "high";
|
|
12
|
+
|
|
13
|
+
/** Normal planning budget used for requests that do not contain a scale signal. */
|
|
14
|
+
export const DEFAULT_PLANNER_TIMEOUT_MS = 300_000;
|
|
15
|
+
/** Adaptive planning never reduces the normal five-minute budget. */
|
|
16
|
+
export const MIN_ADAPTIVE_PLANNER_TIMEOUT_MS = DEFAULT_PLANNER_TIMEOUT_MS;
|
|
17
|
+
/** Adaptive planning is capped at fifteen minutes, even for very large requests. */
|
|
18
|
+
export const MAX_ADAPTIVE_PLANNER_TIMEOUT_MS = 900_000;
|
|
19
|
+
/** The normal budget includes up to four requested deliverables. */
|
|
20
|
+
export const PLANNER_ITEMS_INCLUDED_IN_BASE_BUDGET = 4;
|
|
21
|
+
/** Every additional detected deliverable adds thirty seconds until the cap. */
|
|
22
|
+
export const PLANNER_TIMEOUT_PER_ADDITIONAL_ITEM_MS = 30_000;
|
|
23
|
+
|
|
24
|
+
export type PlannerComplexitySignalKind =
|
|
25
|
+
| "explicit_item_count"
|
|
26
|
+
| "enumerated_deliverables"
|
|
27
|
+
| "separately_planned_tasks";
|
|
28
|
+
|
|
29
|
+
export interface PlannerComplexitySignal {
|
|
30
|
+
kind: PlannerComplexitySignalKind;
|
|
31
|
+
itemCount: number;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export type PlannerBudgetSource = "explicit" | "default" | "adaptive";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Complete, machine-readable record of how a planner deadline was selected.
|
|
38
|
+
* `trigger` is present only when a deterministic complexity signal actually
|
|
39
|
+
* extended the normal budget.
|
|
40
|
+
*/
|
|
41
|
+
export interface PlannerBudget {
|
|
42
|
+
timeoutMs: number;
|
|
43
|
+
baseTimeoutMs: number;
|
|
44
|
+
minimumTimeoutMs: number;
|
|
45
|
+
maximumTimeoutMs: number;
|
|
46
|
+
extensionApplied: boolean;
|
|
47
|
+
extensionMs: number;
|
|
48
|
+
source: PlannerBudgetSource;
|
|
49
|
+
signals: readonly PlannerComplexitySignal[];
|
|
50
|
+
trigger?: PlannerComplexitySignal;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface ResolvePlannerBudgetOptions {
|
|
54
|
+
inputText: string;
|
|
55
|
+
/** Structured or natural-language timeout configuration. It always wins exactly. */
|
|
56
|
+
explicitTimeoutMs?: number;
|
|
57
|
+
defaultTimeoutMs?: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const EXPLICIT_ITEM_COUNT_RE =
|
|
61
|
+
/\b(\d{1,6})\s+(?:(?:separate(?:ly)?|individual(?:ly)?|distinct|independent(?:ly)?)\s+(?:planned\s+)?)?(?:(?:user|job|work)\s+)?(?:stories|tasks|todos?|deliverables?|items?|work\s+items?|features?|components?|pages?|endpoints?|tests?|scenarios?|requirements?)\b/gi;
|
|
62
|
+
const ENUMERATED_DELIVERABLE_RE = /^\s*(?:[-*+]\s+(?:\[[ xX]\]\s+)?|\d{1,6}[.)]\s+)\S.*$/gm;
|
|
63
|
+
const SEPARATE_PLANNING_RE =
|
|
64
|
+
/\b(?:separately|individually|independently)\s+(?:plan(?:ned)?|scope(?:d)?|specif(?:y|ied)|assign(?:ed)?)\b|\b(?:plan|scope|specify)\s+(?:each|every)\b|\b(?:separate|individual|independent)\s+(?:plans?|tasks?|todos?|work\s+items?)\b/i;
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Detects only reproducible textual scale signals:
|
|
68
|
+
*
|
|
69
|
+
* 1. an integer directly attached to a deliverable noun (for example,
|
|
70
|
+
* "24 stories"),
|
|
71
|
+
* 2. line-start bullet or numbered deliverables, and
|
|
72
|
+
* 3. explicit language requiring those items to be planned separately.
|
|
73
|
+
*
|
|
74
|
+
* The detector intentionally does not estimate semantic difficulty or ask the
|
|
75
|
+
* model to grade complexity. Signals are returned in stable priority order.
|
|
76
|
+
*/
|
|
77
|
+
export function detectPlannerComplexitySignals(inputText: string): PlannerComplexitySignal[] {
|
|
78
|
+
const normalized = inputText.replace(/\r\n?/g, "\n");
|
|
79
|
+
const explicitItemCount = maximumMatchedInteger(normalized, EXPLICIT_ITEM_COUNT_RE);
|
|
80
|
+
const enumeratedItemCount = [...normalized.matchAll(ENUMERATED_DELIVERABLE_RE)].length;
|
|
81
|
+
const signals: PlannerComplexitySignal[] = [];
|
|
82
|
+
|
|
83
|
+
if (SEPARATE_PLANNING_RE.test(normalized)) {
|
|
84
|
+
const separatelyPlannedCount = Math.max(explicitItemCount ?? 0, enumeratedItemCount);
|
|
85
|
+
if (separatelyPlannedCount > 0) {
|
|
86
|
+
signals.push({ kind: "separately_planned_tasks", itemCount: separatelyPlannedCount });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (explicitItemCount !== undefined) {
|
|
90
|
+
signals.push({ kind: "explicit_item_count", itemCount: explicitItemCount });
|
|
91
|
+
}
|
|
92
|
+
if (enumeratedItemCount > 0) {
|
|
93
|
+
signals.push({ kind: "enumerated_deliverables", itemCount: enumeratedItemCount });
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return signals;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Selects a deterministic planner deadline. The normal five-minute budget
|
|
101
|
+
* covers four items; each additional item adds thirty seconds, capped at
|
|
102
|
+
* fifteen minutes. Explicit timeout configuration bypasses both detection and
|
|
103
|
+
* adaptive bounds and is returned unchanged after public duration validation.
|
|
104
|
+
*/
|
|
105
|
+
export function resolvePlannerBudget(options: ResolvePlannerBudgetOptions): PlannerBudget {
|
|
106
|
+
const baseTimeoutMs = resolvePlannerTimeoutMs(options.defaultTimeoutMs, DEFAULT_PLANNER_TIMEOUT_MS);
|
|
107
|
+
|
|
108
|
+
if (options.explicitTimeoutMs !== undefined) {
|
|
109
|
+
const timeoutMs = resolvePlannerTimeoutMs(options.explicitTimeoutMs, baseTimeoutMs);
|
|
110
|
+
return {
|
|
111
|
+
timeoutMs,
|
|
112
|
+
baseTimeoutMs,
|
|
113
|
+
minimumTimeoutMs: MIN_ADAPTIVE_PLANNER_TIMEOUT_MS,
|
|
114
|
+
maximumTimeoutMs: MAX_ADAPTIVE_PLANNER_TIMEOUT_MS,
|
|
115
|
+
extensionApplied: false,
|
|
116
|
+
extensionMs: 0,
|
|
117
|
+
source: "explicit",
|
|
118
|
+
signals: [],
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const signals = detectPlannerComplexitySignals(options.inputText);
|
|
123
|
+
const trigger = strongestComplexitySignal(signals);
|
|
124
|
+
const additionalItems = Math.max(0, (trigger?.itemCount ?? 0) - PLANNER_ITEMS_INCLUDED_IN_BASE_BUDGET);
|
|
125
|
+
const requestedTimeoutMs = baseTimeoutMs + additionalItems * PLANNER_TIMEOUT_PER_ADDITIONAL_ITEM_MS;
|
|
126
|
+
const timeoutMs = Math.min(
|
|
127
|
+
MAX_ADAPTIVE_PLANNER_TIMEOUT_MS,
|
|
128
|
+
Math.max(MIN_ADAPTIVE_PLANNER_TIMEOUT_MS, requestedTimeoutMs),
|
|
129
|
+
);
|
|
130
|
+
const extensionMs = Math.max(0, timeoutMs - baseTimeoutMs);
|
|
131
|
+
const extensionApplied = extensionMs > 0;
|
|
132
|
+
|
|
133
|
+
return {
|
|
134
|
+
timeoutMs,
|
|
135
|
+
baseTimeoutMs,
|
|
136
|
+
minimumTimeoutMs: MIN_ADAPTIVE_PLANNER_TIMEOUT_MS,
|
|
137
|
+
maximumTimeoutMs: MAX_ADAPTIVE_PLANNER_TIMEOUT_MS,
|
|
138
|
+
extensionApplied,
|
|
139
|
+
extensionMs,
|
|
140
|
+
source: extensionApplied ? "adaptive" : "default",
|
|
141
|
+
signals,
|
|
142
|
+
...(extensionApplied && trigger ? { trigger } : {}),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export class PlannerDurationConfigError extends Error {
|
|
147
|
+
constructor(message: string) {
|
|
148
|
+
super(message);
|
|
149
|
+
this.name = "PlannerDurationConfigError";
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export function resolvePlannerTimeoutMs(value: number | undefined, fallback: number): number {
|
|
154
|
+
return resolvePlannerDurationMs(value, fallback, "TODO planner timeout", false);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export function resolvePlannerGracefulShutdownMs(value: number | undefined, fallback: number): number {
|
|
158
|
+
return resolvePlannerDurationMs(value, fallback, "TODO planner graceful-shutdown duration", true);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function validatePlannerTimeoutMs(value: number | undefined): number | undefined {
|
|
162
|
+
return value === undefined ? undefined : resolvePlannerTimeoutMs(value, value);
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function validatePlannerGracefulShutdownMs(value: number | undefined): number | undefined {
|
|
166
|
+
return value === undefined ? undefined : resolvePlannerGracefulShutdownMs(value, value);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function maximumMatchedInteger(value: string, expression: RegExp): number | undefined {
|
|
170
|
+
expression.lastIndex = 0;
|
|
171
|
+
let maximum: number | undefined;
|
|
172
|
+
for (const match of value.matchAll(expression)) {
|
|
173
|
+
const count = Number.parseInt(match[1], 10);
|
|
174
|
+
if (Number.isSafeInteger(count) && count > 0) {
|
|
175
|
+
maximum = Math.max(maximum ?? 0, count);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
return maximum;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function strongestComplexitySignal(signals: readonly PlannerComplexitySignal[]): PlannerComplexitySignal | undefined {
|
|
182
|
+
// Stable input order is also the tie-break priority: separate planning,
|
|
183
|
+
// explicit counts, then plain enumeration.
|
|
184
|
+
return signals.reduce<PlannerComplexitySignal | undefined>(
|
|
185
|
+
(strongest, signal) => (!strongest || signal.itemCount > strongest.itemCount ? signal : strongest),
|
|
186
|
+
undefined,
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function resolvePlannerDurationMs(
|
|
191
|
+
value: number | undefined,
|
|
192
|
+
fallback: number,
|
|
193
|
+
label: string,
|
|
194
|
+
allowZero: boolean,
|
|
195
|
+
): number {
|
|
196
|
+
if (value === undefined) {
|
|
197
|
+
return fallback;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const minimumDescription = allowZero ? "a non-negative" : "a positive";
|
|
201
|
+
if (
|
|
202
|
+
typeof value !== "number" ||
|
|
203
|
+
!Number.isSafeInteger(value) ||
|
|
204
|
+
(!allowZero && value <= 0) ||
|
|
205
|
+
(allowZero && value < 0) ||
|
|
206
|
+
value > MAX_PLANNER_DURATION_MS
|
|
207
|
+
) {
|
|
208
|
+
throw new PlannerDurationConfigError(
|
|
209
|
+
`${label} must be ${minimumDescription} whole-millisecond duration no greater than about 24.9 days (${MAX_PLANNER_DURATION_MS} milliseconds); received ${String(value)}.`,
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
return value;
|
|
214
|
+
}
|