pi-long-task 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +139 -10
- package/package.json +2 -2
- package/src/coordinator.ts +1132 -45
- package/src/goal_discovery.ts +2 -0
- package/src/goal_loop.ts +76 -0
- package/src/goal_orchestrator.ts +87 -1
- package/src/goal_review.ts +206 -15
- package/src/goal_todo_execution.ts +3 -0
- package/src/goal_todo_generation.ts +96 -3
- package/src/index.ts +20 -5
- package/src/network_failure.ts +574 -0
- package/src/network_recovery.ts +395 -0
- package/src/network_recovery_config.ts +89 -0
- package/src/render.ts +2 -0
- package/src/session_guard.ts +8 -1
- package/src/todo_generator.ts +2 -2
- package/src/types.ts +32 -0
- package/src/worker_config.ts +137 -14
- package/src/worker_reuse_policy.ts +389 -0
- package/src/worker_session.ts +294 -34
package/src/coordinator.ts
CHANGED
|
@@ -10,7 +10,19 @@ import type {
|
|
|
10
10
|
} from "./types.ts";
|
|
11
11
|
import { commitAfterSession, gitDirtyPaths, shouldCommitOutcome, type CommitAfterSessionResult } from "./git.ts";
|
|
12
12
|
import { formatCoordinatorResultMessage } from "./render.ts";
|
|
13
|
-
import {
|
|
13
|
+
import { classifyNetworkFailure } from "./network_failure.ts";
|
|
14
|
+
import {
|
|
15
|
+
formatNetworkRecoveryStatus,
|
|
16
|
+
recoverNetworkOperation,
|
|
17
|
+
type NetworkRecoveryEvent,
|
|
18
|
+
type NetworkRecoveryEventType,
|
|
19
|
+
} from "./network_recovery.ts";
|
|
20
|
+
import {
|
|
21
|
+
DEFAULT_NETWORK_RECOVERY_CONFIG,
|
|
22
|
+
resolveNetworkRecoveryConfig,
|
|
23
|
+
type NetworkRecoveryConfig,
|
|
24
|
+
} from "./network_recovery_config.ts";
|
|
25
|
+
import { extractResultSummary, hasCompleteTaskResult } from "./result_writer.ts";
|
|
14
26
|
import { runGuardedSessionPrompt } from "./session_guard.ts";
|
|
15
27
|
import {
|
|
16
28
|
generatePlanRevision,
|
|
@@ -28,6 +40,16 @@ import {
|
|
|
28
40
|
} from "./plan_store.ts";
|
|
29
41
|
import type { SerializedSteeringQueue, SteeringMessage } from "./steering.ts";
|
|
30
42
|
import { parseWorkerRuntimeConfig } from "./worker_config.ts";
|
|
43
|
+
import {
|
|
44
|
+
classifyWorkerSessionRetry,
|
|
45
|
+
createWorkerSessionCompatibilityFingerprint,
|
|
46
|
+
decideWorkerSessionReuse,
|
|
47
|
+
DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
|
|
48
|
+
DEFAULT_WORKER_SESSION_REUSE_ENABLED,
|
|
49
|
+
resolveWorkerSessionReuseConfig,
|
|
50
|
+
type WorkerSessionCompatibilityFingerprint,
|
|
51
|
+
type WorkerSessionHealth,
|
|
52
|
+
} from "./worker_reuse_policy.ts";
|
|
31
53
|
import { buildTaskProgressModel, type TaskProgressModel, type TaskProgressStatus } from "./task_progress.ts";
|
|
32
54
|
import {
|
|
33
55
|
applyGoalInstructionsToTodoMarkdown,
|
|
@@ -40,12 +62,21 @@ import {
|
|
|
40
62
|
} from "./todo_generator.ts";
|
|
41
63
|
import { parseTasks, todoGlobalInstructions, type Task } from "./todo_parser.ts";
|
|
42
64
|
import {
|
|
65
|
+
buildWorkerSessionCreationFailureOutcome,
|
|
43
66
|
createIsolatedWorkerSession,
|
|
67
|
+
createWorkerSessionResource,
|
|
68
|
+
DEFAULT_WORKER_TOOLS,
|
|
69
|
+
disposeWorkerSessionResource,
|
|
44
70
|
runWorkerTask,
|
|
71
|
+
runWorkerTaskAssignment,
|
|
72
|
+
workerSessionContextUsagePercent,
|
|
45
73
|
type RunWorkerTaskOptions,
|
|
46
74
|
type SessionOutcome,
|
|
75
|
+
type WorkerSessionDiagnostic,
|
|
47
76
|
type WorkerSessionFactory,
|
|
48
77
|
type WorkerSessionLike,
|
|
78
|
+
type WorkerSessionResource,
|
|
79
|
+
type WorkerUsageTotals,
|
|
49
80
|
} from "./worker_session.ts";
|
|
50
81
|
|
|
51
82
|
export type { CoordinatorStatus } from "./types.ts";
|
|
@@ -58,6 +89,9 @@ export const DEFAULT_COORDINATOR_OPTIONS = {
|
|
|
58
89
|
maxBashTimeoutMs: 300_000,
|
|
59
90
|
taskThinking: "high",
|
|
60
91
|
todoThinking: "xhigh",
|
|
92
|
+
workerSessionReuse: DEFAULT_WORKER_SESSION_REUSE_ENABLED,
|
|
93
|
+
workerSessionReuseContextThresholdPercent: DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
|
|
94
|
+
networkRecovery: DEFAULT_NETWORK_RECOVERY_CONFIG,
|
|
61
95
|
} as const;
|
|
62
96
|
|
|
63
97
|
export type WorkerRunner = (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
|
|
@@ -65,7 +99,9 @@ export type CoordinatorProgressPhase =
|
|
|
65
99
|
| "planning"
|
|
66
100
|
| "planned"
|
|
67
101
|
| "task_start"
|
|
102
|
+
| "worker_session"
|
|
68
103
|
| "worker_tool"
|
|
104
|
+
| "network_wait"
|
|
69
105
|
| "task_done"
|
|
70
106
|
| "task_blocked"
|
|
71
107
|
| "task_failed"
|
|
@@ -124,6 +160,16 @@ export interface CoordinatorProgressUpdate {
|
|
|
124
160
|
plannerDiagnostics?: string[];
|
|
125
161
|
plannerSessionFile?: string;
|
|
126
162
|
plannerSessionId?: string;
|
|
163
|
+
workerSessionEvent?: WorkerSessionDiagnostic["event"];
|
|
164
|
+
workerSessionReason?: string;
|
|
165
|
+
workerSessionContextUsagePercent?: number;
|
|
166
|
+
workerSessionContextThresholdPercent?: number;
|
|
167
|
+
networkRecoveryEvent?: NetworkRecoveryEventType;
|
|
168
|
+
networkRetryCount?: number;
|
|
169
|
+
networkOutageElapsedMs?: number;
|
|
170
|
+
networkNextRetryAtMs?: number;
|
|
171
|
+
networkNextRetryInMs?: number;
|
|
172
|
+
networkFailureReason?: string;
|
|
127
173
|
}
|
|
128
174
|
|
|
129
175
|
export type CoordinatorProgressHandler = (update: CoordinatorProgressUpdate) => void;
|
|
@@ -146,12 +192,18 @@ export interface RunCoordinatorOptions extends PiLongTaskInput {
|
|
|
146
192
|
maxBashTimeoutMs?: number;
|
|
147
193
|
taskThinking?: string;
|
|
148
194
|
todoThinking?: string;
|
|
195
|
+
/** Set false to retain the legacy one-session-per-task lifecycle. */
|
|
196
|
+
workerSessionReuse?: boolean;
|
|
197
|
+
/** Rotate before another assignment when context usage reaches this percentage. */
|
|
198
|
+
workerSessionReuseContextThresholdPercent?: number;
|
|
149
199
|
now?: () => Date;
|
|
150
200
|
onProgress?: CoordinatorProgressHandler;
|
|
151
201
|
/** Run-scoped FIFO populated by the extension input handler during active execution. */
|
|
152
202
|
steeringQueue?: SerializedSteeringQueue;
|
|
153
203
|
/** Runs after rebase/validation and immediately before the revision is atomically persisted. */
|
|
154
204
|
onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
|
|
205
|
+
/** Receives coordinator-level outage lifecycle events for parent orchestrators and status integrations. */
|
|
206
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
155
207
|
}
|
|
156
208
|
|
|
157
209
|
export interface TodoPlannerOptions {
|
|
@@ -170,6 +222,8 @@ export interface TodoPlannerOptions {
|
|
|
170
222
|
plannerPrompt?: string;
|
|
171
223
|
/** Structured revision context supplied alongside plannerPrompt. */
|
|
172
224
|
planRevision?: Readonly<PlanRevisionRequest>;
|
|
225
|
+
/** Normalized coordinator recovery policy; network wait is excluded from operation timeouts. */
|
|
226
|
+
networkRecovery?: Readonly<NetworkRecoveryConfig>;
|
|
173
227
|
}
|
|
174
228
|
|
|
175
229
|
export interface TaskAttemptSummary {
|
|
@@ -190,6 +244,14 @@ export interface TaskAttemptSummary {
|
|
|
190
244
|
resultText?: string;
|
|
191
245
|
}
|
|
192
246
|
|
|
247
|
+
export interface WorkerSessionMetrics {
|
|
248
|
+
starts: number;
|
|
249
|
+
reuses: number;
|
|
250
|
+
rotations: number;
|
|
251
|
+
retained: number;
|
|
252
|
+
rotationReasons: Record<string, number>;
|
|
253
|
+
}
|
|
254
|
+
|
|
193
255
|
export interface CoordinatorResult {
|
|
194
256
|
status: CoordinatorStatus;
|
|
195
257
|
summary: string;
|
|
@@ -210,6 +272,10 @@ export interface CoordinatorResult {
|
|
|
210
272
|
attempts: TaskAttemptSummary[];
|
|
211
273
|
taskProgress: TaskProgressModel;
|
|
212
274
|
workerCostTotal: number;
|
|
275
|
+
/** Sum of task/attempt token deltas; omitted when statistics are unavailable. */
|
|
276
|
+
workerUsageTotal?: WorkerUsageTotals;
|
|
277
|
+
/** Additive lifecycle counters for adaptive worker-session reuse. */
|
|
278
|
+
workerSessionMetrics?: WorkerSessionMetrics;
|
|
213
279
|
commit: boolean;
|
|
214
280
|
goal?: string;
|
|
215
281
|
error?: string;
|
|
@@ -236,9 +302,13 @@ interface RuntimeOptions {
|
|
|
236
302
|
goal?: string;
|
|
237
303
|
taskThinking: string;
|
|
238
304
|
todoThinking: string;
|
|
305
|
+
workerSessionReuse: boolean;
|
|
306
|
+
workerSessionReuseContextThresholdPercent: number;
|
|
307
|
+
networkRecovery: NetworkRecoveryConfig;
|
|
239
308
|
todoTimeoutMs: number;
|
|
240
309
|
todoGracefulShutdownMs: number;
|
|
241
310
|
workerRunner: WorkerRunner;
|
|
311
|
+
useRetainedWorkerLifecycle: boolean;
|
|
242
312
|
todoPlanner: TodoPlanner;
|
|
243
313
|
abortSignal?: AbortSignal;
|
|
244
314
|
workerSessionFactory?: WorkerSessionFactory;
|
|
@@ -250,12 +320,636 @@ interface RuntimeOptions {
|
|
|
250
320
|
workerTextByWorker: Map<string, string>;
|
|
251
321
|
workerTextPublishedLengthByWorker: Map<string, number>;
|
|
252
322
|
plannerDiagnostics: PlannerDiagnostic[];
|
|
323
|
+
workerSessionMetrics: WorkerSessionMetrics;
|
|
253
324
|
steeringQueue?: SerializedSteeringQueue;
|
|
254
325
|
onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
|
|
326
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
327
|
+
lastProgress?: CoordinatorProgressUpdate;
|
|
328
|
+
progressClosed: boolean;
|
|
329
|
+
networkRecoverySequence: number;
|
|
330
|
+
activeNetworkRecoveries: Map<number, NetworkRecoveryEvent>;
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
type RetainedWorkerReuseScope = "sequential_task" | "partial_continuation";
|
|
334
|
+
|
|
335
|
+
export interface WorkerAssignmentIdentity {
|
|
336
|
+
/** Unique invocation identity, even when a replacement reuses a task ID and attempt number. */
|
|
337
|
+
assignmentId: string;
|
|
338
|
+
/** Stable task identity or semantic fingerprint from the plan that launched this assignment. */
|
|
339
|
+
taskIdentity: string;
|
|
340
|
+
/** Accepted-steering generation at the assignment boundary. */
|
|
341
|
+
steeringGeneration: number;
|
|
342
|
+
/** Structural plan generation from which the assignment was selected. */
|
|
343
|
+
planAuthorityToken: string;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
interface RetainedWorkerState {
|
|
347
|
+
resource: WorkerSessionResource;
|
|
348
|
+
compatibility: WorkerSessionCompatibilityFingerprint;
|
|
349
|
+
health: WorkerSessionHealth;
|
|
350
|
+
contextUsagePercent?: number;
|
|
351
|
+
previousTask: Pick<Task, "taskId" | "title">;
|
|
352
|
+
previousAttempt: number;
|
|
353
|
+
previousAssignmentIdentity: WorkerAssignmentIdentity;
|
|
354
|
+
reportDiagnostic: (diagnostic: WorkerSessionDiagnostic) => void;
|
|
355
|
+
reuseScope: RetainedWorkerReuseScope;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
interface ActiveWorkerSessionAssignment {
|
|
359
|
+
identity: WorkerAssignmentIdentity;
|
|
360
|
+
controller: AbortController;
|
|
361
|
+
tainted: boolean;
|
|
362
|
+
rotationReported: boolean;
|
|
363
|
+
resource?: WorkerSessionResource;
|
|
364
|
+
reportDiagnostic?: (diagnostic: WorkerSessionDiagnostic) => void;
|
|
365
|
+
resolveCompletion: () => void;
|
|
366
|
+
completion: Promise<void>;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
export interface CoordinatorWorkerSessionOwnerOptions {
|
|
370
|
+
runId: string;
|
|
371
|
+
cwd: string;
|
|
372
|
+
workerSessionReuse: boolean;
|
|
373
|
+
workerSessionReuseContextThresholdPercent: number;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/** Run-scoped owner for the single retained worker session and its assignment lock. */
|
|
377
|
+
export class CoordinatorWorkerSessionOwner {
|
|
378
|
+
private retained: RetainedWorkerState | undefined;
|
|
379
|
+
private active: ActiveWorkerSessionAssignment | undefined;
|
|
380
|
+
private closed = false;
|
|
381
|
+
private disposePromise: Promise<void> | undefined;
|
|
382
|
+
private assignmentSequence = 0;
|
|
383
|
+
private readonly runtime: CoordinatorWorkerSessionOwnerOptions;
|
|
384
|
+
|
|
385
|
+
constructor(runtime: CoordinatorWorkerSessionOwnerOptions) {
|
|
386
|
+
this.runtime = runtime;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
async run(options: RunWorkerTaskOptions, identity?: WorkerAssignmentIdentity): Promise<SessionOutcome> {
|
|
390
|
+
if (this.closed) {
|
|
391
|
+
throw new Error("retained worker session owner is closed");
|
|
392
|
+
}
|
|
393
|
+
const assignmentIdentity = identity ?? this.defaultIdentity(options);
|
|
394
|
+
if (this.active) {
|
|
395
|
+
throw new Error("retained worker session already has an active assignment");
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
let resolveCompletion: (() => void) | undefined;
|
|
399
|
+
const completion = new Promise<void>((resolve) => {
|
|
400
|
+
resolveCompletion = resolve;
|
|
401
|
+
});
|
|
402
|
+
const active: ActiveWorkerSessionAssignment = {
|
|
403
|
+
identity: assignmentIdentity,
|
|
404
|
+
controller: new AbortController(),
|
|
405
|
+
tainted: false,
|
|
406
|
+
rotationReported: false,
|
|
407
|
+
resolveCompletion: resolveCompletion as () => void,
|
|
408
|
+
completion,
|
|
409
|
+
};
|
|
410
|
+
this.active = active;
|
|
411
|
+
const assignmentOptions = {
|
|
412
|
+
...options,
|
|
413
|
+
abortSignal: combineAbortSignals(options.abortSignal, active.controller.signal),
|
|
414
|
+
};
|
|
415
|
+
try {
|
|
416
|
+
return await this.runExclusive(assignmentOptions, assignmentIdentity, active);
|
|
417
|
+
} finally {
|
|
418
|
+
const activeResource = active.resource;
|
|
419
|
+
const retained = this.retained;
|
|
420
|
+
if (active.tainted && activeResource && retained?.resource === activeResource) {
|
|
421
|
+
retained.health = "cancelled";
|
|
422
|
+
await this.disposeRetainedResource(activeResource);
|
|
423
|
+
}
|
|
424
|
+
if (this.active === active) {
|
|
425
|
+
this.active = undefined;
|
|
426
|
+
}
|
|
427
|
+
active.resolveCompletion();
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
private async runExclusive(
|
|
432
|
+
options: RunWorkerTaskOptions,
|
|
433
|
+
identity: WorkerAssignmentIdentity,
|
|
434
|
+
active: ActiveWorkerSessionAssignment,
|
|
435
|
+
): Promise<SessionOutcome> {
|
|
436
|
+
const compatibility = this.compatibilityFor(options);
|
|
437
|
+
const diagnostics: WorkerSessionDiagnostic[] = [];
|
|
438
|
+
const report = (diagnostic: WorkerSessionDiagnostic) => {
|
|
439
|
+
diagnostics.push(diagnostic);
|
|
440
|
+
options.onSessionDiagnostic?.(diagnostic);
|
|
441
|
+
};
|
|
442
|
+
active.reportDiagnostic = report;
|
|
443
|
+
const diagnosticContext = (retained: RetainedWorkerState) => ({
|
|
444
|
+
...(retained.contextUsagePercent !== undefined ? { contextUsagePercent: retained.contextUsagePercent } : {}),
|
|
445
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
446
|
+
previousTaskId: retained.previousTask.taskId,
|
|
447
|
+
});
|
|
448
|
+
|
|
449
|
+
let reusedFrom: Pick<Task, "taskId" | "title"> | undefined;
|
|
450
|
+
if (this.retained && !this.assignmentMatchesRetainedScope(options, this.retained)) {
|
|
451
|
+
// A partial-work session is continuity for exactly the next attempt of
|
|
452
|
+
// that task. It must never spill into unrelated work or a later retry.
|
|
453
|
+
report({
|
|
454
|
+
event: "session_rotated",
|
|
455
|
+
reasonCode: "partial_continuation_scope_mismatch",
|
|
456
|
+
...diagnosticContext(this.retained),
|
|
457
|
+
});
|
|
458
|
+
await this.disposeRetained();
|
|
459
|
+
}
|
|
460
|
+
if (this.retained) {
|
|
461
|
+
const decision = decideWorkerSessionReuse({
|
|
462
|
+
config: {
|
|
463
|
+
enabled: this.runtime.workerSessionReuse,
|
|
464
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
465
|
+
},
|
|
466
|
+
candidate: {
|
|
467
|
+
health: this.retained.health,
|
|
468
|
+
compatibility: this.retained.compatibility,
|
|
469
|
+
contextUsagePercent: this.retained.contextUsagePercent,
|
|
470
|
+
assignmentState: "idle",
|
|
471
|
+
disposed: this.retained.resource.disposed,
|
|
472
|
+
},
|
|
473
|
+
requestedCompatibility: compatibility,
|
|
474
|
+
});
|
|
475
|
+
if (decision.reusable) {
|
|
476
|
+
reusedFrom = this.retained.previousTask;
|
|
477
|
+
report({
|
|
478
|
+
event: "session_reused",
|
|
479
|
+
reasonCode: decision.reasonCode,
|
|
480
|
+
...diagnosticContext(this.retained),
|
|
481
|
+
});
|
|
482
|
+
} else {
|
|
483
|
+
report({
|
|
484
|
+
event: "session_rotated",
|
|
485
|
+
reasonCode: decision.reasonCode,
|
|
486
|
+
...diagnosticContext(this.retained),
|
|
487
|
+
});
|
|
488
|
+
await this.disposeRetained();
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
if (!this.retained) {
|
|
493
|
+
try {
|
|
494
|
+
this.retained = {
|
|
495
|
+
resource: await createWorkerSessionResource(options, options.sessionFactory ?? createIsolatedWorkerSession),
|
|
496
|
+
compatibility,
|
|
497
|
+
health: "healthy",
|
|
498
|
+
previousTask: options.task,
|
|
499
|
+
previousAttempt: options.attempt,
|
|
500
|
+
previousAssignmentIdentity: identity,
|
|
501
|
+
reportDiagnostic: report,
|
|
502
|
+
reuseScope: "sequential_task",
|
|
503
|
+
};
|
|
504
|
+
report({
|
|
505
|
+
event: "session_started",
|
|
506
|
+
reasonCode: diagnostics.some((item) => item.event === "session_rotated")
|
|
507
|
+
? "rotation_completed"
|
|
508
|
+
: "fresh_session",
|
|
509
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
510
|
+
});
|
|
511
|
+
} catch (error) {
|
|
512
|
+
const failed = buildWorkerSessionCreationFailureOutcome(options, error);
|
|
513
|
+
failed.sessionDiagnostics = diagnostics;
|
|
514
|
+
return failed;
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
active.resource = this.retained.resource;
|
|
519
|
+
let outcome: SessionOutcome;
|
|
520
|
+
try {
|
|
521
|
+
outcome = await runWorkerTaskAssignment(
|
|
522
|
+
options,
|
|
523
|
+
this.retained.resource,
|
|
524
|
+
reusedFrom ? { previousTask: reusedFrom } : undefined,
|
|
525
|
+
);
|
|
526
|
+
} catch (error) {
|
|
527
|
+
this.retained.health = active.tainted ? "cancelled" : "unrecoverable_error";
|
|
528
|
+
if (!active.rotationReported) {
|
|
529
|
+
active.rotationReported = true;
|
|
530
|
+
report({
|
|
531
|
+
event: "session_rotated",
|
|
532
|
+
reasonCode: "health_unrecoverable_error",
|
|
533
|
+
...diagnosticContext(this.retained),
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
await this.disposeRetained();
|
|
537
|
+
throw error;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
if (this.retained) {
|
|
541
|
+
const cancelled = Boolean(options.abortSignal?.aborted);
|
|
542
|
+
this.retained.health = workerSessionHealthForOutcome(outcome, cancelled);
|
|
543
|
+
this.retained.contextUsagePercent = await workerSessionContextUsagePercent(this.retained.resource.session);
|
|
544
|
+
this.retained.previousTask = options.task;
|
|
545
|
+
this.retained.previousAttempt = options.attempt;
|
|
546
|
+
this.retained.previousAssignmentIdentity = identity;
|
|
547
|
+
this.retained.reportDiagnostic = report;
|
|
548
|
+
|
|
549
|
+
if (active.tainted) {
|
|
550
|
+
if (!active.rotationReported) {
|
|
551
|
+
this.reportTaintedRotation(active, "assignment_cancelled");
|
|
552
|
+
}
|
|
553
|
+
await this.disposeRetainedResource(this.retained.resource);
|
|
554
|
+
outcome.sessionDiagnostics = [...(outcome.sessionDiagnostics ?? []), ...diagnostics];
|
|
555
|
+
return outcome;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
const retry = classifyWorkerSessionRetry({
|
|
559
|
+
done: outcome.done,
|
|
560
|
+
reportedStatus: outcome.reportedStatus,
|
|
561
|
+
completeTaskResult: hasCompleteTaskResult(outcome.assistantText),
|
|
562
|
+
timedOut: outcome.timedOut,
|
|
563
|
+
aborted: outcome.aborted,
|
|
564
|
+
cancelled,
|
|
565
|
+
error: outcome.error,
|
|
566
|
+
});
|
|
567
|
+
this.retained.reuseScope = retry.mayContinueInSession ? "partial_continuation" : "sequential_task";
|
|
568
|
+
|
|
569
|
+
const postAssignmentDecision = decideWorkerSessionReuse({
|
|
570
|
+
config: {
|
|
571
|
+
enabled: this.runtime.workerSessionReuse,
|
|
572
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
573
|
+
},
|
|
574
|
+
candidate: {
|
|
575
|
+
health: this.retained.health,
|
|
576
|
+
compatibility: this.retained.compatibility,
|
|
577
|
+
contextUsagePercent: this.retained.contextUsagePercent,
|
|
578
|
+
assignmentState: "idle",
|
|
579
|
+
disposed: this.retained.resource.disposed,
|
|
580
|
+
},
|
|
581
|
+
requestedCompatibility: compatibility,
|
|
582
|
+
});
|
|
583
|
+
// Completed tasks may flow into the next sequential TODO. A retry may
|
|
584
|
+
// remain only when it is an explicitly safe partial continuation and all
|
|
585
|
+
// normal health/compatibility/context checks still pass.
|
|
586
|
+
const rotateForRetry = !outcome.done && !retry.mayContinueInSession;
|
|
587
|
+
if (!postAssignmentDecision.reusable || rotateForRetry) {
|
|
588
|
+
if (!active.rotationReported) {
|
|
589
|
+
active.rotationReported = true;
|
|
590
|
+
report({
|
|
591
|
+
event: "session_rotated",
|
|
592
|
+
reasonCode: postAssignmentDecision.reusable ? retry.reasonCode : postAssignmentDecision.reasonCode,
|
|
593
|
+
...diagnosticContext(this.retained),
|
|
594
|
+
});
|
|
595
|
+
}
|
|
596
|
+
await this.disposeRetained();
|
|
597
|
+
} else {
|
|
598
|
+
report({
|
|
599
|
+
event: "session_retained",
|
|
600
|
+
reasonCode: postAssignmentDecision.reasonCode,
|
|
601
|
+
...diagnosticContext(this.retained),
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
outcome.sessionDiagnostics = [...(outcome.sessionDiagnostics ?? []), ...diagnostics];
|
|
606
|
+
return outcome;
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/**
|
|
610
|
+
* Taint and abort only the matching obsolete assignment. Late cancellation
|
|
611
|
+
* from an older steering generation cannot affect a replacement assignment.
|
|
612
|
+
*/
|
|
613
|
+
async invalidateAssignment(identity: WorkerAssignmentIdentity): Promise<boolean> {
|
|
614
|
+
const active = this.active;
|
|
615
|
+
if (active && sameWorkerAssignment(active.identity, identity)) {
|
|
616
|
+
this.taintActiveAssignment(active, "steering_revision_obsolete");
|
|
617
|
+
if (!active.controller.signal.aborted) {
|
|
618
|
+
active.controller.abort(new Error(`worker assignment ${identity.assignmentId} became obsolete`));
|
|
619
|
+
}
|
|
620
|
+
return true;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
if (this.retained && sameWorkerAssignment(this.retained.previousAssignmentIdentity, identity)) {
|
|
624
|
+
const retained = this.retained;
|
|
625
|
+
retained.health = "cancelled";
|
|
626
|
+
retained.reportDiagnostic({
|
|
627
|
+
event: "session_rotated",
|
|
628
|
+
reasonCode: "steering_revision_obsolete",
|
|
629
|
+
...this.diagnosticContext(retained),
|
|
630
|
+
});
|
|
631
|
+
await this.disposeRetained();
|
|
632
|
+
return true;
|
|
633
|
+
}
|
|
634
|
+
return false;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
/** Abort active work, wait for its ownership path, then dispose retained state once. */
|
|
638
|
+
dispose(): Promise<void> {
|
|
639
|
+
if (!this.disposePromise) {
|
|
640
|
+
this.closed = true;
|
|
641
|
+
this.disposePromise = this.disposeAfterActiveAssignment();
|
|
642
|
+
}
|
|
643
|
+
return this.disposePromise;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
private async disposeAfterActiveAssignment(): Promise<void> {
|
|
647
|
+
const active = this.active;
|
|
648
|
+
if (active) {
|
|
649
|
+
this.taintActiveAssignment(active, "coordinator_shutdown");
|
|
650
|
+
if (!active.controller.signal.aborted) {
|
|
651
|
+
active.controller.abort(new Error("worker session owner disposed"));
|
|
652
|
+
}
|
|
653
|
+
await active.completion;
|
|
654
|
+
}
|
|
655
|
+
await this.disposeRetained();
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
private assignmentMatchesRetainedScope(options: RunWorkerTaskOptions, retained: RetainedWorkerState): boolean {
|
|
659
|
+
if (retained.reuseScope === "sequential_task") {
|
|
660
|
+
return true;
|
|
661
|
+
}
|
|
662
|
+
return (
|
|
663
|
+
options.task.taskId === retained.previousTask.taskId &&
|
|
664
|
+
options.task.title === retained.previousTask.title &&
|
|
665
|
+
options.attempt === retained.previousAttempt + 1
|
|
666
|
+
);
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
private taintActiveAssignment(active: ActiveWorkerSessionAssignment, reasonCode: string): void {
|
|
670
|
+
active.tainted = true;
|
|
671
|
+
const retained = this.retained;
|
|
672
|
+
if (retained && retained.resource === active.resource) {
|
|
673
|
+
retained.health = "cancelled";
|
|
674
|
+
}
|
|
675
|
+
this.reportTaintedRotation(active, reasonCode);
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
private reportTaintedRotation(active: ActiveWorkerSessionAssignment, reasonCode: string): void {
|
|
679
|
+
if (active.rotationReported) return;
|
|
680
|
+
active.rotationReported = true;
|
|
681
|
+
const retained = this.retained;
|
|
682
|
+
active.reportDiagnostic?.({
|
|
683
|
+
event: "session_rotated",
|
|
684
|
+
reasonCode,
|
|
685
|
+
...(retained ? this.diagnosticContext(retained) : {}),
|
|
686
|
+
});
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
private diagnosticContext(retained: RetainedWorkerState): {
|
|
690
|
+
contextUsagePercent?: number;
|
|
691
|
+
contextThresholdPercent: number;
|
|
692
|
+
previousTaskId: string;
|
|
693
|
+
} {
|
|
694
|
+
return {
|
|
695
|
+
...(retained.contextUsagePercent !== undefined ? { contextUsagePercent: retained.contextUsagePercent } : {}),
|
|
696
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
697
|
+
previousTaskId: retained.previousTask.taskId,
|
|
698
|
+
};
|
|
699
|
+
}
|
|
700
|
+
|
|
701
|
+
private defaultIdentity(options: RunWorkerTaskOptions): WorkerAssignmentIdentity {
|
|
702
|
+
const sequence = ++this.assignmentSequence;
|
|
703
|
+
return {
|
|
704
|
+
assignmentId: `${options.task.taskId}:${options.attempt}:${sequence}`,
|
|
705
|
+
taskIdentity: `${options.task.taskId}:${options.task.title}`,
|
|
706
|
+
steeringGeneration: 0,
|
|
707
|
+
planAuthorityToken: "direct-owner",
|
|
708
|
+
};
|
|
709
|
+
}
|
|
710
|
+
|
|
711
|
+
private compatibilityFor(options: RunWorkerTaskOptions): WorkerSessionCompatibilityFingerprint {
|
|
712
|
+
return createWorkerSessionCompatibilityFingerprint({
|
|
713
|
+
coordinatorRunId: this.runtime.runId,
|
|
714
|
+
repositoryRoot: this.runtime.cwd,
|
|
715
|
+
worktreeRoot: options.cwd,
|
|
716
|
+
modelName: options.modelName,
|
|
717
|
+
model: options.model,
|
|
718
|
+
tools: options.tools ?? DEFAULT_WORKER_TOOLS,
|
|
719
|
+
thinkingLevel: options.thinkingLevel,
|
|
720
|
+
agentDir: options.agentDir,
|
|
721
|
+
modelRuntime: options.modelRuntime,
|
|
722
|
+
authStorage: options.authStorage,
|
|
723
|
+
modelRegistry: options.modelRegistry,
|
|
724
|
+
settingsManager: options.settingsManager,
|
|
725
|
+
resourceLoader: options.resourceLoader,
|
|
726
|
+
sessionFactory: options.sessionFactory ?? createIsolatedWorkerSession,
|
|
727
|
+
});
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
private async disposeRetained(): Promise<void> {
|
|
731
|
+
const retained = this.retained;
|
|
732
|
+
if (!retained) {
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
await this.disposeRetainedResource(retained.resource);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
private async disposeRetainedResource(resource: WorkerSessionResource): Promise<void> {
|
|
739
|
+
if (this.retained?.resource === resource) {
|
|
740
|
+
this.retained = undefined;
|
|
741
|
+
}
|
|
742
|
+
try {
|
|
743
|
+
await disposeWorkerSessionResource(resource);
|
|
744
|
+
} catch {
|
|
745
|
+
// Session disposal is best effort; resource ownership is still closed exactly once.
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
function sameWorkerAssignment(left: WorkerAssignmentIdentity, right: WorkerAssignmentIdentity): boolean {
|
|
751
|
+
return (
|
|
752
|
+
left.assignmentId === right.assignmentId &&
|
|
753
|
+
left.taskIdentity === right.taskIdentity &&
|
|
754
|
+
left.steeringGeneration === right.steeringGeneration &&
|
|
755
|
+
left.planAuthorityToken === right.planAuthorityToken
|
|
756
|
+
);
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined {
|
|
760
|
+
const available = signals.filter((signal): signal is AbortSignal => Boolean(signal));
|
|
761
|
+
if (available.length === 0) return undefined;
|
|
762
|
+
if (available.length === 1) return available[0];
|
|
763
|
+
return AbortSignal.any(available);
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
/** Retains the full worker result while exposing its provider error to the shared classifier. */
|
|
767
|
+
class WorkerNetworkFailure extends Error {
|
|
768
|
+
readonly outcome: SessionOutcome;
|
|
769
|
+
|
|
770
|
+
constructor(outcome: SessionOutcome) {
|
|
771
|
+
const message = outcome.error ?? "worker network operation failed";
|
|
772
|
+
super(message, { cause: workerFailureValue(outcome) });
|
|
773
|
+
this.name = "WorkerNetworkFailure";
|
|
774
|
+
this.outcome = outcome;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
interface WorkerRecoveryExecutionOptions {
|
|
779
|
+
workerOptions: RunWorkerTaskOptions;
|
|
780
|
+
run: (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
|
|
781
|
+
taskResultPath: string;
|
|
782
|
+
networkRecovery: Readonly<NetworkRecoveryConfig>;
|
|
783
|
+
signal?: AbortSignal;
|
|
784
|
+
onInterruption?: (outcome: SessionOutcome) => void;
|
|
785
|
+
onNetworkRecovery?: (event: NetworkRecoveryEvent) => void;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* Run one ordinary worker attempt, replacing only transport-failed sessions.
|
|
790
|
+
* Pi owns bounded request retries inside session.prompt(); therefore an outcome
|
|
791
|
+
* reaches this boundary only after those retries have settled. Coordinator
|
|
792
|
+
* probes retain the same task/attempt and always receive a recovery prompt.
|
|
793
|
+
*/
|
|
794
|
+
async function runWorkerAttemptWithNetworkRecovery(options: WorkerRecoveryExecutionOptions): Promise<SessionOutcome> {
|
|
795
|
+
const interrupted: SessionOutcome[] = [];
|
|
796
|
+
const thrownErrors = new Map<SessionOutcome, unknown>();
|
|
797
|
+
const execute = async (workerOptions: RunWorkerTaskOptions): Promise<SessionOutcome> => {
|
|
798
|
+
try {
|
|
799
|
+
return await options.run(workerOptions);
|
|
800
|
+
} catch (error) {
|
|
801
|
+
const outcome = buildWorkerSessionCreationFailureOutcome(workerOptions, error);
|
|
802
|
+
thrownErrors.set(outcome, error);
|
|
803
|
+
return outcome;
|
|
804
|
+
}
|
|
805
|
+
};
|
|
806
|
+
const recordRecoverableInterruption = async (outcome: SessionOutcome): Promise<void> => {
|
|
807
|
+
await appendNetworkInterruptionEvidence(options.taskResultPath, outcome, interrupted.length + 1);
|
|
808
|
+
interrupted.push(outcome);
|
|
809
|
+
options.onInterruption?.(outcome);
|
|
810
|
+
};
|
|
811
|
+
|
|
812
|
+
const initial = await execute(options.workerOptions);
|
|
813
|
+
const initialFailure = workerFailureValue(initial);
|
|
814
|
+
const initialClassification = initialFailure === undefined ? undefined : classifyNetworkFailure(initialFailure);
|
|
815
|
+
if (initialClassification && isFailFastWorkerFailure(initialClassification.reason)) {
|
|
816
|
+
throw new WorkerNetworkFailure(initial);
|
|
817
|
+
}
|
|
818
|
+
if (!options.networkRecovery.enabled || !initialClassification?.recoverable) {
|
|
819
|
+
if (thrownErrors.has(initial)) throw thrownErrors.get(initial);
|
|
820
|
+
return initial;
|
|
821
|
+
}
|
|
822
|
+
await recordRecoverableInterruption(initial);
|
|
823
|
+
|
|
824
|
+
try {
|
|
825
|
+
const recovered = await recoverNetworkOperation({
|
|
826
|
+
initialFailure: new WorkerNetworkFailure(initial),
|
|
827
|
+
config: options.networkRecovery,
|
|
828
|
+
signal: options.signal,
|
|
829
|
+
onEvent: options.onNetworkRecovery,
|
|
830
|
+
retry: async ({ retryCount, signal }) => {
|
|
831
|
+
const previous = interrupted.at(-1)!;
|
|
832
|
+
const resumed = await execute({
|
|
833
|
+
...options.workerOptions,
|
|
834
|
+
// The recovery signal includes both run cancellation and the outage
|
|
835
|
+
// deadline without replacing the assignment/steering cancellation.
|
|
836
|
+
abortSignal: combineAbortSignals(options.workerOptions.abortSignal, signal),
|
|
837
|
+
networkRecoveryContext: {
|
|
838
|
+
retryCount,
|
|
839
|
+
durableEvidencePath: options.taskResultPath,
|
|
840
|
+
priorSessionId: previous.sessionId,
|
|
841
|
+
failure: previous.error ?? "transient provider or transport failure",
|
|
842
|
+
},
|
|
843
|
+
});
|
|
844
|
+
const resumedFailure = workerFailureValue(resumed);
|
|
845
|
+
const classification = resumedFailure === undefined ? undefined : classifyNetworkFailure(resumedFailure);
|
|
846
|
+
if (classification?.recoverable) {
|
|
847
|
+
await recordRecoverableInterruption(resumed);
|
|
848
|
+
}
|
|
849
|
+
if (resumed.error) {
|
|
850
|
+
if (
|
|
851
|
+
thrownErrors.has(resumed) &&
|
|
852
|
+
!classification?.recoverable &&
|
|
853
|
+
!isFailFastWorkerFailure(classification!.reason)
|
|
854
|
+
) {
|
|
855
|
+
throw thrownErrors.get(resumed);
|
|
856
|
+
}
|
|
857
|
+
throw new WorkerNetworkFailure(resumed);
|
|
858
|
+
}
|
|
859
|
+
return resumed;
|
|
860
|
+
},
|
|
861
|
+
});
|
|
862
|
+
return mergeWorkerRecoveryOutcomes(interrupted, recovered.value);
|
|
863
|
+
} catch (error) {
|
|
864
|
+
// If connectivity recovered but the fresh session failed deterministically,
|
|
865
|
+
// hand its outcome back to the ordinary worker failure path immediately.
|
|
866
|
+
if (error instanceof WorkerNetworkFailure) {
|
|
867
|
+
const merged = mergeWorkerRecoveryOutcomes(interrupted, error.outcome);
|
|
868
|
+
const mergedFailure = workerFailureValue(merged);
|
|
869
|
+
const classification = mergedFailure === undefined ? undefined : classifyNetworkFailure(mergedFailure);
|
|
870
|
+
if (classification && isFailFastWorkerFailure(classification.reason)) {
|
|
871
|
+
throw new WorkerNetworkFailure(merged);
|
|
872
|
+
}
|
|
873
|
+
return merged;
|
|
874
|
+
}
|
|
875
|
+
throw error;
|
|
876
|
+
}
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
function workerFailureValue(outcome: SessionOutcome): unknown {
|
|
880
|
+
return outcome.failure ?? outcome.error;
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
function isFailFastWorkerFailure(reason: ReturnType<typeof classifyNetworkFailure>["reason"]): boolean {
|
|
884
|
+
return [
|
|
885
|
+
"authentication",
|
|
886
|
+
"authorization",
|
|
887
|
+
"billing",
|
|
888
|
+
"quota_exhausted",
|
|
889
|
+
"invalid_model",
|
|
890
|
+
"invalid_request",
|
|
891
|
+
"http_client_error",
|
|
892
|
+
"non_retryable_server_error",
|
|
893
|
+
].includes(reason);
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
function mergeWorkerRecoveryOutcomes(interrupted: readonly SessionOutcome[], final: SessionOutcome): SessionOutcome {
|
|
897
|
+
if (interrupted.length === 0) return final;
|
|
898
|
+
const usage = addWorkerUsage([...interrupted.map((item) => item.workerUsage), final.workerUsage]);
|
|
899
|
+
return {
|
|
900
|
+
...final,
|
|
901
|
+
startedAt: interrupted[0].startedAt,
|
|
902
|
+
contextObservations: [
|
|
903
|
+
...interrupted.flatMap((item, index) => [
|
|
904
|
+
`network interruption ${index + 1}: ${item.error ?? "transient provider or transport failure"}`,
|
|
905
|
+
...item.contextObservations,
|
|
906
|
+
]),
|
|
907
|
+
...final.contextObservations,
|
|
908
|
+
],
|
|
909
|
+
compactionEvents: [...interrupted.flatMap((item) => item.compactionEvents), ...final.compactionEvents],
|
|
910
|
+
events: [...interrupted.flatMap((item) => item.events), ...final.events],
|
|
911
|
+
workerCostTotal: [...interrupted, final].reduce((total, item) => total + item.workerCostTotal, 0),
|
|
912
|
+
workerCostSource: "network_recovery_aggregate",
|
|
913
|
+
workerUsage: usage,
|
|
914
|
+
sessionDiagnostics: [
|
|
915
|
+
...interrupted.flatMap((item) => item.sessionDiagnostics ?? []),
|
|
916
|
+
...(final.sessionDiagnostics ?? []),
|
|
917
|
+
],
|
|
918
|
+
};
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
function addWorkerUsage(values: Array<WorkerUsageTotals | undefined>): WorkerUsageTotals | undefined {
|
|
922
|
+
const available = values.filter((value): value is WorkerUsageTotals => Boolean(value));
|
|
923
|
+
if (available.length === 0) return undefined;
|
|
924
|
+
return available.reduce<WorkerUsageTotals>(
|
|
925
|
+
(total, value) => ({
|
|
926
|
+
input: total.input + value.input,
|
|
927
|
+
output: total.output + value.output,
|
|
928
|
+
cacheRead: total.cacheRead + value.cacheRead,
|
|
929
|
+
cacheWrite: total.cacheWrite + value.cacheWrite,
|
|
930
|
+
total: total.total + value.total,
|
|
931
|
+
}),
|
|
932
|
+
{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
933
|
+
);
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
export function workerSessionHealthForOutcome(
|
|
937
|
+
outcome: Pick<SessionOutcome, "timedOut" | "aborted" | "error" | "assistantText">,
|
|
938
|
+
cancelled = false,
|
|
939
|
+
): WorkerSessionHealth {
|
|
940
|
+
if (outcome.timedOut) return "timed_out";
|
|
941
|
+
if (cancelled) return "cancelled";
|
|
942
|
+
if (outcome.aborted) return "aborted";
|
|
943
|
+
if (outcome.error) return "unrecoverable_error";
|
|
944
|
+
if (!hasCompleteTaskResult(outcome.assistantText)) return "invalid_state";
|
|
945
|
+
return "healthy";
|
|
255
946
|
}
|
|
256
947
|
|
|
257
948
|
export async function runCoordinator(options: RunCoordinatorOptions): Promise<CoordinatorResult> {
|
|
258
949
|
const runtime = buildRuntimeOptions(options);
|
|
950
|
+
const workerSessionOwner = runtime.useRetainedWorkerLifecycle
|
|
951
|
+
? new CoordinatorWorkerSessionOwner(runtime)
|
|
952
|
+
: undefined;
|
|
259
953
|
const inputText = coordinatorInputText(options);
|
|
260
954
|
const attempts: TaskAttemptSummary[] = [];
|
|
261
955
|
const outcomes: SessionOutcome[] = [];
|
|
@@ -269,6 +963,12 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
269
963
|
let activeTask: Task | undefined;
|
|
270
964
|
let activeTaskReference: PlanTaskReference | undefined;
|
|
271
965
|
let activeAttempt: number | undefined;
|
|
966
|
+
let activeWorkerAssignment:
|
|
967
|
+
| { identity: WorkerAssignmentIdentity; controller: AbortController; obsolete: boolean }
|
|
968
|
+
| undefined;
|
|
969
|
+
let steeringGeneration = 0;
|
|
970
|
+
let workerExecutionSequence = 0;
|
|
971
|
+
let removeSteeringProcessor: (() => void) | undefined;
|
|
272
972
|
const protectedDirtyPathsByTask = new Map<string, Set<string>>();
|
|
273
973
|
|
|
274
974
|
try {
|
|
@@ -287,7 +987,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
287
987
|
});
|
|
288
988
|
|
|
289
989
|
let failure: string | undefined;
|
|
290
|
-
runtime.steeringQueue?.setProcessor(async (message) => {
|
|
990
|
+
removeSteeringProcessor = runtime.steeringQueue?.setProcessor(async (message) => {
|
|
291
991
|
const baseAtRequest = planStore.snapshot();
|
|
292
992
|
const activeTaskAtRequest = activeTaskReference
|
|
293
993
|
? resolvePlanTaskReference(
|
|
@@ -320,6 +1020,25 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
320
1020
|
// boundary, so this accepted revision continues the same run.
|
|
321
1021
|
latestTodoMarkdown = appliedRevision.todoMarkdown;
|
|
322
1022
|
latestTasks = appliedRevision.reconciliation.activeTasks.map((item) => item.task);
|
|
1023
|
+
steeringGeneration += 1;
|
|
1024
|
+
|
|
1025
|
+
// Once replacement/removal is authoritative, make the exact old
|
|
1026
|
+
// invocation obsolete before aborting it. Event callbacks consult this
|
|
1027
|
+
// identity, so a late old result cannot repaint replacement progress.
|
|
1028
|
+
const assignmentAtAcceptance = activeWorkerAssignment;
|
|
1029
|
+
const activeStillValid = activeTaskReference
|
|
1030
|
+
? Boolean(resolvePlanTaskReference(latestTasks, activeTaskReference, planStore.snapshot().authorityToken))
|
|
1031
|
+
: true;
|
|
1032
|
+
if (assignmentAtAcceptance && !activeStillValid) {
|
|
1033
|
+
assignmentAtAcceptance.obsolete = true;
|
|
1034
|
+
if (!assignmentAtAcceptance.controller.signal.aborted) {
|
|
1035
|
+
assignmentAtAcceptance.controller.abort(
|
|
1036
|
+
new Error(`steering revision ${message.sequence} replaced the active assignment`),
|
|
1037
|
+
);
|
|
1038
|
+
}
|
|
1039
|
+
await workerSessionOwner?.invalidateAssignment(assignmentAtAcceptance.identity);
|
|
1040
|
+
}
|
|
1041
|
+
|
|
323
1042
|
emitProgress(
|
|
324
1043
|
runtime,
|
|
325
1044
|
`Accepted steering revision ${message.sequence} with ${appliedRevision.reconciliation.activeTasks.length} task(s).`,
|
|
@@ -365,6 +1084,23 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
365
1084
|
const initialActivity =
|
|
366
1085
|
nextTask.statusItems.find((item) => !item.done)?.text ?? `Starting TODO ${nextTask.taskId}`;
|
|
367
1086
|
const worker = workerKey(nextTask.taskId, attempt);
|
|
1087
|
+
// Task IDs and attempt numbers may be reused after an in-flight task is
|
|
1088
|
+
// replaced by steering. Accounting needs an invocation identity so the
|
|
1089
|
+
// obsolete attempt's finalized spend cannot be overwritten.
|
|
1090
|
+
const accountingWorker = `${worker}#${++workerExecutionSequence}`;
|
|
1091
|
+
const assignmentIdentity: WorkerAssignmentIdentity = {
|
|
1092
|
+
assignmentId: accountingWorker,
|
|
1093
|
+
taskIdentity: nextTask.stableId ?? taskSemanticFingerprint(nextTask),
|
|
1094
|
+
steeringGeneration,
|
|
1095
|
+
planAuthorityToken: schedulingSnapshot.authorityToken,
|
|
1096
|
+
};
|
|
1097
|
+
const assignmentController = new AbortController();
|
|
1098
|
+
const assignmentState = { identity: assignmentIdentity, controller: assignmentController, obsolete: false };
|
|
1099
|
+
const taskPlanReference = planTaskReference(nextTask, schedulingSnapshot.authorityToken);
|
|
1100
|
+
activeWorkerAssignment = assignmentState;
|
|
1101
|
+
activeTask = nextTask;
|
|
1102
|
+
activeAttempt = attempt;
|
|
1103
|
+
activeTaskReference = taskPlanReference;
|
|
368
1104
|
runtime.workerActivityByWorker.set(worker, initialActivity);
|
|
369
1105
|
runtime.workerTextByWorker.delete(worker);
|
|
370
1106
|
runtime.workerTextPublishedLengthByWorker.delete(worker);
|
|
@@ -393,11 +1129,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
393
1129
|
: new Set<string>();
|
|
394
1130
|
protectedDirtyPathsByTask.set(executionIdentity, preExistingDirtyPaths);
|
|
395
1131
|
}
|
|
396
|
-
|
|
397
|
-
activeAttempt = attempt;
|
|
398
|
-
const taskPlanReference = planTaskReference(nextTask, schedulingSnapshot.authorityToken);
|
|
399
|
-
activeTaskReference = taskPlanReference;
|
|
400
|
-
const outcome = await runtime.workerRunner({
|
|
1132
|
+
const workerOptions: RunWorkerTaskOptions = {
|
|
401
1133
|
cwd: runtime.cwd,
|
|
402
1134
|
todoPath: runtime.todoPath,
|
|
403
1135
|
task: nextTask,
|
|
@@ -415,12 +1147,62 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
415
1147
|
model: runtime.workerModel,
|
|
416
1148
|
modelName: runtime.workerModelName,
|
|
417
1149
|
thinkingLevel: runtime.taskThinking,
|
|
418
|
-
abortSignal: runtime.abortSignal,
|
|
1150
|
+
abortSignal: combineAbortSignals(runtime.abortSignal, assignmentController.signal),
|
|
419
1151
|
sessionFactory: runtime.workerSessionFactory,
|
|
1152
|
+
networkRecovery: runtime.networkRecovery,
|
|
420
1153
|
now: runtime.now,
|
|
421
|
-
onEvent: (event) =>
|
|
422
|
-
|
|
423
|
-
|
|
1154
|
+
onEvent: (event) => {
|
|
1155
|
+
if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
|
|
1156
|
+
emitWorkerEventProgress(runtime, tasksBeforeAttempt, nextTask, attempts, attempt, event, accountingWorker);
|
|
1157
|
+
}
|
|
1158
|
+
},
|
|
1159
|
+
onSessionDiagnostic: (diagnostic) => {
|
|
1160
|
+
if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
|
|
1161
|
+
emitWorkerSessionProgress(runtime, tasksBeforeAttempt, nextTask, attempts, attempt, diagnostic);
|
|
1162
|
+
} else {
|
|
1163
|
+
// Lifecycle accounting remains accurate, but obsolete diagnostics
|
|
1164
|
+
// must not mutate the replacement task's visible progress.
|
|
1165
|
+
recordWorkerSessionMetric(runtime.workerSessionMetrics, diagnostic);
|
|
1166
|
+
}
|
|
1167
|
+
},
|
|
1168
|
+
};
|
|
1169
|
+
let outcome: SessionOutcome;
|
|
1170
|
+
const networkInterruptedOutcomes: SessionOutcome[] = [];
|
|
1171
|
+
try {
|
|
1172
|
+
outcome = await runWorkerAttemptWithNetworkRecovery({
|
|
1173
|
+
workerOptions,
|
|
1174
|
+
run: (resumedOptions) =>
|
|
1175
|
+
workerSessionOwner
|
|
1176
|
+
? workerSessionOwner.run(resumedOptions, assignmentIdentity)
|
|
1177
|
+
: runtime.workerRunner(resumedOptions),
|
|
1178
|
+
taskResultPath: runtime.taskResultPath,
|
|
1179
|
+
networkRecovery: runtime.networkRecovery,
|
|
1180
|
+
signal: workerOptions.abortSignal,
|
|
1181
|
+
onInterruption: (interrupted) => networkInterruptedOutcomes.push(interrupted),
|
|
1182
|
+
onNetworkRecovery: createNetworkRecoveryProgressHandler(runtime),
|
|
1183
|
+
});
|
|
1184
|
+
} catch (error) {
|
|
1185
|
+
if (!assignmentState.obsolete) {
|
|
1186
|
+
const terminalOutcome = error instanceof WorkerNetworkFailure ? error.outcome : undefined;
|
|
1187
|
+
if (terminalOutcome || networkInterruptedOutcomes.length > 0) {
|
|
1188
|
+
finalizeWorkerCost(runtime.workerCostState, accountingWorker, {
|
|
1189
|
+
workerCostTotal:
|
|
1190
|
+
terminalOutcome?.workerCostTotal ??
|
|
1191
|
+
networkInterruptedOutcomes.reduce((total, interrupted) => total + interrupted.workerCostTotal, 0),
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
throw error;
|
|
1195
|
+
}
|
|
1196
|
+
// A cancellation-aware custom runner may reject instead of returning
|
|
1197
|
+
// an aborted outcome. Preserve historical evidence, but never let that
|
|
1198
|
+
// obsolete rejection terminate or update the replacement assignment.
|
|
1199
|
+
outcome = mergeWorkerRecoveryOutcomes(
|
|
1200
|
+
networkInterruptedOutcomes,
|
|
1201
|
+
buildWorkerSessionCreationFailureOutcome(workerOptions, error),
|
|
1202
|
+
);
|
|
1203
|
+
outcome.aborted = true;
|
|
1204
|
+
}
|
|
1205
|
+
finalizeWorkerCost(runtime.workerCostState, accountingWorker, outcome);
|
|
424
1206
|
|
|
425
1207
|
// A revision may have been accepted while the worker was running. Let all
|
|
426
1208
|
// already-received guidance settle, then resolve this exact task identity
|
|
@@ -465,6 +1247,9 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
465
1247
|
activeTask = undefined;
|
|
466
1248
|
activeTaskReference = undefined;
|
|
467
1249
|
activeAttempt = undefined;
|
|
1250
|
+
if (activeWorkerAssignment === assignmentState) {
|
|
1251
|
+
activeWorkerAssignment = undefined;
|
|
1252
|
+
}
|
|
468
1253
|
|
|
469
1254
|
let taskCommitHash: string | undefined;
|
|
470
1255
|
let taskCommitError: string | undefined;
|
|
@@ -587,6 +1372,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
587
1372
|
attempts,
|
|
588
1373
|
taskProgress,
|
|
589
1374
|
workerCostTotal: runtime.workerCostState.total,
|
|
1375
|
+
workerUsageTotal: aggregateWorkerUsage(outcomes),
|
|
1376
|
+
workerSessionMetrics: snapshotWorkerSessionMetrics(runtime.workerSessionMetrics),
|
|
590
1377
|
commit: options.commit,
|
|
591
1378
|
goal: runtime.goal,
|
|
592
1379
|
error: failure,
|
|
@@ -668,6 +1455,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
668
1455
|
attempts,
|
|
669
1456
|
taskProgress,
|
|
670
1457
|
workerCostTotal: runtime.workerCostState.total,
|
|
1458
|
+
workerUsageTotal: aggregateWorkerUsage(outcomes),
|
|
1459
|
+
workerSessionMetrics: snapshotWorkerSessionMetrics(runtime.workerSessionMetrics),
|
|
671
1460
|
commit: options.commit,
|
|
672
1461
|
goal: runtime.goal,
|
|
673
1462
|
error: resultError,
|
|
@@ -680,6 +1469,11 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
680
1469
|
taskProgress,
|
|
681
1470
|
});
|
|
682
1471
|
return result;
|
|
1472
|
+
} finally {
|
|
1473
|
+
runtime.progressClosed = true;
|
|
1474
|
+
runtime.activeNetworkRecoveries.clear();
|
|
1475
|
+
removeSteeringProcessor?.();
|
|
1476
|
+
await workerSessionOwner?.dispose();
|
|
683
1477
|
}
|
|
684
1478
|
}
|
|
685
1479
|
|
|
@@ -749,19 +1543,59 @@ async function extractTodoMarkdownWithOneRepair(
|
|
|
749
1543
|
}
|
|
750
1544
|
|
|
751
1545
|
async function requestTodoPlan(inputText: string, runtime: RuntimeOptions): Promise<string> {
|
|
752
|
-
return
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
1546
|
+
return runPlannerOperationWithNetworkRecovery(
|
|
1547
|
+
{
|
|
1548
|
+
inputText,
|
|
1549
|
+
cwd: runtime.cwd,
|
|
1550
|
+
runDir: runtime.runDir,
|
|
1551
|
+
thinkingLevel: runtime.todoThinking,
|
|
1552
|
+
model: runtime.workerModel,
|
|
1553
|
+
abortSignal: runtime.abortSignal,
|
|
1554
|
+
timeoutMs: runtime.todoTimeoutMs,
|
|
1555
|
+
gracefulShutdownMs: runtime.todoGracefulShutdownMs,
|
|
1556
|
+
sessionFactory: runtime.todoSessionFactory,
|
|
1557
|
+
networkRecovery: runtime.networkRecovery,
|
|
1558
|
+
onDiagnostic: (diagnostic) => recordPlannerDiagnostic(runtime, diagnostic),
|
|
1559
|
+
goal: runtime.goal,
|
|
1560
|
+
},
|
|
1561
|
+
runtime,
|
|
1562
|
+
);
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
/**
|
|
1566
|
+
* Retry a side-effect-free planner request only after its provider boundary has
|
|
1567
|
+
* failed. The default planner disables tools and disposes every session before
|
|
1568
|
+
* rejecting, so each retry rotates unsafe conversation state while replaying
|
|
1569
|
+
* only the complete immutable planning context. Recovery owns no planner
|
|
1570
|
+
* repair/attempt counter, and each fresh call retains the planner timeout;
|
|
1571
|
+
* backoff remains governed solely by the separate outage deadline.
|
|
1572
|
+
*/
|
|
1573
|
+
async function runPlannerOperationWithNetworkRecovery(
|
|
1574
|
+
plannerOptions: TodoPlannerOptions,
|
|
1575
|
+
runtime: RuntimeOptions,
|
|
1576
|
+
): Promise<string> {
|
|
1577
|
+
const run = (recoverySignal?: AbortSignal) =>
|
|
1578
|
+
runtime.todoPlanner({
|
|
1579
|
+
...plannerOptions,
|
|
1580
|
+
abortSignal: combineAbortSignals(plannerOptions.abortSignal, recoverySignal),
|
|
1581
|
+
});
|
|
1582
|
+
|
|
1583
|
+
try {
|
|
1584
|
+
return await run();
|
|
1585
|
+
} catch (initialFailure) {
|
|
1586
|
+
const classification = classifyNetworkFailure(initialFailure);
|
|
1587
|
+
if (!runtime.networkRecovery.enabled || !classification.recoverable) {
|
|
1588
|
+
throw initialFailure;
|
|
1589
|
+
}
|
|
1590
|
+
const recovered = await recoverNetworkOperation({
|
|
1591
|
+
initialFailure,
|
|
1592
|
+
config: runtime.networkRecovery,
|
|
1593
|
+
signal: plannerOptions.abortSignal,
|
|
1594
|
+
onEvent: createNetworkRecoveryProgressHandler(runtime),
|
|
1595
|
+
retry: ({ signal }) => run(signal),
|
|
1596
|
+
});
|
|
1597
|
+
return recovered.value;
|
|
1598
|
+
}
|
|
765
1599
|
}
|
|
766
1600
|
|
|
767
1601
|
async function generateSteeringPlanRevision(options: {
|
|
@@ -794,21 +1628,25 @@ async function generateSteeringPlanRevision(options: {
|
|
|
794
1628
|
}
|
|
795
1629
|
: undefined,
|
|
796
1630
|
planner: ({ prompt, request }) =>
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
1631
|
+
runPlannerOperationWithNetworkRecovery(
|
|
1632
|
+
{
|
|
1633
|
+
inputText: prompt,
|
|
1634
|
+
plannerPrompt: prompt,
|
|
1635
|
+
planRevision: request,
|
|
1636
|
+
cwd: options.runtime.cwd,
|
|
1637
|
+
runDir: options.runtime.runDir,
|
|
1638
|
+
thinkingLevel: options.runtime.todoThinking,
|
|
1639
|
+
model: options.runtime.workerModel,
|
|
1640
|
+
abortSignal: options.runtime.abortSignal,
|
|
1641
|
+
timeoutMs: options.runtime.todoTimeoutMs,
|
|
1642
|
+
gracefulShutdownMs: options.runtime.todoGracefulShutdownMs,
|
|
1643
|
+
sessionFactory: options.runtime.todoSessionFactory,
|
|
1644
|
+
networkRecovery: options.runtime.networkRecovery,
|
|
1645
|
+
onDiagnostic: (diagnostic) => recordPlannerDiagnostic(options.runtime, diagnostic),
|
|
1646
|
+
goal: options.runtime.goal,
|
|
1647
|
+
},
|
|
1648
|
+
options.runtime,
|
|
1649
|
+
),
|
|
812
1650
|
});
|
|
813
1651
|
}
|
|
814
1652
|
|
|
@@ -1031,7 +1869,10 @@ async function runTodoPlannerPrompt(options: {
|
|
|
1031
1869
|
if (promptResult.error) {
|
|
1032
1870
|
const message = `TODO planner failed: ${promptResult.error}`;
|
|
1033
1871
|
options.onDiagnostic?.(plannerPromptDiagnostic("failure", message, promptResult));
|
|
1034
|
-
throw new TodoGenerationError(
|
|
1872
|
+
throw new TodoGenerationError(
|
|
1873
|
+
message,
|
|
1874
|
+
promptResult.failure === undefined ? undefined : { cause: promptResult.failure },
|
|
1875
|
+
);
|
|
1035
1876
|
}
|
|
1036
1877
|
if (!promptResult.assistantText) {
|
|
1037
1878
|
const message = "TODO planner did not return assistant text.";
|
|
@@ -1077,6 +1918,15 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1077
1918
|
const workerModelName = options.workerModelName ?? parsedWorkerConfig.modelName;
|
|
1078
1919
|
const workerModel = workerModelName ? undefined : options.workerModel;
|
|
1079
1920
|
const goal = normalizeOptionalText(options.goal);
|
|
1921
|
+
const workerSessionReuseConfig = resolveWorkerSessionReuseConfig({
|
|
1922
|
+
enabled: options.workerSessionReuse ?? parsedWorkerConfig.workerSessionReuseEnabled,
|
|
1923
|
+
contextThresholdPercent:
|
|
1924
|
+
options.workerSessionReuseContextThresholdPercent ?? parsedWorkerConfig.workerSessionReuseContextThresholdPercent,
|
|
1925
|
+
});
|
|
1926
|
+
const networkRecovery = resolveNetworkRecoveryConfig({
|
|
1927
|
+
...parsedWorkerConfig.networkRecovery,
|
|
1928
|
+
...options.networkRecovery,
|
|
1929
|
+
});
|
|
1080
1930
|
|
|
1081
1931
|
return {
|
|
1082
1932
|
cwd,
|
|
@@ -1098,7 +1948,11 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1098
1948
|
goal,
|
|
1099
1949
|
taskThinking: options.taskThinking ?? DEFAULT_COORDINATOR_OPTIONS.taskThinking,
|
|
1100
1950
|
todoThinking: options.todoThinking ?? DEFAULT_COORDINATOR_OPTIONS.todoThinking,
|
|
1951
|
+
workerSessionReuse: workerSessionReuseConfig.enabled,
|
|
1952
|
+
workerSessionReuseContextThresholdPercent: workerSessionReuseConfig.contextThresholdPercent,
|
|
1953
|
+
networkRecovery,
|
|
1101
1954
|
workerRunner: options.workerRunner ?? runWorkerTask,
|
|
1955
|
+
useRetainedWorkerLifecycle: options.workerRunner === undefined,
|
|
1102
1956
|
todoPlanner: options.todoPlanner ?? runTodoPlanner,
|
|
1103
1957
|
abortSignal: options.abortSignal,
|
|
1104
1958
|
workerSessionFactory: options.workerSessionFactory,
|
|
@@ -1110,8 +1964,13 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1110
1964
|
workerTextByWorker: new Map(),
|
|
1111
1965
|
workerTextPublishedLengthByWorker: new Map(),
|
|
1112
1966
|
plannerDiagnostics: [],
|
|
1967
|
+
workerSessionMetrics: createWorkerSessionMetrics(),
|
|
1113
1968
|
steeringQueue: options.steeringQueue,
|
|
1114
1969
|
onPlanRevisionAccepted: options.onPlanRevisionAccepted,
|
|
1970
|
+
onNetworkRecovery: options.onNetworkRecovery,
|
|
1971
|
+
progressClosed: false,
|
|
1972
|
+
networkRecoverySequence: 0,
|
|
1973
|
+
activeNetworkRecoveries: new Map(),
|
|
1115
1974
|
};
|
|
1116
1975
|
}
|
|
1117
1976
|
|
|
@@ -1120,7 +1979,8 @@ function emitProgress(
|
|
|
1120
1979
|
message: string,
|
|
1121
1980
|
update: Omit<CoordinatorProgressUpdate, "message" | "runId" | "todoPath" | "resultPath" | "workerCostTotal">,
|
|
1122
1981
|
): void {
|
|
1123
|
-
runtime.
|
|
1982
|
+
if (runtime.progressClosed) return;
|
|
1983
|
+
const progress: CoordinatorProgressUpdate = {
|
|
1124
1984
|
message,
|
|
1125
1985
|
runId: runtime.runId,
|
|
1126
1986
|
todoPath: runtime.todoPath,
|
|
@@ -1128,9 +1988,107 @@ function emitProgress(
|
|
|
1128
1988
|
workerCostTotal: runtime.workerCostState.total,
|
|
1129
1989
|
...update,
|
|
1130
1990
|
goal: runtime.goal,
|
|
1991
|
+
};
|
|
1992
|
+
runtime.lastProgress = progress;
|
|
1993
|
+
const activeRecovery = latestNetworkRecovery(runtime.activeNetworkRecoveries);
|
|
1994
|
+
if (activeRecovery) {
|
|
1995
|
+
publishNetworkRecoveryProgress(runtime, activeRecovery);
|
|
1996
|
+
} else {
|
|
1997
|
+
runtime.onProgress?.(progress);
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
/**
|
|
2002
|
+
* Bridge one recovery lifecycle into coordinator progress without replacing the
|
|
2003
|
+
* last stable task/phase update. A recovered operation restores whichever
|
|
2004
|
+
* ordinary status is current; terminal failures are left for the normal final
|
|
2005
|
+
* status path. Operation IDs prevent an older concurrent recovery from
|
|
2006
|
+
* repainting a newer outage or completion.
|
|
2007
|
+
*/
|
|
2008
|
+
function createNetworkRecoveryProgressHandler(runtime: RuntimeOptions): (event: NetworkRecoveryEvent) => void {
|
|
2009
|
+
const operationId = ++runtime.networkRecoverySequence;
|
|
2010
|
+
let cleaned = false;
|
|
2011
|
+
|
|
2012
|
+
return (event) => {
|
|
2013
|
+
runtime.onNetworkRecovery?.(event);
|
|
2014
|
+
if (cleaned || runtime.progressClosed) return;
|
|
2015
|
+
|
|
2016
|
+
if (event.type === "cleanup") {
|
|
2017
|
+
cleaned = true;
|
|
2018
|
+
runtime.activeNetworkRecoveries.delete(operationId);
|
|
2019
|
+
return;
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
if (isTerminalNetworkRecoveryEvent(event.type)) {
|
|
2023
|
+
runtime.activeNetworkRecoveries.delete(operationId);
|
|
2024
|
+
if (event.type === "recovered") {
|
|
2025
|
+
const active = latestNetworkRecovery(runtime.activeNetworkRecoveries);
|
|
2026
|
+
if (active) {
|
|
2027
|
+
publishNetworkRecoveryProgress(runtime, active);
|
|
2028
|
+
} else if (runtime.lastProgress) {
|
|
2029
|
+
runtime.onProgress?.({ ...runtime.lastProgress, workerCostTotal: runtime.workerCostState.total });
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
2032
|
+
return;
|
|
2033
|
+
}
|
|
2034
|
+
|
|
2035
|
+
runtime.activeNetworkRecoveries.set(operationId, event);
|
|
2036
|
+
if (operationId === latestNetworkRecoveryId(runtime.activeNetworkRecoveries)) {
|
|
2037
|
+
publishNetworkRecoveryProgress(runtime, event);
|
|
2038
|
+
}
|
|
2039
|
+
};
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
function publishNetworkRecoveryProgress(runtime: RuntimeOptions, event: NetworkRecoveryEvent): void {
|
|
2043
|
+
if (runtime.progressClosed) return;
|
|
2044
|
+
const stable = runtime.lastProgress;
|
|
2045
|
+
const nowMs = event.state.outageStartedAtMs + event.state.elapsedMs;
|
|
2046
|
+
const message = formatNetworkRecoveryStatus(event);
|
|
2047
|
+
runtime.onProgress?.({
|
|
2048
|
+
message,
|
|
2049
|
+
phase: "network_wait",
|
|
2050
|
+
runId: runtime.runId,
|
|
2051
|
+
todoPath: runtime.todoPath,
|
|
2052
|
+
resultPath: runtime.taskResultPath,
|
|
2053
|
+
workerCostTotal: runtime.workerCostState.total,
|
|
2054
|
+
goal: runtime.goal,
|
|
2055
|
+
taskId: stable?.taskId,
|
|
2056
|
+
title: stable?.title,
|
|
2057
|
+
attempt: stable?.attempt,
|
|
2058
|
+
totalTasks: stable?.totalTasks,
|
|
2059
|
+
currentTask: stable?.currentTask,
|
|
2060
|
+
subtasks: stable?.subtasks,
|
|
2061
|
+
taskProgress: stable?.taskProgress,
|
|
2062
|
+
activeStatus: message,
|
|
2063
|
+
networkRecoveryEvent: event.type,
|
|
2064
|
+
networkRetryCount: event.state.retryCount,
|
|
2065
|
+
networkOutageElapsedMs: event.state.elapsedMs,
|
|
2066
|
+
networkNextRetryAtMs: event.state.nextRetryAtMs,
|
|
2067
|
+
networkNextRetryInMs:
|
|
2068
|
+
event.state.nextRetryAtMs === undefined ? undefined : Math.max(0, event.state.nextRetryAtMs - nowMs),
|
|
2069
|
+
networkFailureReason: event.state.lastFailure.reason,
|
|
1131
2070
|
});
|
|
1132
2071
|
}
|
|
1133
2072
|
|
|
2073
|
+
function latestNetworkRecovery(
|
|
2074
|
+
recoveries: ReadonlyMap<number, NetworkRecoveryEvent>,
|
|
2075
|
+
): NetworkRecoveryEvent | undefined {
|
|
2076
|
+
const id = latestNetworkRecoveryId(recoveries);
|
|
2077
|
+
return id === undefined ? undefined : recoveries.get(id);
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
function latestNetworkRecoveryId(recoveries: ReadonlyMap<number, NetworkRecoveryEvent>): number | undefined {
|
|
2081
|
+
let latest: number | undefined;
|
|
2082
|
+
for (const id of recoveries.keys()) {
|
|
2083
|
+
if (latest === undefined || id > latest) latest = id;
|
|
2084
|
+
}
|
|
2085
|
+
return latest;
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
function isTerminalNetworkRecoveryEvent(type: NetworkRecoveryEventType): boolean {
|
|
2089
|
+
return type === "recovered" || type === "failed" || type === "cancelled" || type === "outage_expired";
|
|
2090
|
+
}
|
|
2091
|
+
|
|
1134
2092
|
function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDiagnostic): void {
|
|
1135
2093
|
const normalized: PlannerDiagnostic = {
|
|
1136
2094
|
kind: diagnostic.kind,
|
|
@@ -1156,6 +2114,41 @@ function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDia
|
|
|
1156
2114
|
});
|
|
1157
2115
|
}
|
|
1158
2116
|
|
|
2117
|
+
function aggregateWorkerUsage(outcomes: readonly SessionOutcome[]): WorkerUsageTotals | undefined {
|
|
2118
|
+
const usage = outcomes.flatMap((outcome) => (outcome.workerUsage ? [outcome.workerUsage] : []));
|
|
2119
|
+
if (usage.length === 0) {
|
|
2120
|
+
return undefined;
|
|
2121
|
+
}
|
|
2122
|
+
return usage.reduce<WorkerUsageTotals>(
|
|
2123
|
+
(total, item) => ({
|
|
2124
|
+
input: total.input + item.input,
|
|
2125
|
+
output: total.output + item.output,
|
|
2126
|
+
cacheRead: total.cacheRead + item.cacheRead,
|
|
2127
|
+
cacheWrite: total.cacheWrite + item.cacheWrite,
|
|
2128
|
+
total: total.total + item.total,
|
|
2129
|
+
}),
|
|
2130
|
+
{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
2131
|
+
);
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2134
|
+
function createWorkerSessionMetrics(): WorkerSessionMetrics {
|
|
2135
|
+
return { starts: 0, reuses: 0, rotations: 0, retained: 0, rotationReasons: {} };
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
function snapshotWorkerSessionMetrics(metrics: WorkerSessionMetrics): WorkerSessionMetrics {
|
|
2139
|
+
return { ...metrics, rotationReasons: { ...metrics.rotationReasons } };
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
function recordWorkerSessionMetric(metrics: WorkerSessionMetrics, diagnostic: WorkerSessionDiagnostic): void {
|
|
2143
|
+
if (diagnostic.event === "session_started") metrics.starts += 1;
|
|
2144
|
+
if (diagnostic.event === "session_reused") metrics.reuses += 1;
|
|
2145
|
+
if (diagnostic.event === "session_retained") metrics.retained += 1;
|
|
2146
|
+
if (diagnostic.event === "session_rotated") {
|
|
2147
|
+
metrics.rotations += 1;
|
|
2148
|
+
metrics.rotationReasons[diagnostic.reasonCode] = (metrics.rotationReasons[diagnostic.reasonCode] ?? 0) + 1;
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
|
|
1159
2152
|
function createWorkerCostState(): WorkerCostState {
|
|
1160
2153
|
return {
|
|
1161
2154
|
total: 0,
|
|
@@ -1192,9 +2185,9 @@ function recordLiveWorkerCost(
|
|
|
1192
2185
|
|
|
1193
2186
|
function finalizeWorkerCost(
|
|
1194
2187
|
state: WorkerCostState,
|
|
1195
|
-
|
|
2188
|
+
worker: string,
|
|
2189
|
+
outcome: Pick<SessionOutcome, "workerCostTotal">,
|
|
1196
2190
|
): void {
|
|
1197
|
-
const worker = workerKey(outcome.task.taskId, outcome.attempt);
|
|
1198
2191
|
state.finalizedByWorker.set(worker, finiteNonNegativeNumber(outcome.workerCostTotal) ?? 0);
|
|
1199
2192
|
state.liveByWorker.delete(worker);
|
|
1200
2193
|
for (const messageKey of state.liveByMessage.keys()) {
|
|
@@ -1263,6 +2256,43 @@ function subtaskProgress(
|
|
|
1263
2256
|
});
|
|
1264
2257
|
}
|
|
1265
2258
|
|
|
2259
|
+
function emitWorkerSessionProgress(
|
|
2260
|
+
runtime: RuntimeOptions,
|
|
2261
|
+
tasks: readonly Task[],
|
|
2262
|
+
task: Pick<Task, "taskId" | "title" | "statusItems">,
|
|
2263
|
+
attempts: readonly TaskAttemptSummary[],
|
|
2264
|
+
attempt: number,
|
|
2265
|
+
diagnostic: WorkerSessionDiagnostic,
|
|
2266
|
+
): void {
|
|
2267
|
+
recordWorkerSessionMetric(runtime.workerSessionMetrics, diagnostic);
|
|
2268
|
+
const contextText =
|
|
2269
|
+
diagnostic.contextUsagePercent === undefined
|
|
2270
|
+
? ""
|
|
2271
|
+
: ` at ${diagnostic.contextUsagePercent.toFixed(1)}% context usage`;
|
|
2272
|
+
const action =
|
|
2273
|
+
diagnostic.event === "session_started"
|
|
2274
|
+
? "started"
|
|
2275
|
+
: diagnostic.event === "session_reused"
|
|
2276
|
+
? "reused"
|
|
2277
|
+
: diagnostic.event === "session_rotated"
|
|
2278
|
+
? "rotated"
|
|
2279
|
+
: "retained";
|
|
2280
|
+
emitProgress(runtime, `Worker session ${action}${contextText} (${diagnostic.reasonCode}).`, {
|
|
2281
|
+
phase: "worker_session",
|
|
2282
|
+
taskId: task.taskId,
|
|
2283
|
+
title: task.title,
|
|
2284
|
+
attempt,
|
|
2285
|
+
status: "in_progress",
|
|
2286
|
+
activeStatus: `Worker session ${action}`,
|
|
2287
|
+
workerSessionEvent: diagnostic.event,
|
|
2288
|
+
workerSessionReason: diagnostic.reasonCode,
|
|
2289
|
+
workerSessionContextUsagePercent: diagnostic.contextUsagePercent,
|
|
2290
|
+
workerSessionContextThresholdPercent: diagnostic.contextThresholdPercent,
|
|
2291
|
+
...currentTaskProgress(task, "in_progress"),
|
|
2292
|
+
taskProgress: buildTaskProgressModel({ tasks, attempts, currentTaskId: task.taskId }),
|
|
2293
|
+
});
|
|
2294
|
+
}
|
|
2295
|
+
|
|
1266
2296
|
function emitWorkerEventProgress(
|
|
1267
2297
|
runtime: RuntimeOptions,
|
|
1268
2298
|
tasks: readonly Task[],
|
|
@@ -1278,6 +2308,7 @@ function emitWorkerEventProgress(
|
|
|
1278
2308
|
usageCostTotal?: number;
|
|
1279
2309
|
usageCostKey?: string;
|
|
1280
2310
|
},
|
|
2311
|
+
accountingWorker = workerKey(task.taskId, attempt),
|
|
1281
2312
|
): void {
|
|
1282
2313
|
const worker = workerKey(task.taskId, attempt);
|
|
1283
2314
|
let activeStatus = runtime.workerActivityByWorker.get(worker);
|
|
@@ -1318,7 +2349,7 @@ function emitWorkerEventProgress(
|
|
|
1318
2349
|
}
|
|
1319
2350
|
|
|
1320
2351
|
const costChanged =
|
|
1321
|
-
event.usageCostTotal !== undefined && recordLiveWorkerCost(runtime.workerCostState,
|
|
2352
|
+
event.usageCostTotal !== undefined && recordLiveWorkerCost(runtime.workerCostState, accountingWorker, event);
|
|
1322
2353
|
|
|
1323
2354
|
if (event.type === "message_end" && event.activity) {
|
|
1324
2355
|
emitProgress(runtime, event.activity, {
|
|
@@ -1390,7 +2421,7 @@ function emitWorkerEventProgress(
|
|
|
1390
2421
|
}
|
|
1391
2422
|
|
|
1392
2423
|
function stripToolOutcomePrefix(activity: string): string {
|
|
1393
|
-
return activity.replace(/^(?:(?:Finished|Failed):\s*)+/i, "");
|
|
2424
|
+
return activity.trim().replace(/^(?:(?:Finished|Failed):\s*)+/i, "");
|
|
1394
2425
|
}
|
|
1395
2426
|
|
|
1396
2427
|
function activeStatusFromWorkerText(text: string): string {
|
|
@@ -1591,6 +2622,43 @@ async function appendCommitNote(pathname: string, result: CommitAfterSessionResu
|
|
|
1591
2622
|
await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
|
|
1592
2623
|
}
|
|
1593
2624
|
|
|
2625
|
+
async function appendNetworkInterruptionEvidence(
|
|
2626
|
+
pathname: string,
|
|
2627
|
+
outcome: SessionOutcome,
|
|
2628
|
+
networkRetry: number,
|
|
2629
|
+
): Promise<void> {
|
|
2630
|
+
const summary = extractResultSummary(outcome.assistantText || "").trim() || "TASK_RESULT:\nstatus: unknown";
|
|
2631
|
+
const lines = [
|
|
2632
|
+
"",
|
|
2633
|
+
`## TODO ${outcome.task.taskId} — ${outcome.task.title} (ordinary attempt ${outcome.attempt}, network interruption ${networkRetry})`,
|
|
2634
|
+
"",
|
|
2635
|
+
"Disposition: transient provider/transport failure; this is durable evidence, not an ordinary task attempt.",
|
|
2636
|
+
`Started: ${outcome.startedAt}`,
|
|
2637
|
+
`Ended: ${outcome.endedAt}`,
|
|
2638
|
+
`Worker error: ${outcome.error ?? "transient provider or transport failure"}`,
|
|
2639
|
+
];
|
|
2640
|
+
if (outcome.sessionId) lines.push(`Session ID: ${outcome.sessionId}`);
|
|
2641
|
+
if (outcome.sessionFile) lines.push(`Session file: ${outcome.sessionFile}`);
|
|
2642
|
+
if (outcome.workerCostSource || outcome.workerCostTotal > 0) {
|
|
2643
|
+
lines.push(`Worker cost: ${outcome.workerCostTotal} (${outcome.workerCostSource ?? "unavailable"})`);
|
|
2644
|
+
}
|
|
2645
|
+
if (outcome.workerUsage) {
|
|
2646
|
+
lines.push(
|
|
2647
|
+
`Worker token usage: input=${outcome.workerUsage.input}, output=${outcome.workerUsage.output}, cacheRead=${outcome.workerUsage.cacheRead}, cacheWrite=${outcome.workerUsage.cacheWrite}, total=${outcome.workerUsage.total}`,
|
|
2648
|
+
);
|
|
2649
|
+
}
|
|
2650
|
+
lines.push(
|
|
2651
|
+
"",
|
|
2652
|
+
"Safety: the replacement session must inspect the working tree and this evidence before continuing; completed side effects must not be blindly replayed.",
|
|
2653
|
+
"",
|
|
2654
|
+
"```text",
|
|
2655
|
+
summary,
|
|
2656
|
+
"```",
|
|
2657
|
+
"",
|
|
2658
|
+
);
|
|
2659
|
+
await appendFile(pathname, `${lines.join("\n")}\n`, "utf8");
|
|
2660
|
+
}
|
|
2661
|
+
|
|
1594
2662
|
async function appendTaskResult(
|
|
1595
2663
|
pathname: string,
|
|
1596
2664
|
task: Task,
|
|
@@ -1629,6 +2697,25 @@ async function appendTaskResult(
|
|
|
1629
2697
|
if (outcome.contextObservations.length > 0) {
|
|
1630
2698
|
lines.push("", "Context observations:", ...outcome.contextObservations.map((item) => `- ${item}`));
|
|
1631
2699
|
}
|
|
2700
|
+
if (outcome.workerCostSource || outcome.workerCostTotal > 0) {
|
|
2701
|
+
lines.push(`Worker cost: ${outcome.workerCostTotal} (${outcome.workerCostSource ?? "unavailable"})`);
|
|
2702
|
+
}
|
|
2703
|
+
if (outcome.workerUsage) {
|
|
2704
|
+
lines.push(
|
|
2705
|
+
`Worker token usage: input=${outcome.workerUsage.input}, output=${outcome.workerUsage.output}, cacheRead=${outcome.workerUsage.cacheRead}, cacheWrite=${outcome.workerUsage.cacheWrite}, total=${outcome.workerUsage.total}`,
|
|
2706
|
+
);
|
|
2707
|
+
}
|
|
2708
|
+
if (outcome.sessionDiagnostics?.length) {
|
|
2709
|
+
lines.push(
|
|
2710
|
+
"",
|
|
2711
|
+
"Worker session diagnostics:",
|
|
2712
|
+
...outcome.sessionDiagnostics.map((item) => {
|
|
2713
|
+
const context =
|
|
2714
|
+
item.contextUsagePercent === undefined ? "" : ` context=${item.contextUsagePercent.toFixed(1)}%`;
|
|
2715
|
+
return `- event=${item.event} reason=${item.reasonCode}${context}`;
|
|
2716
|
+
}),
|
|
2717
|
+
);
|
|
2718
|
+
}
|
|
1632
2719
|
if (outcome.compactionEvents.length > 0) {
|
|
1633
2720
|
lines.push("", "Compaction events:", ...outcome.compactionEvents.map((item) => `- ${item}`));
|
|
1634
2721
|
}
|