pi-long-task 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +54 -8
- package/package.json +2 -2
- package/src/coordinator.ts +686 -15
- package/src/index.ts +18 -5
- package/src/worker_config.ts +63 -14
- package/src/worker_reuse_policy.ts +389 -0
- package/src/worker_session.ts +261 -33
package/src/coordinator.ts
CHANGED
|
@@ -10,7 +10,7 @@ import type {
|
|
|
10
10
|
} from "./types.ts";
|
|
11
11
|
import { commitAfterSession, gitDirtyPaths, shouldCommitOutcome, type CommitAfterSessionResult } from "./git.ts";
|
|
12
12
|
import { formatCoordinatorResultMessage } from "./render.ts";
|
|
13
|
-
import { extractResultSummary } from "./result_writer.ts";
|
|
13
|
+
import { extractResultSummary, hasCompleteTaskResult } from "./result_writer.ts";
|
|
14
14
|
import { runGuardedSessionPrompt } from "./session_guard.ts";
|
|
15
15
|
import {
|
|
16
16
|
generatePlanRevision,
|
|
@@ -28,6 +28,16 @@ import {
|
|
|
28
28
|
} from "./plan_store.ts";
|
|
29
29
|
import type { SerializedSteeringQueue, SteeringMessage } from "./steering.ts";
|
|
30
30
|
import { parseWorkerRuntimeConfig } from "./worker_config.ts";
|
|
31
|
+
import {
|
|
32
|
+
classifyWorkerSessionRetry,
|
|
33
|
+
createWorkerSessionCompatibilityFingerprint,
|
|
34
|
+
decideWorkerSessionReuse,
|
|
35
|
+
DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
|
|
36
|
+
DEFAULT_WORKER_SESSION_REUSE_ENABLED,
|
|
37
|
+
resolveWorkerSessionReuseConfig,
|
|
38
|
+
type WorkerSessionCompatibilityFingerprint,
|
|
39
|
+
type WorkerSessionHealth,
|
|
40
|
+
} from "./worker_reuse_policy.ts";
|
|
31
41
|
import { buildTaskProgressModel, type TaskProgressModel, type TaskProgressStatus } from "./task_progress.ts";
|
|
32
42
|
import {
|
|
33
43
|
applyGoalInstructionsToTodoMarkdown,
|
|
@@ -40,12 +50,21 @@ import {
|
|
|
40
50
|
} from "./todo_generator.ts";
|
|
41
51
|
import { parseTasks, todoGlobalInstructions, type Task } from "./todo_parser.ts";
|
|
42
52
|
import {
|
|
53
|
+
buildWorkerSessionCreationFailureOutcome,
|
|
43
54
|
createIsolatedWorkerSession,
|
|
55
|
+
createWorkerSessionResource,
|
|
56
|
+
DEFAULT_WORKER_TOOLS,
|
|
57
|
+
disposeWorkerSessionResource,
|
|
44
58
|
runWorkerTask,
|
|
59
|
+
runWorkerTaskAssignment,
|
|
60
|
+
workerSessionContextUsagePercent,
|
|
45
61
|
type RunWorkerTaskOptions,
|
|
46
62
|
type SessionOutcome,
|
|
63
|
+
type WorkerSessionDiagnostic,
|
|
47
64
|
type WorkerSessionFactory,
|
|
48
65
|
type WorkerSessionLike,
|
|
66
|
+
type WorkerSessionResource,
|
|
67
|
+
type WorkerUsageTotals,
|
|
49
68
|
} from "./worker_session.ts";
|
|
50
69
|
|
|
51
70
|
export type { CoordinatorStatus } from "./types.ts";
|
|
@@ -58,6 +77,8 @@ export const DEFAULT_COORDINATOR_OPTIONS = {
|
|
|
58
77
|
maxBashTimeoutMs: 300_000,
|
|
59
78
|
taskThinking: "high",
|
|
60
79
|
todoThinking: "xhigh",
|
|
80
|
+
workerSessionReuse: DEFAULT_WORKER_SESSION_REUSE_ENABLED,
|
|
81
|
+
workerSessionReuseContextThresholdPercent: DEFAULT_WORKER_SESSION_REUSE_CONTEXT_THRESHOLD_PERCENT,
|
|
61
82
|
} as const;
|
|
62
83
|
|
|
63
84
|
export type WorkerRunner = (options: RunWorkerTaskOptions) => Promise<SessionOutcome>;
|
|
@@ -65,6 +86,7 @@ export type CoordinatorProgressPhase =
|
|
|
65
86
|
| "planning"
|
|
66
87
|
| "planned"
|
|
67
88
|
| "task_start"
|
|
89
|
+
| "worker_session"
|
|
68
90
|
| "worker_tool"
|
|
69
91
|
| "task_done"
|
|
70
92
|
| "task_blocked"
|
|
@@ -124,6 +146,10 @@ export interface CoordinatorProgressUpdate {
|
|
|
124
146
|
plannerDiagnostics?: string[];
|
|
125
147
|
plannerSessionFile?: string;
|
|
126
148
|
plannerSessionId?: string;
|
|
149
|
+
workerSessionEvent?: WorkerSessionDiagnostic["event"];
|
|
150
|
+
workerSessionReason?: string;
|
|
151
|
+
workerSessionContextUsagePercent?: number;
|
|
152
|
+
workerSessionContextThresholdPercent?: number;
|
|
127
153
|
}
|
|
128
154
|
|
|
129
155
|
export type CoordinatorProgressHandler = (update: CoordinatorProgressUpdate) => void;
|
|
@@ -146,6 +172,10 @@ export interface RunCoordinatorOptions extends PiLongTaskInput {
|
|
|
146
172
|
maxBashTimeoutMs?: number;
|
|
147
173
|
taskThinking?: string;
|
|
148
174
|
todoThinking?: string;
|
|
175
|
+
/** Set false to retain the legacy one-session-per-task lifecycle. */
|
|
176
|
+
workerSessionReuse?: boolean;
|
|
177
|
+
/** Rotate before another assignment when context usage reaches this percentage. */
|
|
178
|
+
workerSessionReuseContextThresholdPercent?: number;
|
|
149
179
|
now?: () => Date;
|
|
150
180
|
onProgress?: CoordinatorProgressHandler;
|
|
151
181
|
/** Run-scoped FIFO populated by the extension input handler during active execution. */
|
|
@@ -190,6 +220,14 @@ export interface TaskAttemptSummary {
|
|
|
190
220
|
resultText?: string;
|
|
191
221
|
}
|
|
192
222
|
|
|
223
|
+
export interface WorkerSessionMetrics {
|
|
224
|
+
starts: number;
|
|
225
|
+
reuses: number;
|
|
226
|
+
rotations: number;
|
|
227
|
+
retained: number;
|
|
228
|
+
rotationReasons: Record<string, number>;
|
|
229
|
+
}
|
|
230
|
+
|
|
193
231
|
export interface CoordinatorResult {
|
|
194
232
|
status: CoordinatorStatus;
|
|
195
233
|
summary: string;
|
|
@@ -210,6 +248,10 @@ export interface CoordinatorResult {
|
|
|
210
248
|
attempts: TaskAttemptSummary[];
|
|
211
249
|
taskProgress: TaskProgressModel;
|
|
212
250
|
workerCostTotal: number;
|
|
251
|
+
/** Sum of task/attempt token deltas; omitted when statistics are unavailable. */
|
|
252
|
+
workerUsageTotal?: WorkerUsageTotals;
|
|
253
|
+
/** Additive lifecycle counters for adaptive worker-session reuse. */
|
|
254
|
+
workerSessionMetrics?: WorkerSessionMetrics;
|
|
213
255
|
commit: boolean;
|
|
214
256
|
goal?: string;
|
|
215
257
|
error?: string;
|
|
@@ -236,9 +278,12 @@ interface RuntimeOptions {
|
|
|
236
278
|
goal?: string;
|
|
237
279
|
taskThinking: string;
|
|
238
280
|
todoThinking: string;
|
|
281
|
+
workerSessionReuse: boolean;
|
|
282
|
+
workerSessionReuseContextThresholdPercent: number;
|
|
239
283
|
todoTimeoutMs: number;
|
|
240
284
|
todoGracefulShutdownMs: number;
|
|
241
285
|
workerRunner: WorkerRunner;
|
|
286
|
+
useRetainedWorkerLifecycle: boolean;
|
|
242
287
|
todoPlanner: TodoPlanner;
|
|
243
288
|
abortSignal?: AbortSignal;
|
|
244
289
|
workerSessionFactory?: WorkerSessionFactory;
|
|
@@ -250,12 +295,461 @@ interface RuntimeOptions {
|
|
|
250
295
|
workerTextByWorker: Map<string, string>;
|
|
251
296
|
workerTextPublishedLengthByWorker: Map<string, number>;
|
|
252
297
|
plannerDiagnostics: PlannerDiagnostic[];
|
|
298
|
+
workerSessionMetrics: WorkerSessionMetrics;
|
|
253
299
|
steeringQueue?: SerializedSteeringQueue;
|
|
254
300
|
onPlanRevisionAccepted?: (revision: GeneratedPlanRevision) => void | Promise<void>;
|
|
255
301
|
}
|
|
256
302
|
|
|
303
|
+
type RetainedWorkerReuseScope = "sequential_task" | "partial_continuation";
|
|
304
|
+
|
|
305
|
+
export interface WorkerAssignmentIdentity {
|
|
306
|
+
/** Unique invocation identity, even when a replacement reuses a task ID and attempt number. */
|
|
307
|
+
assignmentId: string;
|
|
308
|
+
/** Stable task identity or semantic fingerprint from the plan that launched this assignment. */
|
|
309
|
+
taskIdentity: string;
|
|
310
|
+
/** Accepted-steering generation at the assignment boundary. */
|
|
311
|
+
steeringGeneration: number;
|
|
312
|
+
/** Structural plan generation from which the assignment was selected. */
|
|
313
|
+
planAuthorityToken: string;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
interface RetainedWorkerState {
|
|
317
|
+
resource: WorkerSessionResource;
|
|
318
|
+
compatibility: WorkerSessionCompatibilityFingerprint;
|
|
319
|
+
health: WorkerSessionHealth;
|
|
320
|
+
contextUsagePercent?: number;
|
|
321
|
+
previousTask: Pick<Task, "taskId" | "title">;
|
|
322
|
+
previousAttempt: number;
|
|
323
|
+
previousAssignmentIdentity: WorkerAssignmentIdentity;
|
|
324
|
+
reportDiagnostic: (diagnostic: WorkerSessionDiagnostic) => void;
|
|
325
|
+
reuseScope: RetainedWorkerReuseScope;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
interface ActiveWorkerSessionAssignment {
|
|
329
|
+
identity: WorkerAssignmentIdentity;
|
|
330
|
+
controller: AbortController;
|
|
331
|
+
tainted: boolean;
|
|
332
|
+
rotationReported: boolean;
|
|
333
|
+
resource?: WorkerSessionResource;
|
|
334
|
+
reportDiagnostic?: (diagnostic: WorkerSessionDiagnostic) => void;
|
|
335
|
+
resolveCompletion: () => void;
|
|
336
|
+
completion: Promise<void>;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
export interface CoordinatorWorkerSessionOwnerOptions {
|
|
340
|
+
runId: string;
|
|
341
|
+
cwd: string;
|
|
342
|
+
workerSessionReuse: boolean;
|
|
343
|
+
workerSessionReuseContextThresholdPercent: number;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/** Run-scoped owner for the single retained worker session and its assignment lock. */
|
|
347
|
+
export class CoordinatorWorkerSessionOwner {
|
|
348
|
+
private retained: RetainedWorkerState | undefined;
|
|
349
|
+
private active: ActiveWorkerSessionAssignment | undefined;
|
|
350
|
+
private closed = false;
|
|
351
|
+
private disposePromise: Promise<void> | undefined;
|
|
352
|
+
private assignmentSequence = 0;
|
|
353
|
+
private readonly runtime: CoordinatorWorkerSessionOwnerOptions;
|
|
354
|
+
|
|
355
|
+
constructor(runtime: CoordinatorWorkerSessionOwnerOptions) {
|
|
356
|
+
this.runtime = runtime;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
async run(options: RunWorkerTaskOptions, identity?: WorkerAssignmentIdentity): Promise<SessionOutcome> {
|
|
360
|
+
if (this.closed) {
|
|
361
|
+
throw new Error("retained worker session owner is closed");
|
|
362
|
+
}
|
|
363
|
+
const assignmentIdentity = identity ?? this.defaultIdentity(options);
|
|
364
|
+
if (this.active) {
|
|
365
|
+
throw new Error("retained worker session already has an active assignment");
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
let resolveCompletion: (() => void) | undefined;
|
|
369
|
+
const completion = new Promise<void>((resolve) => {
|
|
370
|
+
resolveCompletion = resolve;
|
|
371
|
+
});
|
|
372
|
+
const active: ActiveWorkerSessionAssignment = {
|
|
373
|
+
identity: assignmentIdentity,
|
|
374
|
+
controller: new AbortController(),
|
|
375
|
+
tainted: false,
|
|
376
|
+
rotationReported: false,
|
|
377
|
+
resolveCompletion: resolveCompletion as () => void,
|
|
378
|
+
completion,
|
|
379
|
+
};
|
|
380
|
+
this.active = active;
|
|
381
|
+
const assignmentOptions = {
|
|
382
|
+
...options,
|
|
383
|
+
abortSignal: combineAbortSignals(options.abortSignal, active.controller.signal),
|
|
384
|
+
};
|
|
385
|
+
try {
|
|
386
|
+
return await this.runExclusive(assignmentOptions, assignmentIdentity, active);
|
|
387
|
+
} finally {
|
|
388
|
+
const activeResource = active.resource;
|
|
389
|
+
const retained = this.retained;
|
|
390
|
+
if (active.tainted && activeResource && retained?.resource === activeResource) {
|
|
391
|
+
retained.health = "cancelled";
|
|
392
|
+
await this.disposeRetainedResource(activeResource);
|
|
393
|
+
}
|
|
394
|
+
if (this.active === active) {
|
|
395
|
+
this.active = undefined;
|
|
396
|
+
}
|
|
397
|
+
active.resolveCompletion();
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
private async runExclusive(
|
|
402
|
+
options: RunWorkerTaskOptions,
|
|
403
|
+
identity: WorkerAssignmentIdentity,
|
|
404
|
+
active: ActiveWorkerSessionAssignment,
|
|
405
|
+
): Promise<SessionOutcome> {
|
|
406
|
+
const compatibility = this.compatibilityFor(options);
|
|
407
|
+
const diagnostics: WorkerSessionDiagnostic[] = [];
|
|
408
|
+
const report = (diagnostic: WorkerSessionDiagnostic) => {
|
|
409
|
+
diagnostics.push(diagnostic);
|
|
410
|
+
options.onSessionDiagnostic?.(diagnostic);
|
|
411
|
+
};
|
|
412
|
+
active.reportDiagnostic = report;
|
|
413
|
+
const diagnosticContext = (retained: RetainedWorkerState) => ({
|
|
414
|
+
...(retained.contextUsagePercent !== undefined ? { contextUsagePercent: retained.contextUsagePercent } : {}),
|
|
415
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
416
|
+
previousTaskId: retained.previousTask.taskId,
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
let reusedFrom: Pick<Task, "taskId" | "title"> | undefined;
|
|
420
|
+
if (this.retained && !this.assignmentMatchesRetainedScope(options, this.retained)) {
|
|
421
|
+
// A partial-work session is continuity for exactly the next attempt of
|
|
422
|
+
// that task. It must never spill into unrelated work or a later retry.
|
|
423
|
+
report({
|
|
424
|
+
event: "session_rotated",
|
|
425
|
+
reasonCode: "partial_continuation_scope_mismatch",
|
|
426
|
+
...diagnosticContext(this.retained),
|
|
427
|
+
});
|
|
428
|
+
await this.disposeRetained();
|
|
429
|
+
}
|
|
430
|
+
if (this.retained) {
|
|
431
|
+
const decision = decideWorkerSessionReuse({
|
|
432
|
+
config: {
|
|
433
|
+
enabled: this.runtime.workerSessionReuse,
|
|
434
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
435
|
+
},
|
|
436
|
+
candidate: {
|
|
437
|
+
health: this.retained.health,
|
|
438
|
+
compatibility: this.retained.compatibility,
|
|
439
|
+
contextUsagePercent: this.retained.contextUsagePercent,
|
|
440
|
+
assignmentState: "idle",
|
|
441
|
+
disposed: this.retained.resource.disposed,
|
|
442
|
+
},
|
|
443
|
+
requestedCompatibility: compatibility,
|
|
444
|
+
});
|
|
445
|
+
if (decision.reusable) {
|
|
446
|
+
reusedFrom = this.retained.previousTask;
|
|
447
|
+
report({
|
|
448
|
+
event: "session_reused",
|
|
449
|
+
reasonCode: decision.reasonCode,
|
|
450
|
+
...diagnosticContext(this.retained),
|
|
451
|
+
});
|
|
452
|
+
} else {
|
|
453
|
+
report({
|
|
454
|
+
event: "session_rotated",
|
|
455
|
+
reasonCode: decision.reasonCode,
|
|
456
|
+
...diagnosticContext(this.retained),
|
|
457
|
+
});
|
|
458
|
+
await this.disposeRetained();
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
if (!this.retained) {
|
|
463
|
+
try {
|
|
464
|
+
this.retained = {
|
|
465
|
+
resource: await createWorkerSessionResource(options, options.sessionFactory ?? createIsolatedWorkerSession),
|
|
466
|
+
compatibility,
|
|
467
|
+
health: "healthy",
|
|
468
|
+
previousTask: options.task,
|
|
469
|
+
previousAttempt: options.attempt,
|
|
470
|
+
previousAssignmentIdentity: identity,
|
|
471
|
+
reportDiagnostic: report,
|
|
472
|
+
reuseScope: "sequential_task",
|
|
473
|
+
};
|
|
474
|
+
report({
|
|
475
|
+
event: "session_started",
|
|
476
|
+
reasonCode: diagnostics.some((item) => item.event === "session_rotated")
|
|
477
|
+
? "rotation_completed"
|
|
478
|
+
: "fresh_session",
|
|
479
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
480
|
+
});
|
|
481
|
+
} catch (error) {
|
|
482
|
+
const failed = buildWorkerSessionCreationFailureOutcome(options, error);
|
|
483
|
+
failed.sessionDiagnostics = diagnostics;
|
|
484
|
+
return failed;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
active.resource = this.retained.resource;
|
|
489
|
+
let outcome: SessionOutcome;
|
|
490
|
+
try {
|
|
491
|
+
outcome = await runWorkerTaskAssignment(
|
|
492
|
+
options,
|
|
493
|
+
this.retained.resource,
|
|
494
|
+
reusedFrom ? { previousTask: reusedFrom } : undefined,
|
|
495
|
+
);
|
|
496
|
+
} catch (error) {
|
|
497
|
+
this.retained.health = active.tainted ? "cancelled" : "unrecoverable_error";
|
|
498
|
+
if (!active.rotationReported) {
|
|
499
|
+
active.rotationReported = true;
|
|
500
|
+
report({
|
|
501
|
+
event: "session_rotated",
|
|
502
|
+
reasonCode: "health_unrecoverable_error",
|
|
503
|
+
...diagnosticContext(this.retained),
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
await this.disposeRetained();
|
|
507
|
+
throw error;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
if (this.retained) {
|
|
511
|
+
const cancelled = Boolean(options.abortSignal?.aborted);
|
|
512
|
+
this.retained.health = workerSessionHealthForOutcome(outcome, cancelled);
|
|
513
|
+
this.retained.contextUsagePercent = await workerSessionContextUsagePercent(this.retained.resource.session);
|
|
514
|
+
this.retained.previousTask = options.task;
|
|
515
|
+
this.retained.previousAttempt = options.attempt;
|
|
516
|
+
this.retained.previousAssignmentIdentity = identity;
|
|
517
|
+
this.retained.reportDiagnostic = report;
|
|
518
|
+
|
|
519
|
+
if (active.tainted) {
|
|
520
|
+
if (!active.rotationReported) {
|
|
521
|
+
this.reportTaintedRotation(active, "assignment_cancelled");
|
|
522
|
+
}
|
|
523
|
+
await this.disposeRetainedResource(this.retained.resource);
|
|
524
|
+
outcome.sessionDiagnostics = [...(outcome.sessionDiagnostics ?? []), ...diagnostics];
|
|
525
|
+
return outcome;
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
const retry = classifyWorkerSessionRetry({
|
|
529
|
+
done: outcome.done,
|
|
530
|
+
reportedStatus: outcome.reportedStatus,
|
|
531
|
+
completeTaskResult: hasCompleteTaskResult(outcome.assistantText),
|
|
532
|
+
timedOut: outcome.timedOut,
|
|
533
|
+
aborted: outcome.aborted,
|
|
534
|
+
cancelled,
|
|
535
|
+
error: outcome.error,
|
|
536
|
+
});
|
|
537
|
+
this.retained.reuseScope = retry.mayContinueInSession ? "partial_continuation" : "sequential_task";
|
|
538
|
+
|
|
539
|
+
const postAssignmentDecision = decideWorkerSessionReuse({
|
|
540
|
+
config: {
|
|
541
|
+
enabled: this.runtime.workerSessionReuse,
|
|
542
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
543
|
+
},
|
|
544
|
+
candidate: {
|
|
545
|
+
health: this.retained.health,
|
|
546
|
+
compatibility: this.retained.compatibility,
|
|
547
|
+
contextUsagePercent: this.retained.contextUsagePercent,
|
|
548
|
+
assignmentState: "idle",
|
|
549
|
+
disposed: this.retained.resource.disposed,
|
|
550
|
+
},
|
|
551
|
+
requestedCompatibility: compatibility,
|
|
552
|
+
});
|
|
553
|
+
// Completed tasks may flow into the next sequential TODO. A retry may
|
|
554
|
+
// remain only when it is an explicitly safe partial continuation and all
|
|
555
|
+
// normal health/compatibility/context checks still pass.
|
|
556
|
+
const rotateForRetry = !outcome.done && !retry.mayContinueInSession;
|
|
557
|
+
if (!postAssignmentDecision.reusable || rotateForRetry) {
|
|
558
|
+
if (!active.rotationReported) {
|
|
559
|
+
active.rotationReported = true;
|
|
560
|
+
report({
|
|
561
|
+
event: "session_rotated",
|
|
562
|
+
reasonCode: postAssignmentDecision.reusable ? retry.reasonCode : postAssignmentDecision.reasonCode,
|
|
563
|
+
...diagnosticContext(this.retained),
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
await this.disposeRetained();
|
|
567
|
+
} else {
|
|
568
|
+
report({
|
|
569
|
+
event: "session_retained",
|
|
570
|
+
reasonCode: postAssignmentDecision.reasonCode,
|
|
571
|
+
...diagnosticContext(this.retained),
|
|
572
|
+
});
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
outcome.sessionDiagnostics = [...(outcome.sessionDiagnostics ?? []), ...diagnostics];
|
|
576
|
+
return outcome;
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
/**
|
|
580
|
+
* Taint and abort only the matching obsolete assignment. Late cancellation
|
|
581
|
+
* from an older steering generation cannot affect a replacement assignment.
|
|
582
|
+
*/
|
|
583
|
+
async invalidateAssignment(identity: WorkerAssignmentIdentity): Promise<boolean> {
|
|
584
|
+
const active = this.active;
|
|
585
|
+
if (active && sameWorkerAssignment(active.identity, identity)) {
|
|
586
|
+
this.taintActiveAssignment(active, "steering_revision_obsolete");
|
|
587
|
+
if (!active.controller.signal.aborted) {
|
|
588
|
+
active.controller.abort(new Error(`worker assignment ${identity.assignmentId} became obsolete`));
|
|
589
|
+
}
|
|
590
|
+
return true;
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
if (this.retained && sameWorkerAssignment(this.retained.previousAssignmentIdentity, identity)) {
|
|
594
|
+
const retained = this.retained;
|
|
595
|
+
retained.health = "cancelled";
|
|
596
|
+
retained.reportDiagnostic({
|
|
597
|
+
event: "session_rotated",
|
|
598
|
+
reasonCode: "steering_revision_obsolete",
|
|
599
|
+
...this.diagnosticContext(retained),
|
|
600
|
+
});
|
|
601
|
+
await this.disposeRetained();
|
|
602
|
+
return true;
|
|
603
|
+
}
|
|
604
|
+
return false;
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
/** Abort active work, wait for its ownership path, then dispose retained state once. */
|
|
608
|
+
dispose(): Promise<void> {
|
|
609
|
+
if (!this.disposePromise) {
|
|
610
|
+
this.closed = true;
|
|
611
|
+
this.disposePromise = this.disposeAfterActiveAssignment();
|
|
612
|
+
}
|
|
613
|
+
return this.disposePromise;
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
private async disposeAfterActiveAssignment(): Promise<void> {
|
|
617
|
+
const active = this.active;
|
|
618
|
+
if (active) {
|
|
619
|
+
this.taintActiveAssignment(active, "coordinator_shutdown");
|
|
620
|
+
if (!active.controller.signal.aborted) {
|
|
621
|
+
active.controller.abort(new Error("worker session owner disposed"));
|
|
622
|
+
}
|
|
623
|
+
await active.completion;
|
|
624
|
+
}
|
|
625
|
+
await this.disposeRetained();
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
private assignmentMatchesRetainedScope(options: RunWorkerTaskOptions, retained: RetainedWorkerState): boolean {
|
|
629
|
+
if (retained.reuseScope === "sequential_task") {
|
|
630
|
+
return true;
|
|
631
|
+
}
|
|
632
|
+
return (
|
|
633
|
+
options.task.taskId === retained.previousTask.taskId &&
|
|
634
|
+
options.task.title === retained.previousTask.title &&
|
|
635
|
+
options.attempt === retained.previousAttempt + 1
|
|
636
|
+
);
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
private taintActiveAssignment(active: ActiveWorkerSessionAssignment, reasonCode: string): void {
|
|
640
|
+
active.tainted = true;
|
|
641
|
+
const retained = this.retained;
|
|
642
|
+
if (retained && retained.resource === active.resource) {
|
|
643
|
+
retained.health = "cancelled";
|
|
644
|
+
}
|
|
645
|
+
this.reportTaintedRotation(active, reasonCode);
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
private reportTaintedRotation(active: ActiveWorkerSessionAssignment, reasonCode: string): void {
|
|
649
|
+
if (active.rotationReported) return;
|
|
650
|
+
active.rotationReported = true;
|
|
651
|
+
const retained = this.retained;
|
|
652
|
+
active.reportDiagnostic?.({
|
|
653
|
+
event: "session_rotated",
|
|
654
|
+
reasonCode,
|
|
655
|
+
...(retained ? this.diagnosticContext(retained) : {}),
|
|
656
|
+
});
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
private diagnosticContext(retained: RetainedWorkerState): {
|
|
660
|
+
contextUsagePercent?: number;
|
|
661
|
+
contextThresholdPercent: number;
|
|
662
|
+
previousTaskId: string;
|
|
663
|
+
} {
|
|
664
|
+
return {
|
|
665
|
+
...(retained.contextUsagePercent !== undefined ? { contextUsagePercent: retained.contextUsagePercent } : {}),
|
|
666
|
+
contextThresholdPercent: this.runtime.workerSessionReuseContextThresholdPercent,
|
|
667
|
+
previousTaskId: retained.previousTask.taskId,
|
|
668
|
+
};
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
private defaultIdentity(options: RunWorkerTaskOptions): WorkerAssignmentIdentity {
|
|
672
|
+
const sequence = ++this.assignmentSequence;
|
|
673
|
+
return {
|
|
674
|
+
assignmentId: `${options.task.taskId}:${options.attempt}:${sequence}`,
|
|
675
|
+
taskIdentity: `${options.task.taskId}:${options.task.title}`,
|
|
676
|
+
steeringGeneration: 0,
|
|
677
|
+
planAuthorityToken: "direct-owner",
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
private compatibilityFor(options: RunWorkerTaskOptions): WorkerSessionCompatibilityFingerprint {
|
|
682
|
+
return createWorkerSessionCompatibilityFingerprint({
|
|
683
|
+
coordinatorRunId: this.runtime.runId,
|
|
684
|
+
repositoryRoot: this.runtime.cwd,
|
|
685
|
+
worktreeRoot: options.cwd,
|
|
686
|
+
modelName: options.modelName,
|
|
687
|
+
model: options.model,
|
|
688
|
+
tools: options.tools ?? DEFAULT_WORKER_TOOLS,
|
|
689
|
+
thinkingLevel: options.thinkingLevel,
|
|
690
|
+
agentDir: options.agentDir,
|
|
691
|
+
modelRuntime: options.modelRuntime,
|
|
692
|
+
authStorage: options.authStorage,
|
|
693
|
+
modelRegistry: options.modelRegistry,
|
|
694
|
+
settingsManager: options.settingsManager,
|
|
695
|
+
resourceLoader: options.resourceLoader,
|
|
696
|
+
sessionFactory: options.sessionFactory ?? createIsolatedWorkerSession,
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
private async disposeRetained(): Promise<void> {
|
|
701
|
+
const retained = this.retained;
|
|
702
|
+
if (!retained) {
|
|
703
|
+
return;
|
|
704
|
+
}
|
|
705
|
+
await this.disposeRetainedResource(retained.resource);
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
private async disposeRetainedResource(resource: WorkerSessionResource): Promise<void> {
|
|
709
|
+
if (this.retained?.resource === resource) {
|
|
710
|
+
this.retained = undefined;
|
|
711
|
+
}
|
|
712
|
+
try {
|
|
713
|
+
await disposeWorkerSessionResource(resource);
|
|
714
|
+
} catch {
|
|
715
|
+
// Session disposal is best effort; resource ownership is still closed exactly once.
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
function sameWorkerAssignment(left: WorkerAssignmentIdentity, right: WorkerAssignmentIdentity): boolean {
|
|
721
|
+
return (
|
|
722
|
+
left.assignmentId === right.assignmentId &&
|
|
723
|
+
left.taskIdentity === right.taskIdentity &&
|
|
724
|
+
left.steeringGeneration === right.steeringGeneration &&
|
|
725
|
+
left.planAuthorityToken === right.planAuthorityToken
|
|
726
|
+
);
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
function combineAbortSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined {
|
|
730
|
+
const available = signals.filter((signal): signal is AbortSignal => Boolean(signal));
|
|
731
|
+
if (available.length === 0) return undefined;
|
|
732
|
+
if (available.length === 1) return available[0];
|
|
733
|
+
return AbortSignal.any(available);
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
export function workerSessionHealthForOutcome(
|
|
737
|
+
outcome: Pick<SessionOutcome, "timedOut" | "aborted" | "error" | "assistantText">,
|
|
738
|
+
cancelled = false,
|
|
739
|
+
): WorkerSessionHealth {
|
|
740
|
+
if (outcome.timedOut) return "timed_out";
|
|
741
|
+
if (cancelled) return "cancelled";
|
|
742
|
+
if (outcome.aborted) return "aborted";
|
|
743
|
+
if (outcome.error) return "unrecoverable_error";
|
|
744
|
+
if (!hasCompleteTaskResult(outcome.assistantText)) return "invalid_state";
|
|
745
|
+
return "healthy";
|
|
746
|
+
}
|
|
747
|
+
|
|
257
748
|
export async function runCoordinator(options: RunCoordinatorOptions): Promise<CoordinatorResult> {
|
|
258
749
|
const runtime = buildRuntimeOptions(options);
|
|
750
|
+
const workerSessionOwner = runtime.useRetainedWorkerLifecycle
|
|
751
|
+
? new CoordinatorWorkerSessionOwner(runtime)
|
|
752
|
+
: undefined;
|
|
259
753
|
const inputText = coordinatorInputText(options);
|
|
260
754
|
const attempts: TaskAttemptSummary[] = [];
|
|
261
755
|
const outcomes: SessionOutcome[] = [];
|
|
@@ -269,6 +763,12 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
269
763
|
let activeTask: Task | undefined;
|
|
270
764
|
let activeTaskReference: PlanTaskReference | undefined;
|
|
271
765
|
let activeAttempt: number | undefined;
|
|
766
|
+
let activeWorkerAssignment:
|
|
767
|
+
| { identity: WorkerAssignmentIdentity; controller: AbortController; obsolete: boolean }
|
|
768
|
+
| undefined;
|
|
769
|
+
let steeringGeneration = 0;
|
|
770
|
+
let workerExecutionSequence = 0;
|
|
771
|
+
let removeSteeringProcessor: (() => void) | undefined;
|
|
272
772
|
const protectedDirtyPathsByTask = new Map<string, Set<string>>();
|
|
273
773
|
|
|
274
774
|
try {
|
|
@@ -287,7 +787,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
287
787
|
});
|
|
288
788
|
|
|
289
789
|
let failure: string | undefined;
|
|
290
|
-
runtime.steeringQueue?.setProcessor(async (message) => {
|
|
790
|
+
removeSteeringProcessor = runtime.steeringQueue?.setProcessor(async (message) => {
|
|
291
791
|
const baseAtRequest = planStore.snapshot();
|
|
292
792
|
const activeTaskAtRequest = activeTaskReference
|
|
293
793
|
? resolvePlanTaskReference(
|
|
@@ -320,6 +820,25 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
320
820
|
// boundary, so this accepted revision continues the same run.
|
|
321
821
|
latestTodoMarkdown = appliedRevision.todoMarkdown;
|
|
322
822
|
latestTasks = appliedRevision.reconciliation.activeTasks.map((item) => item.task);
|
|
823
|
+
steeringGeneration += 1;
|
|
824
|
+
|
|
825
|
+
// Once replacement/removal is authoritative, make the exact old
|
|
826
|
+
// invocation obsolete before aborting it. Event callbacks consult this
|
|
827
|
+
// identity, so a late old result cannot repaint replacement progress.
|
|
828
|
+
const assignmentAtAcceptance = activeWorkerAssignment;
|
|
829
|
+
const activeStillValid = activeTaskReference
|
|
830
|
+
? Boolean(resolvePlanTaskReference(latestTasks, activeTaskReference, planStore.snapshot().authorityToken))
|
|
831
|
+
: true;
|
|
832
|
+
if (assignmentAtAcceptance && !activeStillValid) {
|
|
833
|
+
assignmentAtAcceptance.obsolete = true;
|
|
834
|
+
if (!assignmentAtAcceptance.controller.signal.aborted) {
|
|
835
|
+
assignmentAtAcceptance.controller.abort(
|
|
836
|
+
new Error(`steering revision ${message.sequence} replaced the active assignment`),
|
|
837
|
+
);
|
|
838
|
+
}
|
|
839
|
+
await workerSessionOwner?.invalidateAssignment(assignmentAtAcceptance.identity);
|
|
840
|
+
}
|
|
841
|
+
|
|
323
842
|
emitProgress(
|
|
324
843
|
runtime,
|
|
325
844
|
`Accepted steering revision ${message.sequence} with ${appliedRevision.reconciliation.activeTasks.length} task(s).`,
|
|
@@ -365,6 +884,23 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
365
884
|
const initialActivity =
|
|
366
885
|
nextTask.statusItems.find((item) => !item.done)?.text ?? `Starting TODO ${nextTask.taskId}`;
|
|
367
886
|
const worker = workerKey(nextTask.taskId, attempt);
|
|
887
|
+
// Task IDs and attempt numbers may be reused after an in-flight task is
|
|
888
|
+
// replaced by steering. Accounting needs an invocation identity so the
|
|
889
|
+
// obsolete attempt's finalized spend cannot be overwritten.
|
|
890
|
+
const accountingWorker = `${worker}#${++workerExecutionSequence}`;
|
|
891
|
+
const assignmentIdentity: WorkerAssignmentIdentity = {
|
|
892
|
+
assignmentId: accountingWorker,
|
|
893
|
+
taskIdentity: nextTask.stableId ?? taskSemanticFingerprint(nextTask),
|
|
894
|
+
steeringGeneration,
|
|
895
|
+
planAuthorityToken: schedulingSnapshot.authorityToken,
|
|
896
|
+
};
|
|
897
|
+
const assignmentController = new AbortController();
|
|
898
|
+
const assignmentState = { identity: assignmentIdentity, controller: assignmentController, obsolete: false };
|
|
899
|
+
const taskPlanReference = planTaskReference(nextTask, schedulingSnapshot.authorityToken);
|
|
900
|
+
activeWorkerAssignment = assignmentState;
|
|
901
|
+
activeTask = nextTask;
|
|
902
|
+
activeAttempt = attempt;
|
|
903
|
+
activeTaskReference = taskPlanReference;
|
|
368
904
|
runtime.workerActivityByWorker.set(worker, initialActivity);
|
|
369
905
|
runtime.workerTextByWorker.delete(worker);
|
|
370
906
|
runtime.workerTextPublishedLengthByWorker.delete(worker);
|
|
@@ -393,11 +929,7 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
393
929
|
: new Set<string>();
|
|
394
930
|
protectedDirtyPathsByTask.set(executionIdentity, preExistingDirtyPaths);
|
|
395
931
|
}
|
|
396
|
-
|
|
397
|
-
activeAttempt = attempt;
|
|
398
|
-
const taskPlanReference = planTaskReference(nextTask, schedulingSnapshot.authorityToken);
|
|
399
|
-
activeTaskReference = taskPlanReference;
|
|
400
|
-
const outcome = await runtime.workerRunner({
|
|
932
|
+
const workerOptions: RunWorkerTaskOptions = {
|
|
401
933
|
cwd: runtime.cwd,
|
|
402
934
|
todoPath: runtime.todoPath,
|
|
403
935
|
task: nextTask,
|
|
@@ -415,12 +947,40 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
415
947
|
model: runtime.workerModel,
|
|
416
948
|
modelName: runtime.workerModelName,
|
|
417
949
|
thinkingLevel: runtime.taskThinking,
|
|
418
|
-
abortSignal: runtime.abortSignal,
|
|
950
|
+
abortSignal: combineAbortSignals(runtime.abortSignal, assignmentController.signal),
|
|
419
951
|
sessionFactory: runtime.workerSessionFactory,
|
|
420
952
|
now: runtime.now,
|
|
421
|
-
onEvent: (event) =>
|
|
422
|
-
|
|
423
|
-
|
|
953
|
+
onEvent: (event) => {
|
|
954
|
+
if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
|
|
955
|
+
emitWorkerEventProgress(runtime, tasksBeforeAttempt, nextTask, attempts, attempt, event, accountingWorker);
|
|
956
|
+
}
|
|
957
|
+
},
|
|
958
|
+
onSessionDiagnostic: (diagnostic) => {
|
|
959
|
+
if (activeWorkerAssignment === assignmentState && !assignmentState.obsolete) {
|
|
960
|
+
emitWorkerSessionProgress(runtime, tasksBeforeAttempt, nextTask, attempts, attempt, diagnostic);
|
|
961
|
+
} else {
|
|
962
|
+
// Lifecycle accounting remains accurate, but obsolete diagnostics
|
|
963
|
+
// must not mutate the replacement task's visible progress.
|
|
964
|
+
recordWorkerSessionMetric(runtime.workerSessionMetrics, diagnostic);
|
|
965
|
+
}
|
|
966
|
+
},
|
|
967
|
+
};
|
|
968
|
+
let outcome: SessionOutcome;
|
|
969
|
+
try {
|
|
970
|
+
outcome = workerSessionOwner
|
|
971
|
+
? await workerSessionOwner.run(workerOptions, assignmentIdentity)
|
|
972
|
+
: await runtime.workerRunner(workerOptions);
|
|
973
|
+
} catch (error) {
|
|
974
|
+
if (!assignmentState.obsolete) {
|
|
975
|
+
throw error;
|
|
976
|
+
}
|
|
977
|
+
// A cancellation-aware custom runner may reject instead of returning
|
|
978
|
+
// an aborted outcome. Preserve historical evidence, but never let that
|
|
979
|
+
// obsolete rejection terminate or update the replacement assignment.
|
|
980
|
+
outcome = buildWorkerSessionCreationFailureOutcome(workerOptions, error);
|
|
981
|
+
outcome.aborted = true;
|
|
982
|
+
}
|
|
983
|
+
finalizeWorkerCost(runtime.workerCostState, accountingWorker, outcome);
|
|
424
984
|
|
|
425
985
|
// A revision may have been accepted while the worker was running. Let all
|
|
426
986
|
// already-received guidance settle, then resolve this exact task identity
|
|
@@ -465,6 +1025,9 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
465
1025
|
activeTask = undefined;
|
|
466
1026
|
activeTaskReference = undefined;
|
|
467
1027
|
activeAttempt = undefined;
|
|
1028
|
+
if (activeWorkerAssignment === assignmentState) {
|
|
1029
|
+
activeWorkerAssignment = undefined;
|
|
1030
|
+
}
|
|
468
1031
|
|
|
469
1032
|
let taskCommitHash: string | undefined;
|
|
470
1033
|
let taskCommitError: string | undefined;
|
|
@@ -587,6 +1150,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
587
1150
|
attempts,
|
|
588
1151
|
taskProgress,
|
|
589
1152
|
workerCostTotal: runtime.workerCostState.total,
|
|
1153
|
+
workerUsageTotal: aggregateWorkerUsage(outcomes),
|
|
1154
|
+
workerSessionMetrics: snapshotWorkerSessionMetrics(runtime.workerSessionMetrics),
|
|
590
1155
|
commit: options.commit,
|
|
591
1156
|
goal: runtime.goal,
|
|
592
1157
|
error: failure,
|
|
@@ -668,6 +1233,8 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
668
1233
|
attempts,
|
|
669
1234
|
taskProgress,
|
|
670
1235
|
workerCostTotal: runtime.workerCostState.total,
|
|
1236
|
+
workerUsageTotal: aggregateWorkerUsage(outcomes),
|
|
1237
|
+
workerSessionMetrics: snapshotWorkerSessionMetrics(runtime.workerSessionMetrics),
|
|
671
1238
|
commit: options.commit,
|
|
672
1239
|
goal: runtime.goal,
|
|
673
1240
|
error: resultError,
|
|
@@ -680,6 +1247,9 @@ export async function runCoordinator(options: RunCoordinatorOptions): Promise<Co
|
|
|
680
1247
|
taskProgress,
|
|
681
1248
|
});
|
|
682
1249
|
return result;
|
|
1250
|
+
} finally {
|
|
1251
|
+
removeSteeringProcessor?.();
|
|
1252
|
+
await workerSessionOwner?.dispose();
|
|
683
1253
|
}
|
|
684
1254
|
}
|
|
685
1255
|
|
|
@@ -1077,6 +1647,11 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1077
1647
|
const workerModelName = options.workerModelName ?? parsedWorkerConfig.modelName;
|
|
1078
1648
|
const workerModel = workerModelName ? undefined : options.workerModel;
|
|
1079
1649
|
const goal = normalizeOptionalText(options.goal);
|
|
1650
|
+
const workerSessionReuseConfig = resolveWorkerSessionReuseConfig({
|
|
1651
|
+
enabled: options.workerSessionReuse ?? parsedWorkerConfig.workerSessionReuseEnabled,
|
|
1652
|
+
contextThresholdPercent:
|
|
1653
|
+
options.workerSessionReuseContextThresholdPercent ?? parsedWorkerConfig.workerSessionReuseContextThresholdPercent,
|
|
1654
|
+
});
|
|
1080
1655
|
|
|
1081
1656
|
return {
|
|
1082
1657
|
cwd,
|
|
@@ -1098,7 +1673,10 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1098
1673
|
goal,
|
|
1099
1674
|
taskThinking: options.taskThinking ?? DEFAULT_COORDINATOR_OPTIONS.taskThinking,
|
|
1100
1675
|
todoThinking: options.todoThinking ?? DEFAULT_COORDINATOR_OPTIONS.todoThinking,
|
|
1676
|
+
workerSessionReuse: workerSessionReuseConfig.enabled,
|
|
1677
|
+
workerSessionReuseContextThresholdPercent: workerSessionReuseConfig.contextThresholdPercent,
|
|
1101
1678
|
workerRunner: options.workerRunner ?? runWorkerTask,
|
|
1679
|
+
useRetainedWorkerLifecycle: options.workerRunner === undefined,
|
|
1102
1680
|
todoPlanner: options.todoPlanner ?? runTodoPlanner,
|
|
1103
1681
|
abortSignal: options.abortSignal,
|
|
1104
1682
|
workerSessionFactory: options.workerSessionFactory,
|
|
@@ -1110,6 +1688,7 @@ function buildRuntimeOptions(options: RunCoordinatorOptions): RuntimeOptions {
|
|
|
1110
1688
|
workerTextByWorker: new Map(),
|
|
1111
1689
|
workerTextPublishedLengthByWorker: new Map(),
|
|
1112
1690
|
plannerDiagnostics: [],
|
|
1691
|
+
workerSessionMetrics: createWorkerSessionMetrics(),
|
|
1113
1692
|
steeringQueue: options.steeringQueue,
|
|
1114
1693
|
onPlanRevisionAccepted: options.onPlanRevisionAccepted,
|
|
1115
1694
|
};
|
|
@@ -1156,6 +1735,41 @@ function recordPlannerDiagnostic(runtime: RuntimeOptions, diagnostic: PlannerDia
|
|
|
1156
1735
|
});
|
|
1157
1736
|
}
|
|
1158
1737
|
|
|
1738
|
+
function aggregateWorkerUsage(outcomes: readonly SessionOutcome[]): WorkerUsageTotals | undefined {
|
|
1739
|
+
const usage = outcomes.flatMap((outcome) => (outcome.workerUsage ? [outcome.workerUsage] : []));
|
|
1740
|
+
if (usage.length === 0) {
|
|
1741
|
+
return undefined;
|
|
1742
|
+
}
|
|
1743
|
+
return usage.reduce<WorkerUsageTotals>(
|
|
1744
|
+
(total, item) => ({
|
|
1745
|
+
input: total.input + item.input,
|
|
1746
|
+
output: total.output + item.output,
|
|
1747
|
+
cacheRead: total.cacheRead + item.cacheRead,
|
|
1748
|
+
cacheWrite: total.cacheWrite + item.cacheWrite,
|
|
1749
|
+
total: total.total + item.total,
|
|
1750
|
+
}),
|
|
1751
|
+
{ input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
|
1752
|
+
);
|
|
1753
|
+
}
|
|
1754
|
+
|
|
1755
|
+
function createWorkerSessionMetrics(): WorkerSessionMetrics {
|
|
1756
|
+
return { starts: 0, reuses: 0, rotations: 0, retained: 0, rotationReasons: {} };
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1759
|
+
function snapshotWorkerSessionMetrics(metrics: WorkerSessionMetrics): WorkerSessionMetrics {
|
|
1760
|
+
return { ...metrics, rotationReasons: { ...metrics.rotationReasons } };
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
function recordWorkerSessionMetric(metrics: WorkerSessionMetrics, diagnostic: WorkerSessionDiagnostic): void {
|
|
1764
|
+
if (diagnostic.event === "session_started") metrics.starts += 1;
|
|
1765
|
+
if (diagnostic.event === "session_reused") metrics.reuses += 1;
|
|
1766
|
+
if (diagnostic.event === "session_retained") metrics.retained += 1;
|
|
1767
|
+
if (diagnostic.event === "session_rotated") {
|
|
1768
|
+
metrics.rotations += 1;
|
|
1769
|
+
metrics.rotationReasons[diagnostic.reasonCode] = (metrics.rotationReasons[diagnostic.reasonCode] ?? 0) + 1;
|
|
1770
|
+
}
|
|
1771
|
+
}
|
|
1772
|
+
|
|
1159
1773
|
function createWorkerCostState(): WorkerCostState {
|
|
1160
1774
|
return {
|
|
1161
1775
|
total: 0,
|
|
@@ -1192,9 +1806,9 @@ function recordLiveWorkerCost(
|
|
|
1192
1806
|
|
|
1193
1807
|
function finalizeWorkerCost(
|
|
1194
1808
|
state: WorkerCostState,
|
|
1195
|
-
|
|
1809
|
+
worker: string,
|
|
1810
|
+
outcome: Pick<SessionOutcome, "workerCostTotal">,
|
|
1196
1811
|
): void {
|
|
1197
|
-
const worker = workerKey(outcome.task.taskId, outcome.attempt);
|
|
1198
1812
|
state.finalizedByWorker.set(worker, finiteNonNegativeNumber(outcome.workerCostTotal) ?? 0);
|
|
1199
1813
|
state.liveByWorker.delete(worker);
|
|
1200
1814
|
for (const messageKey of state.liveByMessage.keys()) {
|
|
@@ -1263,6 +1877,43 @@ function subtaskProgress(
|
|
|
1263
1877
|
});
|
|
1264
1878
|
}
|
|
1265
1879
|
|
|
1880
|
+
function emitWorkerSessionProgress(
|
|
1881
|
+
runtime: RuntimeOptions,
|
|
1882
|
+
tasks: readonly Task[],
|
|
1883
|
+
task: Pick<Task, "taskId" | "title" | "statusItems">,
|
|
1884
|
+
attempts: readonly TaskAttemptSummary[],
|
|
1885
|
+
attempt: number,
|
|
1886
|
+
diagnostic: WorkerSessionDiagnostic,
|
|
1887
|
+
): void {
|
|
1888
|
+
recordWorkerSessionMetric(runtime.workerSessionMetrics, diagnostic);
|
|
1889
|
+
const contextText =
|
|
1890
|
+
diagnostic.contextUsagePercent === undefined
|
|
1891
|
+
? ""
|
|
1892
|
+
: ` at ${diagnostic.contextUsagePercent.toFixed(1)}% context usage`;
|
|
1893
|
+
const action =
|
|
1894
|
+
diagnostic.event === "session_started"
|
|
1895
|
+
? "started"
|
|
1896
|
+
: diagnostic.event === "session_reused"
|
|
1897
|
+
? "reused"
|
|
1898
|
+
: diagnostic.event === "session_rotated"
|
|
1899
|
+
? "rotated"
|
|
1900
|
+
: "retained";
|
|
1901
|
+
emitProgress(runtime, `Worker session ${action}${contextText} (${diagnostic.reasonCode}).`, {
|
|
1902
|
+
phase: "worker_session",
|
|
1903
|
+
taskId: task.taskId,
|
|
1904
|
+
title: task.title,
|
|
1905
|
+
attempt,
|
|
1906
|
+
status: "in_progress",
|
|
1907
|
+
activeStatus: `Worker session ${action}`,
|
|
1908
|
+
workerSessionEvent: diagnostic.event,
|
|
1909
|
+
workerSessionReason: diagnostic.reasonCode,
|
|
1910
|
+
workerSessionContextUsagePercent: diagnostic.contextUsagePercent,
|
|
1911
|
+
workerSessionContextThresholdPercent: diagnostic.contextThresholdPercent,
|
|
1912
|
+
...currentTaskProgress(task, "in_progress"),
|
|
1913
|
+
taskProgress: buildTaskProgressModel({ tasks, attempts, currentTaskId: task.taskId }),
|
|
1914
|
+
});
|
|
1915
|
+
}
|
|
1916
|
+
|
|
1266
1917
|
function emitWorkerEventProgress(
|
|
1267
1918
|
runtime: RuntimeOptions,
|
|
1268
1919
|
tasks: readonly Task[],
|
|
@@ -1278,6 +1929,7 @@ function emitWorkerEventProgress(
|
|
|
1278
1929
|
usageCostTotal?: number;
|
|
1279
1930
|
usageCostKey?: string;
|
|
1280
1931
|
},
|
|
1932
|
+
accountingWorker = workerKey(task.taskId, attempt),
|
|
1281
1933
|
): void {
|
|
1282
1934
|
const worker = workerKey(task.taskId, attempt);
|
|
1283
1935
|
let activeStatus = runtime.workerActivityByWorker.get(worker);
|
|
@@ -1318,7 +1970,7 @@ function emitWorkerEventProgress(
|
|
|
1318
1970
|
}
|
|
1319
1971
|
|
|
1320
1972
|
const costChanged =
|
|
1321
|
-
event.usageCostTotal !== undefined && recordLiveWorkerCost(runtime.workerCostState,
|
|
1973
|
+
event.usageCostTotal !== undefined && recordLiveWorkerCost(runtime.workerCostState, accountingWorker, event);
|
|
1322
1974
|
|
|
1323
1975
|
if (event.type === "message_end" && event.activity) {
|
|
1324
1976
|
emitProgress(runtime, event.activity, {
|
|
@@ -1390,7 +2042,7 @@ function emitWorkerEventProgress(
|
|
|
1390
2042
|
}
|
|
1391
2043
|
|
|
1392
2044
|
function stripToolOutcomePrefix(activity: string): string {
|
|
1393
|
-
return activity.replace(/^(?:(?:Finished|Failed):\s*)+/i, "");
|
|
2045
|
+
return activity.trim().replace(/^(?:(?:Finished|Failed):\s*)+/i, "");
|
|
1394
2046
|
}
|
|
1395
2047
|
|
|
1396
2048
|
function activeStatusFromWorkerText(text: string): string {
|
|
@@ -1629,6 +2281,25 @@ async function appendTaskResult(
|
|
|
1629
2281
|
if (outcome.contextObservations.length > 0) {
|
|
1630
2282
|
lines.push("", "Context observations:", ...outcome.contextObservations.map((item) => `- ${item}`));
|
|
1631
2283
|
}
|
|
2284
|
+
if (outcome.workerCostSource || outcome.workerCostTotal > 0) {
|
|
2285
|
+
lines.push(`Worker cost: ${outcome.workerCostTotal} (${outcome.workerCostSource ?? "unavailable"})`);
|
|
2286
|
+
}
|
|
2287
|
+
if (outcome.workerUsage) {
|
|
2288
|
+
lines.push(
|
|
2289
|
+
`Worker token usage: input=${outcome.workerUsage.input}, output=${outcome.workerUsage.output}, cacheRead=${outcome.workerUsage.cacheRead}, cacheWrite=${outcome.workerUsage.cacheWrite}, total=${outcome.workerUsage.total}`,
|
|
2290
|
+
);
|
|
2291
|
+
}
|
|
2292
|
+
if (outcome.sessionDiagnostics?.length) {
|
|
2293
|
+
lines.push(
|
|
2294
|
+
"",
|
|
2295
|
+
"Worker session diagnostics:",
|
|
2296
|
+
...outcome.sessionDiagnostics.map((item) => {
|
|
2297
|
+
const context =
|
|
2298
|
+
item.contextUsagePercent === undefined ? "" : ` context=${item.contextUsagePercent.toFixed(1)}%`;
|
|
2299
|
+
return `- event=${item.event} reason=${item.reasonCode}${context}`;
|
|
2300
|
+
}),
|
|
2301
|
+
);
|
|
2302
|
+
}
|
|
1632
2303
|
if (outcome.compactionEvents.length > 0) {
|
|
1633
2304
|
lines.push("", "Compaction events:", ...outcome.compactionEvents.map((item) => `- ${item}`));
|
|
1634
2305
|
}
|