taskplane 0.24.7 → 0.24.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/public/app.js +105 -4
- package/dashboard/public/style.css +36 -0
- package/extensions/taskplane/agent-bridge-extension.ts +4 -4
- package/extensions/taskplane/engine.ts +526 -21
- package/extensions/taskplane/execution.ts +18 -9
- package/extensions/taskplane/extension.ts +158 -0
- package/extensions/taskplane/lane-runner.ts +58 -18
- package/extensions/taskplane/persistence.ts +12 -0
- package/extensions/taskplane/resume.ts +267 -24
- package/extensions/taskplane/supervisor-primer.md +10 -0
- package/extensions/taskplane/supervisor.ts +97 -0
- package/extensions/taskplane/types.ts +90 -0
- package/package.json +1 -1
|
@@ -19,9 +19,9 @@ import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-rep
|
|
|
19
19
|
import { resolveOperatorId } from "./naming.ts";
|
|
20
20
|
import { applyPartialProgressToOutcomes, buildTier0EventBase, deleteBatchState, emitEngineEvent, emitTier0Event, loadBatchHistory, loadBatchState, persistRuntimeState, saveBatchHistory, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
21
21
|
import { readRegistrySnapshot, isTerminalStatus, isProcessAlive as registryIsProcessAlive } from "./process-registry.ts";
|
|
22
|
-
import { buildBatchProgressSnapshot, buildEngineEventBase, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
|
|
23
|
-
import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, SupervisorAlert, SupervisorAlertCallback, TaskRunnerConfig, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
24
|
-
import { buildDependencyGraph,
|
|
22
|
+
import { buildBatchProgressSnapshot, buildEngineEventBase, buildSupervisorSegmentFrontierSnapshot, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
|
|
23
|
+
import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, ParsedTask, PersistedSegmentRecord, SupervisorAlert, SupervisorAlertCallback, TaskRunnerConfig, TaskSegmentPlan, TaskSegmentPlanMap, TaskSegmentNode, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
24
|
+
import { buildDependencyGraph, computeWaveAssignments, resolveBaseBranch, resolveRepoRoot, validateGraph } from "./waves.ts";
|
|
25
25
|
import { deleteBranchBestEffort, forceCleanupWorktree, formatPreflightResults, listWorktrees, preserveFailedLaneProgress, removeAllWorktrees, removeWorktree, runPreflight, safeResetWorktree, sleepSync } from "./worktree.ts";
|
|
26
26
|
import { runPreflightCleanup, formatPreflightCleanup } from "./cleanup.ts";
|
|
27
27
|
|
|
@@ -119,6 +119,331 @@ export function resolveBatchHistoryTaskTokens(
|
|
|
119
119
|
return { ...ZERO_TOKENS };
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
+
// ── Segment Frontier Helpers (TP-133) ───────────────────────────────
|
|
123
|
+
|
|
124
|
+
type SegmentLifecycleStatus = "pending" | "running" | "succeeded" | "failed" | "skipped";
|
|
125
|
+
|
|
126
|
+
interface SegmentFrontierTaskState {
|
|
127
|
+
taskId: string;
|
|
128
|
+
orderedSegments: TaskSegmentNode[];
|
|
129
|
+
nextSegmentIndex: number;
|
|
130
|
+
statusBySegmentId: Map<string, SegmentLifecycleStatus>;
|
|
131
|
+
dependsOnBySegmentId: Map<string, string[]>;
|
|
132
|
+
terminalStatus: "pending" | "succeeded" | "failed" | "skipped";
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function buildSegmentDependencyMap(plan: TaskSegmentPlan): Map<string, string[]> {
|
|
136
|
+
const depsBySegmentId = new Map<string, string[]>();
|
|
137
|
+
for (const segment of plan.segments) {
|
|
138
|
+
depsBySegmentId.set(segment.segmentId, []);
|
|
139
|
+
}
|
|
140
|
+
for (const edge of plan.edges) {
|
|
141
|
+
if (!depsBySegmentId.has(edge.toSegmentId)) continue;
|
|
142
|
+
depsBySegmentId.get(edge.toSegmentId)!.push(edge.fromSegmentId);
|
|
143
|
+
}
|
|
144
|
+
for (const [segmentId, deps] of depsBySegmentId.entries()) {
|
|
145
|
+
depsBySegmentId.set(segmentId, [...new Set(deps)].sort((a, b) => a.localeCompare(b)));
|
|
146
|
+
}
|
|
147
|
+
return depsBySegmentId;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function ensureSegmentRecords(batchState: OrchBatchRuntimeState): PersistedSegmentRecord[] {
|
|
151
|
+
if (!batchState.segments) {
|
|
152
|
+
batchState.segments = [];
|
|
153
|
+
}
|
|
154
|
+
return batchState.segments;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function upsertRunningSegmentRecord(
|
|
158
|
+
batchState: OrchBatchRuntimeState,
|
|
159
|
+
task: ParsedTask,
|
|
160
|
+
segmentState: SegmentFrontierTaskState,
|
|
161
|
+
lane: AllocatedLane,
|
|
162
|
+
): boolean {
|
|
163
|
+
const activeSegmentId = task.activeSegmentId;
|
|
164
|
+
if (!activeSegmentId) return false;
|
|
165
|
+
|
|
166
|
+
const activeSegment = segmentState.orderedSegments.find((segment) => segment.segmentId === activeSegmentId);
|
|
167
|
+
if (!activeSegment) return false;
|
|
168
|
+
|
|
169
|
+
const segmentRecords = ensureSegmentRecords(batchState);
|
|
170
|
+
const dependsOnSegmentIds = segmentState.dependsOnBySegmentId.get(activeSegmentId) ?? [];
|
|
171
|
+
const existing = segmentRecords.find((record) => record.segmentId === activeSegmentId);
|
|
172
|
+
const now = Date.now();
|
|
173
|
+
|
|
174
|
+
const restarted = !!existing
|
|
175
|
+
&& existing.status !== "running"
|
|
176
|
+
&& existing.startedAt !== null;
|
|
177
|
+
|
|
178
|
+
const next: PersistedSegmentRecord = {
|
|
179
|
+
segmentId: activeSegmentId,
|
|
180
|
+
taskId: task.taskId,
|
|
181
|
+
repoId: activeSegment.repoId,
|
|
182
|
+
status: "running",
|
|
183
|
+
laneId: lane.laneId,
|
|
184
|
+
sessionName: lane.laneSessionId,
|
|
185
|
+
worktreePath: lane.worktreePath,
|
|
186
|
+
branch: lane.branch,
|
|
187
|
+
startedAt: existing?.status === "running"
|
|
188
|
+
? existing.startedAt
|
|
189
|
+
: (existing?.startedAt ?? now),
|
|
190
|
+
endedAt: null,
|
|
191
|
+
retries: existing
|
|
192
|
+
? existing.retries + (restarted ? 1 : 0)
|
|
193
|
+
: 0,
|
|
194
|
+
exitReason: existing?.status === "running"
|
|
195
|
+
? existing.exitReason
|
|
196
|
+
: "Segment running",
|
|
197
|
+
dependsOnSegmentIds,
|
|
198
|
+
exitDiagnostic: existing?.status === "running"
|
|
199
|
+
? existing.exitDiagnostic
|
|
200
|
+
: undefined,
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
if (!existing) {
|
|
204
|
+
segmentRecords.push(next);
|
|
205
|
+
return true;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const changed =
|
|
209
|
+
existing.taskId !== next.taskId
|
|
210
|
+
|| existing.repoId !== next.repoId
|
|
211
|
+
|| existing.status !== next.status
|
|
212
|
+
|| existing.laneId !== next.laneId
|
|
213
|
+
|| existing.sessionName !== next.sessionName
|
|
214
|
+
|| existing.worktreePath !== next.worktreePath
|
|
215
|
+
|| existing.branch !== next.branch
|
|
216
|
+
|| existing.startedAt !== next.startedAt
|
|
217
|
+
|| existing.endedAt !== next.endedAt
|
|
218
|
+
|| existing.retries !== next.retries
|
|
219
|
+
|| existing.exitReason !== next.exitReason
|
|
220
|
+
|| existing.dependsOnSegmentIds.length !== next.dependsOnSegmentIds.length
|
|
221
|
+
|| existing.dependsOnSegmentIds.some((segmentId, idx) => segmentId !== next.dependsOnSegmentIds[idx])
|
|
222
|
+
|| existing.exitDiagnostic !== next.exitDiagnostic;
|
|
223
|
+
|
|
224
|
+
if (changed) {
|
|
225
|
+
Object.assign(existing, next);
|
|
226
|
+
}
|
|
227
|
+
return changed;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function upsertTerminalSegmentRecord(
|
|
231
|
+
batchState: OrchBatchRuntimeState,
|
|
232
|
+
task: ParsedTask,
|
|
233
|
+
segmentState: SegmentFrontierTaskState,
|
|
234
|
+
segmentId: string,
|
|
235
|
+
status: "succeeded" | "failed" | "skipped",
|
|
236
|
+
outcome: LaneTaskOutcome | undefined,
|
|
237
|
+
lane: AllocatedLane | undefined,
|
|
238
|
+
): boolean {
|
|
239
|
+
const segment = segmentState.orderedSegments.find((candidate) => candidate.segmentId === segmentId);
|
|
240
|
+
if (!segment) return false;
|
|
241
|
+
|
|
242
|
+
const segmentRecords = ensureSegmentRecords(batchState);
|
|
243
|
+
const existing = segmentRecords.find((record) => record.segmentId === segmentId);
|
|
244
|
+
const now = Date.now();
|
|
245
|
+
const dependsOnSegmentIds = segmentState.dependsOnBySegmentId.get(segmentId) ?? [];
|
|
246
|
+
const nextExitDiagnostic = status === "failed"
|
|
247
|
+
? (outcome?.exitDiagnostic ?? existing?.exitDiagnostic)
|
|
248
|
+
: undefined;
|
|
249
|
+
|
|
250
|
+
const next: PersistedSegmentRecord = {
|
|
251
|
+
segmentId,
|
|
252
|
+
taskId: task.taskId,
|
|
253
|
+
repoId: segment.repoId,
|
|
254
|
+
status,
|
|
255
|
+
laneId: lane?.laneId ?? existing?.laneId ?? "",
|
|
256
|
+
sessionName: lane?.laneSessionId ?? existing?.sessionName ?? "",
|
|
257
|
+
worktreePath: lane?.worktreePath ?? existing?.worktreePath ?? "",
|
|
258
|
+
branch: lane?.branch ?? existing?.branch ?? "",
|
|
259
|
+
startedAt: existing?.startedAt ?? outcome?.startTime ?? now,
|
|
260
|
+
endedAt: outcome?.endTime ?? now,
|
|
261
|
+
retries: existing?.retries ?? 0,
|
|
262
|
+
exitReason: outcome?.exitReason ?? (status === "succeeded"
|
|
263
|
+
? "Segment completed"
|
|
264
|
+
: status === "failed"
|
|
265
|
+
? "Segment failed"
|
|
266
|
+
: "Segment skipped"),
|
|
267
|
+
dependsOnSegmentIds,
|
|
268
|
+
exitDiagnostic: nextExitDiagnostic,
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
if (!existing) {
|
|
272
|
+
segmentRecords.push(next);
|
|
273
|
+
return true;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const changed =
|
|
277
|
+
existing.taskId !== next.taskId
|
|
278
|
+
|| existing.repoId !== next.repoId
|
|
279
|
+
|| existing.status !== next.status
|
|
280
|
+
|| existing.laneId !== next.laneId
|
|
281
|
+
|| existing.sessionName !== next.sessionName
|
|
282
|
+
|| existing.worktreePath !== next.worktreePath
|
|
283
|
+
|| existing.branch !== next.branch
|
|
284
|
+
|| existing.startedAt !== next.startedAt
|
|
285
|
+
|| existing.endedAt !== next.endedAt
|
|
286
|
+
|| existing.retries !== next.retries
|
|
287
|
+
|| existing.exitReason !== next.exitReason
|
|
288
|
+
|| existing.dependsOnSegmentIds.length !== next.dependsOnSegmentIds.length
|
|
289
|
+
|| existing.dependsOnSegmentIds.some((depSegmentId, idx) => depSegmentId !== next.dependsOnSegmentIds[idx])
|
|
290
|
+
|| existing.exitDiagnostic !== next.exitDiagnostic;
|
|
291
|
+
|
|
292
|
+
if (changed) {
|
|
293
|
+
Object.assign(existing, next);
|
|
294
|
+
}
|
|
295
|
+
return changed;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function buildFallbackSegmentPlan(taskId: string, task: ParsedTask): TaskSegmentPlan {
|
|
299
|
+
const repoId = (task.resolvedRepoId && task.resolvedRepoId.trim()) || "default";
|
|
300
|
+
return {
|
|
301
|
+
taskId,
|
|
302
|
+
mode: "repo-singleton",
|
|
303
|
+
segments: [
|
|
304
|
+
{
|
|
305
|
+
segmentId: `${taskId}::${repoId}`,
|
|
306
|
+
taskId,
|
|
307
|
+
repoId,
|
|
308
|
+
order: 0,
|
|
309
|
+
},
|
|
310
|
+
],
|
|
311
|
+
edges: [],
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Deterministically linearize one task's segment DAG into a sequential order.
|
|
317
|
+
*
|
|
318
|
+
* Runtime V2 executes one segment per task at a time, so even explicit DAGs
|
|
319
|
+
* are consumed through a deterministic topological order.
|
|
320
|
+
*/
|
|
321
|
+
export function linearizeTaskSegmentPlan(plan: TaskSegmentPlan): TaskSegmentNode[] {
|
|
322
|
+
const nodeById = new Map<string, TaskSegmentNode>();
|
|
323
|
+
for (const segment of plan.segments) {
|
|
324
|
+
nodeById.set(segment.segmentId, segment);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const indegree = new Map<string, number>();
|
|
328
|
+
const outgoing = new Map<string, string[]>();
|
|
329
|
+
for (const segment of plan.segments) {
|
|
330
|
+
indegree.set(segment.segmentId, 0);
|
|
331
|
+
outgoing.set(segment.segmentId, []);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
for (const edge of plan.edges) {
|
|
335
|
+
if (!nodeById.has(edge.fromSegmentId) || !nodeById.has(edge.toSegmentId)) {
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
outgoing.get(edge.fromSegmentId)!.push(edge.toSegmentId);
|
|
339
|
+
indegree.set(edge.toSegmentId, (indegree.get(edge.toSegmentId) ?? 0) + 1);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
for (const list of outgoing.values()) {
|
|
343
|
+
list.sort((a, b) => a.localeCompare(b));
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const ready: TaskSegmentNode[] = plan.segments
|
|
347
|
+
.filter((segment) => (indegree.get(segment.segmentId) ?? 0) === 0)
|
|
348
|
+
.sort((a, b) => (a.order - b.order) || a.segmentId.localeCompare(b.segmentId));
|
|
349
|
+
|
|
350
|
+
const ordered: TaskSegmentNode[] = [];
|
|
351
|
+
while (ready.length > 0) {
|
|
352
|
+
const next = ready.shift()!;
|
|
353
|
+
ordered.push(next);
|
|
354
|
+
for (const dep of outgoing.get(next.segmentId) ?? []) {
|
|
355
|
+
const count = (indegree.get(dep) ?? 0) - 1;
|
|
356
|
+
indegree.set(dep, count);
|
|
357
|
+
if (count === 0) {
|
|
358
|
+
const depNode = nodeById.get(dep);
|
|
359
|
+
if (depNode) {
|
|
360
|
+
ready.push(depNode);
|
|
361
|
+
ready.sort((a, b) => (a.order - b.order) || a.segmentId.localeCompare(b.segmentId));
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// Defensive fallback: malformed/cyclic plans retain deterministic segment order.
|
|
368
|
+
if (ordered.length !== plan.segments.length) {
|
|
369
|
+
return [...plan.segments].sort((a, b) => (a.order - b.order) || a.segmentId.localeCompare(b.segmentId));
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
return ordered;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Expand task waves into segment-frontier rounds.
|
|
377
|
+
*
|
|
378
|
+
* Each original task-wave becomes N rounds where N is the max segment count
|
|
379
|
+
* among tasks in that wave. A task with fewer segments simply drops out once
|
|
380
|
+
* its segment list is exhausted.
|
|
381
|
+
*/
|
|
382
|
+
export function buildSegmentFrontierWaves(
|
|
383
|
+
baseTaskWaves: string[][],
|
|
384
|
+
pending: Map<string, ParsedTask>,
|
|
385
|
+
segmentPlans?: TaskSegmentPlanMap,
|
|
386
|
+
packetRepoId?: string,
|
|
387
|
+
workspaceRoot?: string,
|
|
388
|
+
): { waves: string[][]; taskStateById: Map<string, SegmentFrontierTaskState> } {
|
|
389
|
+
const taskStateById = new Map<string, SegmentFrontierTaskState>();
|
|
390
|
+
|
|
391
|
+
for (const [taskId, task] of pending.entries()) {
|
|
392
|
+
const plan = segmentPlans?.get(taskId) ?? buildFallbackSegmentPlan(taskId, task);
|
|
393
|
+
const orderedSegments = linearizeTaskSegmentPlan(plan);
|
|
394
|
+
const dependsOnBySegmentId = buildSegmentDependencyMap(plan);
|
|
395
|
+
task.segmentIds = orderedSegments.map((segment) => segment.segmentId);
|
|
396
|
+
task.activeSegmentId = null;
|
|
397
|
+
if (packetRepoId) {
|
|
398
|
+
task.packetRepoId = packetRepoId;
|
|
399
|
+
// Resolve packetTaskPath to absolute so it works from any repo's worktree.
|
|
400
|
+
// task.taskFolder is relative to workspace root (e.g., "shared-libs/task-management/.../TP-004").
|
|
401
|
+
// When a segment executes in a different repo, the lane worktree won't contain this path.
|
|
402
|
+
task.packetTaskPath = workspaceRoot
|
|
403
|
+
? resolve(workspaceRoot, task.taskFolder)
|
|
404
|
+
: task.taskFolder;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
taskStateById.set(taskId, {
|
|
408
|
+
taskId,
|
|
409
|
+
orderedSegments,
|
|
410
|
+
nextSegmentIndex: 0,
|
|
411
|
+
statusBySegmentId: new Map(orderedSegments.map((segment) => [segment.segmentId, "pending" as SegmentLifecycleStatus])),
|
|
412
|
+
dependsOnBySegmentId,
|
|
413
|
+
terminalStatus: "pending",
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
const expanded: string[][] = [];
|
|
418
|
+
for (const waveTasks of baseTaskWaves) {
|
|
419
|
+
let maxSegmentsInWave = 0;
|
|
420
|
+
for (const taskId of waveTasks) {
|
|
421
|
+
const state = taskStateById.get(taskId);
|
|
422
|
+
if (!state) continue;
|
|
423
|
+
maxSegmentsInWave = Math.max(maxSegmentsInWave, state.orderedSegments.length);
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
for (let segmentIndex = 0; segmentIndex < maxSegmentsInWave; segmentIndex++) {
|
|
427
|
+
const segmentRound: string[] = [];
|
|
428
|
+
for (const taskId of waveTasks) {
|
|
429
|
+
const state = taskStateById.get(taskId);
|
|
430
|
+
if (!state) continue;
|
|
431
|
+
if (segmentIndex < state.orderedSegments.length) {
|
|
432
|
+
segmentRound.push(taskId);
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
if (segmentRound.length > 0) {
|
|
436
|
+
expanded.push(segmentRound);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
return {
|
|
442
|
+
waves: expanded,
|
|
443
|
+
taskStateById,
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
|
|
122
447
|
/**
|
|
123
448
|
* Attempt automatic retry for failed tasks with retryable exit classifications.
|
|
124
449
|
*
|
|
@@ -950,6 +1275,10 @@ export async function executeOrchBatch(
|
|
|
950
1275
|
let latestAllocatedLanes: AllocatedLane[] = [];
|
|
951
1276
|
// Wave plan as array of task ID arrays (set after wave computation).
|
|
952
1277
|
let wavePlan: string[][] = [];
|
|
1278
|
+
// Segment frontier runtime state keyed by parent task ID.
|
|
1279
|
+
let segmentStateByTask = new Map<string, SegmentFrontierTaskState>();
|
|
1280
|
+
// Tasks that have reached terminal status at segment frontier level.
|
|
1281
|
+
const terminalSegmentTasks = new Set<string>();
|
|
953
1282
|
// Reference to discovery result for enriching taskFolder paths.
|
|
954
1283
|
let discoveryRef: DiscoveryResult | null = null;
|
|
955
1284
|
// TP-029: Track all repo roots encountered during execution.
|
|
@@ -1072,20 +1401,39 @@ export async function executeOrchBatch(
|
|
|
1072
1401
|
return;
|
|
1073
1402
|
}
|
|
1074
1403
|
|
|
1075
|
-
// Compute waves
|
|
1076
|
-
const
|
|
1077
|
-
|
|
1404
|
+
// Compute waves + segment plans (task-level waves with additive segment metadata)
|
|
1405
|
+
const waveComputation = computeWaveAssignments(
|
|
1406
|
+
discovery.pending,
|
|
1407
|
+
discovery.completed,
|
|
1408
|
+
orchConfig,
|
|
1409
|
+
{
|
|
1410
|
+
workspaceRepoIds: workspaceConfig ? workspaceConfig.repos.keys() : undefined,
|
|
1411
|
+
},
|
|
1412
|
+
);
|
|
1413
|
+
if (waveComputation.errors.length > 0) {
|
|
1078
1414
|
batchState.phase = "failed";
|
|
1079
1415
|
batchState.endedAt = Date.now();
|
|
1080
|
-
const errMsgs =
|
|
1416
|
+
const errMsgs = waveComputation.errors.map(e => `[${e.code}] ${e.message}`).join("\n");
|
|
1081
1417
|
batchState.errors.push(`Wave computation failed:\n${errMsgs}`);
|
|
1082
1418
|
onNotify(`❌ Wave computation errors:\n${errMsgs}`, "error");
|
|
1083
1419
|
emitTerminalEvent();
|
|
1084
1420
|
return;
|
|
1085
1421
|
}
|
|
1086
1422
|
|
|
1423
|
+
const taskWaves = waveComputation.waves.map((wave) => wave.tasks.map((assignment) => assignment.taskId));
|
|
1424
|
+
const packetRepoId = workspaceConfig?.routing?.taskPacketRepo;
|
|
1425
|
+
const frontier = buildSegmentFrontierWaves(
|
|
1426
|
+
taskWaves,
|
|
1427
|
+
discovery.pending,
|
|
1428
|
+
waveComputation.segmentPlans,
|
|
1429
|
+
packetRepoId,
|
|
1430
|
+
stateRoot,
|
|
1431
|
+
);
|
|
1432
|
+
const rawWaves = frontier.waves;
|
|
1433
|
+
segmentStateByTask = frontier.taskStateById;
|
|
1434
|
+
|
|
1087
1435
|
batchState.totalWaves = rawWaves.length;
|
|
1088
|
-
batchState.totalTasks =
|
|
1436
|
+
batchState.totalTasks = discovery.pending.size;
|
|
1089
1437
|
|
|
1090
1438
|
// Store wave plan and discovery for state persistence
|
|
1091
1439
|
wavePlan = rawWaves;
|
|
@@ -1180,24 +1528,62 @@ export async function executeOrchBatch(
|
|
|
1180
1528
|
// ── TS-009: Persist state on wave index change ──
|
|
1181
1529
|
persistRuntimeState("wave-index-change", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1182
1530
|
|
|
1183
|
-
// Filter wave tasks against
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1531
|
+
// Filter wave tasks against blocked + terminal task sets, then bind the
|
|
1532
|
+
// next active segment for each surviving task.
|
|
1533
|
+
const scheduledWaveTasks = rawWaves[waveIdx];
|
|
1534
|
+
const blockedInWave: string[] = [];
|
|
1535
|
+
const terminalInWave: string[] = [];
|
|
1536
|
+
let waveTasks: string[] = [];
|
|
1537
|
+
for (const taskId of scheduledWaveTasks) {
|
|
1538
|
+
if (batchState.blockedTaskIds.has(taskId)) {
|
|
1539
|
+
blockedInWave.push(taskId);
|
|
1540
|
+
continue;
|
|
1541
|
+
}
|
|
1542
|
+
if (terminalSegmentTasks.has(taskId)) {
|
|
1543
|
+
terminalInWave.push(taskId);
|
|
1544
|
+
continue;
|
|
1545
|
+
}
|
|
1546
|
+
|
|
1547
|
+
const task = discovery.pending.get(taskId);
|
|
1548
|
+
const segmentState = segmentStateByTask.get(taskId);
|
|
1549
|
+
if (!task || !segmentState) {
|
|
1550
|
+
continue;
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
task.segmentIds = segmentState.orderedSegments.map((segment) => segment.segmentId);
|
|
1554
|
+
const activeSegment = segmentState.orderedSegments[segmentState.nextSegmentIndex] ?? null;
|
|
1555
|
+
if (!activeSegment) {
|
|
1556
|
+
segmentState.terminalStatus = "succeeded";
|
|
1557
|
+
task.activeSegmentId = null;
|
|
1558
|
+
terminalSegmentTasks.add(taskId);
|
|
1559
|
+
terminalInWave.push(taskId);
|
|
1560
|
+
continue;
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
task.activeSegmentId = activeSegment.segmentId;
|
|
1564
|
+
if (workspaceConfig) {
|
|
1565
|
+
task.resolvedRepoId = activeSegment.repoId;
|
|
1566
|
+
}
|
|
1567
|
+
if (segmentState.statusBySegmentId.get(activeSegment.segmentId) === "pending") {
|
|
1568
|
+
segmentState.statusBySegmentId.set(activeSegment.segmentId, "running");
|
|
1569
|
+
}
|
|
1570
|
+
waveTasks.push(taskId);
|
|
1571
|
+
}
|
|
1187
1572
|
|
|
1188
|
-
// Log blocked tasks if any were filtered
|
|
1189
|
-
const blockedInWave = rawWaves[waveIdx].filter(
|
|
1190
|
-
taskId => batchState.blockedTaskIds.has(taskId),
|
|
1191
|
-
);
|
|
1192
1573
|
if (blockedInWave.length > 0) {
|
|
1193
1574
|
execLog("batch", batchState.batchId, `wave ${waveIdx + 1}: skipping ${blockedInWave.length} blocked task(s)`, {
|
|
1194
1575
|
blocked: blockedInWave.join(","),
|
|
1195
1576
|
});
|
|
1196
1577
|
batchState.blockedTasks += blockedInWave.length;
|
|
1197
1578
|
}
|
|
1579
|
+
if (terminalInWave.length > 0) {
|
|
1580
|
+
execLog("batch", batchState.batchId, `wave ${waveIdx + 1}: skipping ${terminalInWave.length} terminal task(s)`, {
|
|
1581
|
+
terminal: terminalInWave.join(","),
|
|
1582
|
+
});
|
|
1583
|
+
}
|
|
1198
1584
|
|
|
1199
1585
|
if (waveTasks.length === 0) {
|
|
1200
|
-
execLog("batch", batchState.batchId, `wave ${waveIdx + 1}: no tasks to execute (all blocked)`);
|
|
1586
|
+
execLog("batch", batchState.batchId, `wave ${waveIdx + 1}: no tasks to execute (all blocked or terminal)`);
|
|
1201
1587
|
continue;
|
|
1202
1588
|
}
|
|
1203
1589
|
|
|
@@ -1229,8 +1615,26 @@ export async function executeOrchBatch(
|
|
|
1229
1615
|
const laneRepoRoot = resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig);
|
|
1230
1616
|
encounteredRepoRoots.set(laneRepoRoot, lane.repoId);
|
|
1231
1617
|
}
|
|
1232
|
-
|
|
1233
|
-
|
|
1618
|
+
const seededPendingOutcomes = seedPendingOutcomesForAllocatedLanes(lanes, allTaskOutcomes);
|
|
1619
|
+
let startedSegments = false;
|
|
1620
|
+
for (const lane of lanes) {
|
|
1621
|
+
for (const laneTask of lane.tasks) {
|
|
1622
|
+
const task = discovery.pending.get(laneTask.taskId);
|
|
1623
|
+
const segmentState = segmentStateByTask.get(laneTask.taskId);
|
|
1624
|
+
if (!task || !segmentState) continue;
|
|
1625
|
+
startedSegments = upsertRunningSegmentRecord(batchState, task, segmentState, lane) || startedSegments;
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
if (seededPendingOutcomes || startedSegments) {
|
|
1629
|
+
persistRuntimeState(
|
|
1630
|
+
startedSegments ? "segment-start" : "wave-lanes-allocated",
|
|
1631
|
+
batchState,
|
|
1632
|
+
wavePlan,
|
|
1633
|
+
latestAllocatedLanes,
|
|
1634
|
+
allTaskOutcomes,
|
|
1635
|
+
discoveryRef,
|
|
1636
|
+
stateRoot,
|
|
1637
|
+
);
|
|
1234
1638
|
}
|
|
1235
1639
|
};
|
|
1236
1640
|
|
|
@@ -1413,6 +1817,76 @@ export async function executeOrchBatch(
|
|
|
1413
1817
|
}
|
|
1414
1818
|
}
|
|
1415
1819
|
|
|
1820
|
+
// Segment frontier lifecycle transitions (pending → running → terminal).
|
|
1821
|
+
const succeededSegmentTaskIdsForMerge = [...waveResult.succeededTaskIds];
|
|
1822
|
+
const completedTaskIdsThisWave: string[] = [];
|
|
1823
|
+
const failedTaskIdsThisWave: string[] = [];
|
|
1824
|
+
const skippedTaskIdsThisWave: string[] = [];
|
|
1825
|
+
const laneByTaskId = new Map<string, AllocatedLane>();
|
|
1826
|
+
for (const lane of latestAllocatedLanes) {
|
|
1827
|
+
for (const laneTask of lane.tasks) {
|
|
1828
|
+
laneByTaskId.set(laneTask.taskId, lane);
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1832
|
+
for (const taskId of waveResult.succeededTaskIds) {
|
|
1833
|
+
const task = discovery.pending.get(taskId);
|
|
1834
|
+
const segmentState = segmentStateByTask.get(taskId);
|
|
1835
|
+
if (!task || !segmentState) continue;
|
|
1836
|
+
|
|
1837
|
+
const activeSegmentId = task.activeSegmentId;
|
|
1838
|
+
if (activeSegmentId) {
|
|
1839
|
+
segmentState.statusBySegmentId.set(activeSegmentId, "succeeded");
|
|
1840
|
+
const outcome = allTaskOutcomes.find((candidate) => candidate.taskId === taskId);
|
|
1841
|
+
upsertTerminalSegmentRecord(batchState, task, segmentState, activeSegmentId, "succeeded", outcome, laneByTaskId.get(taskId));
|
|
1842
|
+
}
|
|
1843
|
+
segmentState.nextSegmentIndex += 1;
|
|
1844
|
+
task.activeSegmentId = null;
|
|
1845
|
+
|
|
1846
|
+
if (segmentState.nextSegmentIndex >= segmentState.orderedSegments.length) {
|
|
1847
|
+
segmentState.terminalStatus = "succeeded";
|
|
1848
|
+
terminalSegmentTasks.add(taskId);
|
|
1849
|
+
completedTaskIdsThisWave.push(taskId);
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
|
|
1853
|
+
for (const taskId of waveResult.failedTaskIds) {
|
|
1854
|
+
const task = discovery.pending.get(taskId);
|
|
1855
|
+
const segmentState = segmentStateByTask.get(taskId);
|
|
1856
|
+
if (!task || !segmentState) continue;
|
|
1857
|
+
const activeSegmentId = task.activeSegmentId;
|
|
1858
|
+
if (activeSegmentId) {
|
|
1859
|
+
segmentState.statusBySegmentId.set(activeSegmentId, "failed");
|
|
1860
|
+
const outcome = allTaskOutcomes.find((candidate) => candidate.taskId === taskId);
|
|
1861
|
+
upsertTerminalSegmentRecord(batchState, task, segmentState, activeSegmentId, "failed", outcome, laneByTaskId.get(taskId));
|
|
1862
|
+
}
|
|
1863
|
+
task.activeSegmentId = null;
|
|
1864
|
+
segmentState.terminalStatus = "failed";
|
|
1865
|
+
terminalSegmentTasks.add(taskId);
|
|
1866
|
+
failedTaskIdsThisWave.push(taskId);
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
for (const taskId of waveResult.skippedTaskIds) {
|
|
1870
|
+
const task = discovery.pending.get(taskId);
|
|
1871
|
+
const segmentState = segmentStateByTask.get(taskId);
|
|
1872
|
+
if (!task || !segmentState) continue;
|
|
1873
|
+
const activeSegmentId = task.activeSegmentId;
|
|
1874
|
+
if (activeSegmentId) {
|
|
1875
|
+
segmentState.statusBySegmentId.set(activeSegmentId, "skipped");
|
|
1876
|
+
const outcome = allTaskOutcomes.find((candidate) => candidate.taskId === taskId);
|
|
1877
|
+
upsertTerminalSegmentRecord(batchState, task, segmentState, activeSegmentId, "skipped", outcome, laneByTaskId.get(taskId));
|
|
1878
|
+
}
|
|
1879
|
+
task.activeSegmentId = null;
|
|
1880
|
+
segmentState.terminalStatus = "skipped";
|
|
1881
|
+
terminalSegmentTasks.add(taskId);
|
|
1882
|
+
skippedTaskIdsThisWave.push(taskId);
|
|
1883
|
+
}
|
|
1884
|
+
|
|
1885
|
+
// Project task-level completion/failure onto wave result arrays.
|
|
1886
|
+
waveResult.succeededTaskIds = [...new Set(completedTaskIdsThisWave)].sort();
|
|
1887
|
+
waveResult.failedTaskIds = [...new Set(failedTaskIdsThisWave)].sort();
|
|
1888
|
+
waveResult.skippedTaskIds = [...new Set(skippedTaskIdsThisWave)].sort();
|
|
1889
|
+
|
|
1416
1890
|
// Accumulate results (after retry so counts reflect recovered tasks)
|
|
1417
1891
|
batchState.succeededTasks += waveResult.succeededTaskIds.length;
|
|
1418
1892
|
batchState.failedTasks += waveResult.failedTaskIds.length;
|
|
@@ -1450,13 +1924,36 @@ export async function executeOrchBatch(
|
|
|
1450
1924
|
|
|
1451
1925
|
// ── TP-076: Emit supervisor alert for task failure ──────
|
|
1452
1926
|
const laneForTask = latestAllocatedLanes.find(l => l.tasks.some(t => t.taskId === taskId));
|
|
1927
|
+
const allocatedTask = laneForTask?.tasks.find(t => t.taskId === taskId)?.task;
|
|
1453
1928
|
const exitReason = outcome?.exitReason || "unknown";
|
|
1454
1929
|
const hasPartialProgress = (outcome?.partialProgressCommits ?? 0) > 0;
|
|
1930
|
+
const segmentFrontier = buildSupervisorSegmentFrontierSnapshot(
|
|
1931
|
+
taskId,
|
|
1932
|
+
allocatedTask?.segmentIds,
|
|
1933
|
+
allocatedTask?.activeSegmentId,
|
|
1934
|
+
batchState.segments,
|
|
1935
|
+
outcome?.segmentId,
|
|
1936
|
+
);
|
|
1937
|
+
const segmentId = outcome?.segmentId
|
|
1938
|
+
?? allocatedTask?.activeSegmentId
|
|
1939
|
+
?? segmentFrontier?.activeSegmentId
|
|
1940
|
+
?? undefined;
|
|
1941
|
+
const repoId = segmentId
|
|
1942
|
+
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ?? laneForTask?.repoId)
|
|
1943
|
+
: laneForTask?.repoId;
|
|
1944
|
+
const segmentSummary = segmentId
|
|
1945
|
+
? ` Segment: ${segmentId}${repoId ? ` (repo: ${repoId})` : ""}\n`
|
|
1946
|
+
: "";
|
|
1947
|
+
const frontierSummary = segmentFrontier
|
|
1948
|
+
? ` Segment frontier: ${segmentFrontier.terminalSegments}/${segmentFrontier.totalSegments} terminal\n`
|
|
1949
|
+
: "";
|
|
1455
1950
|
emitAlert({
|
|
1456
1951
|
category: "task-failure",
|
|
1457
1952
|
summary:
|
|
1458
1953
|
`⚠️ Task failure: ${taskId}\n` +
|
|
1459
1954
|
` Exit reason: ${exitReason}\n` +
|
|
1955
|
+
segmentSummary +
|
|
1956
|
+
frontierSummary +
|
|
1460
1957
|
` Lane: ${laneForTask?.laneId ?? "unknown"} (lane ${laneForTask?.laneNumber ?? "?"})\n` +
|
|
1461
1958
|
` Partial progress preserved: ${hasPartialProgress ? "yes" : "no"}\n` +
|
|
1462
1959
|
` Batch: wave ${waveIdx + 1}/${batchState.totalWaves}, ` +
|
|
@@ -1467,6 +1964,9 @@ export async function executeOrchBatch(
|
|
|
1467
1964
|
` - Read STATUS.md and lane logs for diagnosis`,
|
|
1468
1965
|
context: {
|
|
1469
1966
|
taskId,
|
|
1967
|
+
segmentId,
|
|
1968
|
+
repoId,
|
|
1969
|
+
segmentFrontier,
|
|
1470
1970
|
laneId: laneForTask?.laneId,
|
|
1471
1971
|
laneNumber: laneForTask?.laneNumber,
|
|
1472
1972
|
waveIndex: waveIdx,
|
|
@@ -1519,7 +2019,7 @@ export async function executeOrchBatch(
|
|
|
1519
2019
|
}
|
|
1520
2020
|
|
|
1521
2021
|
// ── Wave Merge ───────────────────────────────────────────
|
|
1522
|
-
//
|
|
2022
|
+
// Merge when at least one segment execution succeeded in this wave.
|
|
1523
2023
|
let mergeResult: MergeWaveResult | null = null;
|
|
1524
2024
|
|
|
1525
2025
|
// Build lane outcome lookup and detect mixed-outcome lanes
|
|
@@ -1536,7 +2036,7 @@ export async function executeOrchBatch(
|
|
|
1536
2036
|
return hasSucceeded && hasHardFailure;
|
|
1537
2037
|
});
|
|
1538
2038
|
|
|
1539
|
-
if (
|
|
2039
|
+
if (succeededSegmentTaskIdsForMerge.length > 0) {
|
|
1540
2040
|
const mergeableLaneCount = waveResult.allocatedLanes.filter(lane => {
|
|
1541
2041
|
const outcome = laneOutcomeByNumber.get(lane.laneNumber);
|
|
1542
2042
|
if (!outcome) return false;
|
|
@@ -1753,6 +2253,7 @@ export async function executeOrchBatch(
|
|
|
1753
2253
|
|
|
1754
2254
|
// ── TP-076: Emit supervisor alert for rollback safe-stop ──
|
|
1755
2255
|
const rollbackError = `Safe-stop at wave ${waveIdx + 1}: verification rollback failed.${persistWarning}`;
|
|
2256
|
+
const rollbackRepoId = extractFailedRepoId(mergeResult) ?? undefined;
|
|
1756
2257
|
emitAlert({
|
|
1757
2258
|
category: "merge-failure",
|
|
1758
2259
|
summary:
|
|
@@ -1766,6 +2267,7 @@ export async function executeOrchBatch(
|
|
|
1766
2267
|
context: {
|
|
1767
2268
|
waveIndex: waveIdx,
|
|
1768
2269
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2270
|
+
repoId: rollbackRepoId,
|
|
1769
2271
|
mergeError: rollbackError,
|
|
1770
2272
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1771
2273
|
},
|
|
@@ -1874,6 +2376,7 @@ export async function executeOrchBatch(
|
|
|
1874
2376
|
context: {
|
|
1875
2377
|
waveIndex: waveIdx,
|
|
1876
2378
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2379
|
+
repoId: mergeRepoId ?? undefined,
|
|
1877
2380
|
mergeError: retryOutcome.errorMessage,
|
|
1878
2381
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1879
2382
|
},
|
|
@@ -1950,6 +2453,7 @@ export async function executeOrchBatch(
|
|
|
1950
2453
|
context: {
|
|
1951
2454
|
waveIndex: waveIdx,
|
|
1952
2455
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2456
|
+
repoId: mergeRepoId ?? undefined,
|
|
1953
2457
|
mergeError: exhaustionMsg,
|
|
1954
2458
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1955
2459
|
},
|
|
@@ -1987,6 +2491,7 @@ export async function executeOrchBatch(
|
|
|
1987
2491
|
context: {
|
|
1988
2492
|
waveIndex: waveIdx,
|
|
1989
2493
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2494
|
+
repoId: mergeRepoId ?? undefined,
|
|
1990
2495
|
mergeError: mergeResult.failureReason || "unknown",
|
|
1991
2496
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1992
2497
|
},
|