taskplane 0.24.6 → 0.24.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/public/app.js +105 -4
- package/dashboard/public/style.css +36 -0
- package/extensions/taskplane/agent-bridge-extension.ts +4 -4
- package/extensions/taskplane/agent-host.ts +3 -1
- package/extensions/taskplane/engine.ts +519 -21
- package/extensions/taskplane/execution.ts +18 -9
- package/extensions/taskplane/extension.ts +158 -0
- package/extensions/taskplane/lane-runner.ts +58 -18
- package/extensions/taskplane/persistence.ts +12 -0
- package/extensions/taskplane/resume.ts +267 -24
- package/extensions/taskplane/supervisor-primer.md +10 -0
- package/extensions/taskplane/supervisor.ts +97 -0
- package/extensions/taskplane/types.ts +90 -0
- package/package.json +1 -1
|
@@ -19,9 +19,9 @@ import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-rep
|
|
|
19
19
|
import { resolveOperatorId } from "./naming.ts";
|
|
20
20
|
import { applyPartialProgressToOutcomes, buildTier0EventBase, deleteBatchState, emitEngineEvent, emitTier0Event, loadBatchHistory, loadBatchState, persistRuntimeState, saveBatchHistory, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
21
21
|
import { readRegistrySnapshot, isTerminalStatus, isProcessAlive as registryIsProcessAlive } from "./process-registry.ts";
|
|
22
|
-
import { buildBatchProgressSnapshot, buildEngineEventBase, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
|
|
23
|
-
import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, SupervisorAlert, SupervisorAlertCallback, TaskRunnerConfig, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
24
|
-
import { buildDependencyGraph,
|
|
22
|
+
import { buildBatchProgressSnapshot, buildEngineEventBase, buildSupervisorSegmentFrontierSnapshot, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
|
|
23
|
+
import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, ParsedTask, PersistedSegmentRecord, SupervisorAlert, SupervisorAlertCallback, TaskRunnerConfig, TaskSegmentPlan, TaskSegmentPlanMap, TaskSegmentNode, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
24
|
+
import { buildDependencyGraph, computeWaveAssignments, resolveBaseBranch, resolveRepoRoot, validateGraph } from "./waves.ts";
|
|
25
25
|
import { deleteBranchBestEffort, forceCleanupWorktree, formatPreflightResults, listWorktrees, preserveFailedLaneProgress, removeAllWorktrees, removeWorktree, runPreflight, safeResetWorktree, sleepSync } from "./worktree.ts";
|
|
26
26
|
import { runPreflightCleanup, formatPreflightCleanup } from "./cleanup.ts";
|
|
27
27
|
|
|
@@ -119,6 +119,325 @@ export function resolveBatchHistoryTaskTokens(
|
|
|
119
119
|
return { ...ZERO_TOKENS };
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
+
// ── Segment Frontier Helpers (TP-133) ───────────────────────────────
|
|
123
|
+
|
|
124
|
+
type SegmentLifecycleStatus = "pending" | "running" | "succeeded" | "failed" | "skipped";
|
|
125
|
+
|
|
126
|
+
interface SegmentFrontierTaskState {
|
|
127
|
+
taskId: string;
|
|
128
|
+
orderedSegments: TaskSegmentNode[];
|
|
129
|
+
nextSegmentIndex: number;
|
|
130
|
+
statusBySegmentId: Map<string, SegmentLifecycleStatus>;
|
|
131
|
+
dependsOnBySegmentId: Map<string, string[]>;
|
|
132
|
+
terminalStatus: "pending" | "succeeded" | "failed" | "skipped";
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function buildSegmentDependencyMap(plan: TaskSegmentPlan): Map<string, string[]> {
|
|
136
|
+
const depsBySegmentId = new Map<string, string[]>();
|
|
137
|
+
for (const segment of plan.segments) {
|
|
138
|
+
depsBySegmentId.set(segment.segmentId, []);
|
|
139
|
+
}
|
|
140
|
+
for (const edge of plan.edges) {
|
|
141
|
+
if (!depsBySegmentId.has(edge.toSegmentId)) continue;
|
|
142
|
+
depsBySegmentId.get(edge.toSegmentId)!.push(edge.fromSegmentId);
|
|
143
|
+
}
|
|
144
|
+
for (const [segmentId, deps] of depsBySegmentId.entries()) {
|
|
145
|
+
depsBySegmentId.set(segmentId, [...new Set(deps)].sort((a, b) => a.localeCompare(b)));
|
|
146
|
+
}
|
|
147
|
+
return depsBySegmentId;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function ensureSegmentRecords(batchState: OrchBatchRuntimeState): PersistedSegmentRecord[] {
|
|
151
|
+
if (!batchState.segments) {
|
|
152
|
+
batchState.segments = [];
|
|
153
|
+
}
|
|
154
|
+
return batchState.segments;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function upsertRunningSegmentRecord(
|
|
158
|
+
batchState: OrchBatchRuntimeState,
|
|
159
|
+
task: ParsedTask,
|
|
160
|
+
segmentState: SegmentFrontierTaskState,
|
|
161
|
+
lane: AllocatedLane,
|
|
162
|
+
): boolean {
|
|
163
|
+
const activeSegmentId = task.activeSegmentId;
|
|
164
|
+
if (!activeSegmentId) return false;
|
|
165
|
+
|
|
166
|
+
const activeSegment = segmentState.orderedSegments.find((segment) => segment.segmentId === activeSegmentId);
|
|
167
|
+
if (!activeSegment) return false;
|
|
168
|
+
|
|
169
|
+
const segmentRecords = ensureSegmentRecords(batchState);
|
|
170
|
+
const dependsOnSegmentIds = segmentState.dependsOnBySegmentId.get(activeSegmentId) ?? [];
|
|
171
|
+
const existing = segmentRecords.find((record) => record.segmentId === activeSegmentId);
|
|
172
|
+
const now = Date.now();
|
|
173
|
+
|
|
174
|
+
const restarted = !!existing
|
|
175
|
+
&& existing.status !== "running"
|
|
176
|
+
&& existing.startedAt !== null;
|
|
177
|
+
|
|
178
|
+
const next: PersistedSegmentRecord = {
|
|
179
|
+
segmentId: activeSegmentId,
|
|
180
|
+
taskId: task.taskId,
|
|
181
|
+
repoId: activeSegment.repoId,
|
|
182
|
+
status: "running",
|
|
183
|
+
laneId: lane.laneId,
|
|
184
|
+
sessionName: lane.laneSessionId,
|
|
185
|
+
worktreePath: lane.worktreePath,
|
|
186
|
+
branch: lane.branch,
|
|
187
|
+
startedAt: existing?.status === "running"
|
|
188
|
+
? existing.startedAt
|
|
189
|
+
: (existing?.startedAt ?? now),
|
|
190
|
+
endedAt: null,
|
|
191
|
+
retries: existing
|
|
192
|
+
? existing.retries + (restarted ? 1 : 0)
|
|
193
|
+
: 0,
|
|
194
|
+
exitReason: existing?.status === "running"
|
|
195
|
+
? existing.exitReason
|
|
196
|
+
: "Segment running",
|
|
197
|
+
dependsOnSegmentIds,
|
|
198
|
+
exitDiagnostic: existing?.status === "running"
|
|
199
|
+
? existing.exitDiagnostic
|
|
200
|
+
: undefined,
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
if (!existing) {
|
|
204
|
+
segmentRecords.push(next);
|
|
205
|
+
return true;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const changed =
|
|
209
|
+
existing.taskId !== next.taskId
|
|
210
|
+
|| existing.repoId !== next.repoId
|
|
211
|
+
|| existing.status !== next.status
|
|
212
|
+
|| existing.laneId !== next.laneId
|
|
213
|
+
|| existing.sessionName !== next.sessionName
|
|
214
|
+
|| existing.worktreePath !== next.worktreePath
|
|
215
|
+
|| existing.branch !== next.branch
|
|
216
|
+
|| existing.startedAt !== next.startedAt
|
|
217
|
+
|| existing.endedAt !== next.endedAt
|
|
218
|
+
|| existing.retries !== next.retries
|
|
219
|
+
|| existing.exitReason !== next.exitReason
|
|
220
|
+
|| existing.dependsOnSegmentIds.length !== next.dependsOnSegmentIds.length
|
|
221
|
+
|| existing.dependsOnSegmentIds.some((segmentId, idx) => segmentId !== next.dependsOnSegmentIds[idx])
|
|
222
|
+
|| existing.exitDiagnostic !== next.exitDiagnostic;
|
|
223
|
+
|
|
224
|
+
if (changed) {
|
|
225
|
+
Object.assign(existing, next);
|
|
226
|
+
}
|
|
227
|
+
return changed;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function upsertTerminalSegmentRecord(
|
|
231
|
+
batchState: OrchBatchRuntimeState,
|
|
232
|
+
task: ParsedTask,
|
|
233
|
+
segmentState: SegmentFrontierTaskState,
|
|
234
|
+
segmentId: string,
|
|
235
|
+
status: "succeeded" | "failed" | "skipped",
|
|
236
|
+
outcome: LaneTaskOutcome | undefined,
|
|
237
|
+
lane: AllocatedLane | undefined,
|
|
238
|
+
): boolean {
|
|
239
|
+
const segment = segmentState.orderedSegments.find((candidate) => candidate.segmentId === segmentId);
|
|
240
|
+
if (!segment) return false;
|
|
241
|
+
|
|
242
|
+
const segmentRecords = ensureSegmentRecords(batchState);
|
|
243
|
+
const existing = segmentRecords.find((record) => record.segmentId === segmentId);
|
|
244
|
+
const now = Date.now();
|
|
245
|
+
const dependsOnSegmentIds = segmentState.dependsOnBySegmentId.get(segmentId) ?? [];
|
|
246
|
+
const nextExitDiagnostic = status === "failed"
|
|
247
|
+
? (outcome?.exitDiagnostic ?? existing?.exitDiagnostic)
|
|
248
|
+
: undefined;
|
|
249
|
+
|
|
250
|
+
const next: PersistedSegmentRecord = {
|
|
251
|
+
segmentId,
|
|
252
|
+
taskId: task.taskId,
|
|
253
|
+
repoId: segment.repoId,
|
|
254
|
+
status,
|
|
255
|
+
laneId: lane?.laneId ?? existing?.laneId ?? "",
|
|
256
|
+
sessionName: lane?.laneSessionId ?? existing?.sessionName ?? "",
|
|
257
|
+
worktreePath: lane?.worktreePath ?? existing?.worktreePath ?? "",
|
|
258
|
+
branch: lane?.branch ?? existing?.branch ?? "",
|
|
259
|
+
startedAt: existing?.startedAt ?? outcome?.startTime ?? now,
|
|
260
|
+
endedAt: outcome?.endTime ?? now,
|
|
261
|
+
retries: existing?.retries ?? 0,
|
|
262
|
+
exitReason: outcome?.exitReason ?? (status === "succeeded"
|
|
263
|
+
? "Segment completed"
|
|
264
|
+
: status === "failed"
|
|
265
|
+
? "Segment failed"
|
|
266
|
+
: "Segment skipped"),
|
|
267
|
+
dependsOnSegmentIds,
|
|
268
|
+
exitDiagnostic: nextExitDiagnostic,
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
if (!existing) {
|
|
272
|
+
segmentRecords.push(next);
|
|
273
|
+
return true;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const changed =
|
|
277
|
+
existing.taskId !== next.taskId
|
|
278
|
+
|| existing.repoId !== next.repoId
|
|
279
|
+
|| existing.status !== next.status
|
|
280
|
+
|| existing.laneId !== next.laneId
|
|
281
|
+
|| existing.sessionName !== next.sessionName
|
|
282
|
+
|| existing.worktreePath !== next.worktreePath
|
|
283
|
+
|| existing.branch !== next.branch
|
|
284
|
+
|| existing.startedAt !== next.startedAt
|
|
285
|
+
|| existing.endedAt !== next.endedAt
|
|
286
|
+
|| existing.retries !== next.retries
|
|
287
|
+
|| existing.exitReason !== next.exitReason
|
|
288
|
+
|| existing.dependsOnSegmentIds.length !== next.dependsOnSegmentIds.length
|
|
289
|
+
|| existing.dependsOnSegmentIds.some((depSegmentId, idx) => depSegmentId !== next.dependsOnSegmentIds[idx])
|
|
290
|
+
|| existing.exitDiagnostic !== next.exitDiagnostic;
|
|
291
|
+
|
|
292
|
+
if (changed) {
|
|
293
|
+
Object.assign(existing, next);
|
|
294
|
+
}
|
|
295
|
+
return changed;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function buildFallbackSegmentPlan(taskId: string, task: ParsedTask): TaskSegmentPlan {
|
|
299
|
+
const repoId = (task.resolvedRepoId && task.resolvedRepoId.trim()) || "default";
|
|
300
|
+
return {
|
|
301
|
+
taskId,
|
|
302
|
+
mode: "repo-singleton",
|
|
303
|
+
segments: [
|
|
304
|
+
{
|
|
305
|
+
segmentId: `${taskId}::${repoId}`,
|
|
306
|
+
taskId,
|
|
307
|
+
repoId,
|
|
308
|
+
order: 0,
|
|
309
|
+
},
|
|
310
|
+
],
|
|
311
|
+
edges: [],
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Deterministically linearize one task's segment DAG into a sequential order.
|
|
317
|
+
*
|
|
318
|
+
* Runtime V2 executes one segment per task at a time, so even explicit DAGs
|
|
319
|
+
* are consumed through a deterministic topological order.
|
|
320
|
+
*/
|
|
321
|
+
export function linearizeTaskSegmentPlan(plan: TaskSegmentPlan): TaskSegmentNode[] {
|
|
322
|
+
const nodeById = new Map<string, TaskSegmentNode>();
|
|
323
|
+
for (const segment of plan.segments) {
|
|
324
|
+
nodeById.set(segment.segmentId, segment);
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const indegree = new Map<string, number>();
|
|
328
|
+
const outgoing = new Map<string, string[]>();
|
|
329
|
+
for (const segment of plan.segments) {
|
|
330
|
+
indegree.set(segment.segmentId, 0);
|
|
331
|
+
outgoing.set(segment.segmentId, []);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
for (const edge of plan.edges) {
|
|
335
|
+
if (!nodeById.has(edge.fromSegmentId) || !nodeById.has(edge.toSegmentId)) {
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
outgoing.get(edge.fromSegmentId)!.push(edge.toSegmentId);
|
|
339
|
+
indegree.set(edge.toSegmentId, (indegree.get(edge.toSegmentId) ?? 0) + 1);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
for (const list of outgoing.values()) {
|
|
343
|
+
list.sort((a, b) => a.localeCompare(b));
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const ready: TaskSegmentNode[] = plan.segments
|
|
347
|
+
.filter((segment) => (indegree.get(segment.segmentId) ?? 0) === 0)
|
|
348
|
+
.sort((a, b) => (a.order - b.order) || a.segmentId.localeCompare(b.segmentId));
|
|
349
|
+
|
|
350
|
+
const ordered: TaskSegmentNode[] = [];
|
|
351
|
+
while (ready.length > 0) {
|
|
352
|
+
const next = ready.shift()!;
|
|
353
|
+
ordered.push(next);
|
|
354
|
+
for (const dep of outgoing.get(next.segmentId) ?? []) {
|
|
355
|
+
const count = (indegree.get(dep) ?? 0) - 1;
|
|
356
|
+
indegree.set(dep, count);
|
|
357
|
+
if (count === 0) {
|
|
358
|
+
const depNode = nodeById.get(dep);
|
|
359
|
+
if (depNode) {
|
|
360
|
+
ready.push(depNode);
|
|
361
|
+
ready.sort((a, b) => (a.order - b.order) || a.segmentId.localeCompare(b.segmentId));
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// Defensive fallback: malformed/cyclic plans retain deterministic segment order.
|
|
368
|
+
if (ordered.length !== plan.segments.length) {
|
|
369
|
+
return [...plan.segments].sort((a, b) => (a.order - b.order) || a.segmentId.localeCompare(b.segmentId));
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
return ordered;
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/**
|
|
376
|
+
* Expand task waves into segment-frontier rounds.
|
|
377
|
+
*
|
|
378
|
+
* Each original task-wave becomes N rounds where N is the max segment count
|
|
379
|
+
* among tasks in that wave. A task with fewer segments simply drops out once
|
|
380
|
+
* its segment list is exhausted.
|
|
381
|
+
*/
|
|
382
|
+
export function buildSegmentFrontierWaves(
|
|
383
|
+
baseTaskWaves: string[][],
|
|
384
|
+
pending: Map<string, ParsedTask>,
|
|
385
|
+
segmentPlans?: TaskSegmentPlanMap,
|
|
386
|
+
packetRepoId?: string,
|
|
387
|
+
): { waves: string[][]; taskStateById: Map<string, SegmentFrontierTaskState> } {
|
|
388
|
+
const taskStateById = new Map<string, SegmentFrontierTaskState>();
|
|
389
|
+
|
|
390
|
+
for (const [taskId, task] of pending.entries()) {
|
|
391
|
+
const plan = segmentPlans?.get(taskId) ?? buildFallbackSegmentPlan(taskId, task);
|
|
392
|
+
const orderedSegments = linearizeTaskSegmentPlan(plan);
|
|
393
|
+
const dependsOnBySegmentId = buildSegmentDependencyMap(plan);
|
|
394
|
+
task.segmentIds = orderedSegments.map((segment) => segment.segmentId);
|
|
395
|
+
task.activeSegmentId = null;
|
|
396
|
+
if (packetRepoId) {
|
|
397
|
+
task.packetRepoId = packetRepoId;
|
|
398
|
+
task.packetTaskPath = task.taskFolder;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
taskStateById.set(taskId, {
|
|
402
|
+
taskId,
|
|
403
|
+
orderedSegments,
|
|
404
|
+
nextSegmentIndex: 0,
|
|
405
|
+
statusBySegmentId: new Map(orderedSegments.map((segment) => [segment.segmentId, "pending" as SegmentLifecycleStatus])),
|
|
406
|
+
dependsOnBySegmentId,
|
|
407
|
+
terminalStatus: "pending",
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const expanded: string[][] = [];
|
|
412
|
+
for (const waveTasks of baseTaskWaves) {
|
|
413
|
+
let maxSegmentsInWave = 0;
|
|
414
|
+
for (const taskId of waveTasks) {
|
|
415
|
+
const state = taskStateById.get(taskId);
|
|
416
|
+
if (!state) continue;
|
|
417
|
+
maxSegmentsInWave = Math.max(maxSegmentsInWave, state.orderedSegments.length);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
for (let segmentIndex = 0; segmentIndex < maxSegmentsInWave; segmentIndex++) {
|
|
421
|
+
const segmentRound: string[] = [];
|
|
422
|
+
for (const taskId of waveTasks) {
|
|
423
|
+
const state = taskStateById.get(taskId);
|
|
424
|
+
if (!state) continue;
|
|
425
|
+
if (segmentIndex < state.orderedSegments.length) {
|
|
426
|
+
segmentRound.push(taskId);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
if (segmentRound.length > 0) {
|
|
430
|
+
expanded.push(segmentRound);
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
return {
|
|
436
|
+
waves: expanded,
|
|
437
|
+
taskStateById,
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
|
|
122
441
|
/**
|
|
123
442
|
* Attempt automatic retry for failed tasks with retryable exit classifications.
|
|
124
443
|
*
|
|
@@ -950,6 +1269,10 @@ export async function executeOrchBatch(
|
|
|
950
1269
|
let latestAllocatedLanes: AllocatedLane[] = [];
|
|
951
1270
|
// Wave plan as array of task ID arrays (set after wave computation).
|
|
952
1271
|
let wavePlan: string[][] = [];
|
|
1272
|
+
// Segment frontier runtime state keyed by parent task ID.
|
|
1273
|
+
let segmentStateByTask = new Map<string, SegmentFrontierTaskState>();
|
|
1274
|
+
// Tasks that have reached terminal status at segment frontier level.
|
|
1275
|
+
const terminalSegmentTasks = new Set<string>();
|
|
953
1276
|
// Reference to discovery result for enriching taskFolder paths.
|
|
954
1277
|
let discoveryRef: DiscoveryResult | null = null;
|
|
955
1278
|
// TP-029: Track all repo roots encountered during execution.
|
|
@@ -1072,20 +1395,38 @@ export async function executeOrchBatch(
|
|
|
1072
1395
|
return;
|
|
1073
1396
|
}
|
|
1074
1397
|
|
|
1075
|
-
// Compute waves
|
|
1076
|
-
const
|
|
1077
|
-
|
|
1398
|
+
// Compute waves + segment plans (task-level waves with additive segment metadata)
|
|
1399
|
+
const waveComputation = computeWaveAssignments(
|
|
1400
|
+
discovery.pending,
|
|
1401
|
+
discovery.completed,
|
|
1402
|
+
orchConfig,
|
|
1403
|
+
{
|
|
1404
|
+
workspaceRepoIds: workspaceConfig ? workspaceConfig.repos.keys() : undefined,
|
|
1405
|
+
},
|
|
1406
|
+
);
|
|
1407
|
+
if (waveComputation.errors.length > 0) {
|
|
1078
1408
|
batchState.phase = "failed";
|
|
1079
1409
|
batchState.endedAt = Date.now();
|
|
1080
|
-
const errMsgs =
|
|
1410
|
+
const errMsgs = waveComputation.errors.map(e => `[${e.code}] ${e.message}`).join("\n");
|
|
1081
1411
|
batchState.errors.push(`Wave computation failed:\n${errMsgs}`);
|
|
1082
1412
|
onNotify(`❌ Wave computation errors:\n${errMsgs}`, "error");
|
|
1083
1413
|
emitTerminalEvent();
|
|
1084
1414
|
return;
|
|
1085
1415
|
}
|
|
1086
1416
|
|
|
1417
|
+
const taskWaves = waveComputation.waves.map((wave) => wave.tasks.map((assignment) => assignment.taskId));
|
|
1418
|
+
const packetRepoId = workspaceConfig?.routing?.taskPacketRepo;
|
|
1419
|
+
const frontier = buildSegmentFrontierWaves(
|
|
1420
|
+
taskWaves,
|
|
1421
|
+
discovery.pending,
|
|
1422
|
+
waveComputation.segmentPlans,
|
|
1423
|
+
packetRepoId,
|
|
1424
|
+
);
|
|
1425
|
+
const rawWaves = frontier.waves;
|
|
1426
|
+
segmentStateByTask = frontier.taskStateById;
|
|
1427
|
+
|
|
1087
1428
|
batchState.totalWaves = rawWaves.length;
|
|
1088
|
-
batchState.totalTasks =
|
|
1429
|
+
batchState.totalTasks = discovery.pending.size;
|
|
1089
1430
|
|
|
1090
1431
|
// Store wave plan and discovery for state persistence
|
|
1091
1432
|
wavePlan = rawWaves;
|
|
@@ -1180,24 +1521,62 @@ export async function executeOrchBatch(
|
|
|
1180
1521
|
// ── TS-009: Persist state on wave index change ──
|
|
1181
1522
|
persistRuntimeState("wave-index-change", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1182
1523
|
|
|
1183
|
-
// Filter wave tasks against
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1524
|
+
// Filter wave tasks against blocked + terminal task sets, then bind the
|
|
1525
|
+
// next active segment for each surviving task.
|
|
1526
|
+
const scheduledWaveTasks = rawWaves[waveIdx];
|
|
1527
|
+
const blockedInWave: string[] = [];
|
|
1528
|
+
const terminalInWave: string[] = [];
|
|
1529
|
+
let waveTasks: string[] = [];
|
|
1530
|
+
for (const taskId of scheduledWaveTasks) {
|
|
1531
|
+
if (batchState.blockedTaskIds.has(taskId)) {
|
|
1532
|
+
blockedInWave.push(taskId);
|
|
1533
|
+
continue;
|
|
1534
|
+
}
|
|
1535
|
+
if (terminalSegmentTasks.has(taskId)) {
|
|
1536
|
+
terminalInWave.push(taskId);
|
|
1537
|
+
continue;
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
const task = discovery.pending.get(taskId);
|
|
1541
|
+
const segmentState = segmentStateByTask.get(taskId);
|
|
1542
|
+
if (!task || !segmentState) {
|
|
1543
|
+
continue;
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
task.segmentIds = segmentState.orderedSegments.map((segment) => segment.segmentId);
|
|
1547
|
+
const activeSegment = segmentState.orderedSegments[segmentState.nextSegmentIndex] ?? null;
|
|
1548
|
+
if (!activeSegment) {
|
|
1549
|
+
segmentState.terminalStatus = "succeeded";
|
|
1550
|
+
task.activeSegmentId = null;
|
|
1551
|
+
terminalSegmentTasks.add(taskId);
|
|
1552
|
+
terminalInWave.push(taskId);
|
|
1553
|
+
continue;
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
task.activeSegmentId = activeSegment.segmentId;
|
|
1557
|
+
if (workspaceConfig) {
|
|
1558
|
+
task.resolvedRepoId = activeSegment.repoId;
|
|
1559
|
+
}
|
|
1560
|
+
if (segmentState.statusBySegmentId.get(activeSegment.segmentId) === "pending") {
|
|
1561
|
+
segmentState.statusBySegmentId.set(activeSegment.segmentId, "running");
|
|
1562
|
+
}
|
|
1563
|
+
waveTasks.push(taskId);
|
|
1564
|
+
}
|
|
1187
1565
|
|
|
1188
|
-
// Log blocked tasks if any were filtered
|
|
1189
|
-
const blockedInWave = rawWaves[waveIdx].filter(
|
|
1190
|
-
taskId => batchState.blockedTaskIds.has(taskId),
|
|
1191
|
-
);
|
|
1192
1566
|
if (blockedInWave.length > 0) {
|
|
1193
1567
|
execLog("batch", batchState.batchId, `wave ${waveIdx + 1}: skipping ${blockedInWave.length} blocked task(s)`, {
|
|
1194
1568
|
blocked: blockedInWave.join(","),
|
|
1195
1569
|
});
|
|
1196
1570
|
batchState.blockedTasks += blockedInWave.length;
|
|
1197
1571
|
}
|
|
1572
|
+
if (terminalInWave.length > 0) {
|
|
1573
|
+
execLog("batch", batchState.batchId, `wave ${waveIdx + 1}: skipping ${terminalInWave.length} terminal task(s)`, {
|
|
1574
|
+
terminal: terminalInWave.join(","),
|
|
1575
|
+
});
|
|
1576
|
+
}
|
|
1198
1577
|
|
|
1199
1578
|
if (waveTasks.length === 0) {
|
|
1200
|
-
execLog("batch", batchState.batchId, `wave ${waveIdx + 1}: no tasks to execute (all blocked)`);
|
|
1579
|
+
execLog("batch", batchState.batchId, `wave ${waveIdx + 1}: no tasks to execute (all blocked or terminal)`);
|
|
1201
1580
|
continue;
|
|
1202
1581
|
}
|
|
1203
1582
|
|
|
@@ -1229,8 +1608,26 @@ export async function executeOrchBatch(
|
|
|
1229
1608
|
const laneRepoRoot = resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig);
|
|
1230
1609
|
encounteredRepoRoots.set(laneRepoRoot, lane.repoId);
|
|
1231
1610
|
}
|
|
1232
|
-
|
|
1233
|
-
|
|
1611
|
+
const seededPendingOutcomes = seedPendingOutcomesForAllocatedLanes(lanes, allTaskOutcomes);
|
|
1612
|
+
let startedSegments = false;
|
|
1613
|
+
for (const lane of lanes) {
|
|
1614
|
+
for (const laneTask of lane.tasks) {
|
|
1615
|
+
const task = discovery.pending.get(laneTask.taskId);
|
|
1616
|
+
const segmentState = segmentStateByTask.get(laneTask.taskId);
|
|
1617
|
+
if (!task || !segmentState) continue;
|
|
1618
|
+
startedSegments = upsertRunningSegmentRecord(batchState, task, segmentState, lane) || startedSegments;
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
if (seededPendingOutcomes || startedSegments) {
|
|
1622
|
+
persistRuntimeState(
|
|
1623
|
+
startedSegments ? "segment-start" : "wave-lanes-allocated",
|
|
1624
|
+
batchState,
|
|
1625
|
+
wavePlan,
|
|
1626
|
+
latestAllocatedLanes,
|
|
1627
|
+
allTaskOutcomes,
|
|
1628
|
+
discoveryRef,
|
|
1629
|
+
stateRoot,
|
|
1630
|
+
);
|
|
1234
1631
|
}
|
|
1235
1632
|
};
|
|
1236
1633
|
|
|
@@ -1413,6 +1810,76 @@ export async function executeOrchBatch(
|
|
|
1413
1810
|
}
|
|
1414
1811
|
}
|
|
1415
1812
|
|
|
1813
|
+
// Segment frontier lifecycle transitions (pending → running → terminal).
|
|
1814
|
+
const succeededSegmentTaskIdsForMerge = [...waveResult.succeededTaskIds];
|
|
1815
|
+
const completedTaskIdsThisWave: string[] = [];
|
|
1816
|
+
const failedTaskIdsThisWave: string[] = [];
|
|
1817
|
+
const skippedTaskIdsThisWave: string[] = [];
|
|
1818
|
+
const laneByTaskId = new Map<string, AllocatedLane>();
|
|
1819
|
+
for (const lane of latestAllocatedLanes) {
|
|
1820
|
+
for (const laneTask of lane.tasks) {
|
|
1821
|
+
laneByTaskId.set(laneTask.taskId, lane);
|
|
1822
|
+
}
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
for (const taskId of waveResult.succeededTaskIds) {
|
|
1826
|
+
const task = discovery.pending.get(taskId);
|
|
1827
|
+
const segmentState = segmentStateByTask.get(taskId);
|
|
1828
|
+
if (!task || !segmentState) continue;
|
|
1829
|
+
|
|
1830
|
+
const activeSegmentId = task.activeSegmentId;
|
|
1831
|
+
if (activeSegmentId) {
|
|
1832
|
+
segmentState.statusBySegmentId.set(activeSegmentId, "succeeded");
|
|
1833
|
+
const outcome = allTaskOutcomes.find((candidate) => candidate.taskId === taskId);
|
|
1834
|
+
upsertTerminalSegmentRecord(batchState, task, segmentState, activeSegmentId, "succeeded", outcome, laneByTaskId.get(taskId));
|
|
1835
|
+
}
|
|
1836
|
+
segmentState.nextSegmentIndex += 1;
|
|
1837
|
+
task.activeSegmentId = null;
|
|
1838
|
+
|
|
1839
|
+
if (segmentState.nextSegmentIndex >= segmentState.orderedSegments.length) {
|
|
1840
|
+
segmentState.terminalStatus = "succeeded";
|
|
1841
|
+
terminalSegmentTasks.add(taskId);
|
|
1842
|
+
completedTaskIdsThisWave.push(taskId);
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
|
|
1846
|
+
for (const taskId of waveResult.failedTaskIds) {
|
|
1847
|
+
const task = discovery.pending.get(taskId);
|
|
1848
|
+
const segmentState = segmentStateByTask.get(taskId);
|
|
1849
|
+
if (!task || !segmentState) continue;
|
|
1850
|
+
const activeSegmentId = task.activeSegmentId;
|
|
1851
|
+
if (activeSegmentId) {
|
|
1852
|
+
segmentState.statusBySegmentId.set(activeSegmentId, "failed");
|
|
1853
|
+
const outcome = allTaskOutcomes.find((candidate) => candidate.taskId === taskId);
|
|
1854
|
+
upsertTerminalSegmentRecord(batchState, task, segmentState, activeSegmentId, "failed", outcome, laneByTaskId.get(taskId));
|
|
1855
|
+
}
|
|
1856
|
+
task.activeSegmentId = null;
|
|
1857
|
+
segmentState.terminalStatus = "failed";
|
|
1858
|
+
terminalSegmentTasks.add(taskId);
|
|
1859
|
+
failedTaskIdsThisWave.push(taskId);
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
for (const taskId of waveResult.skippedTaskIds) {
|
|
1863
|
+
const task = discovery.pending.get(taskId);
|
|
1864
|
+
const segmentState = segmentStateByTask.get(taskId);
|
|
1865
|
+
if (!task || !segmentState) continue;
|
|
1866
|
+
const activeSegmentId = task.activeSegmentId;
|
|
1867
|
+
if (activeSegmentId) {
|
|
1868
|
+
segmentState.statusBySegmentId.set(activeSegmentId, "skipped");
|
|
1869
|
+
const outcome = allTaskOutcomes.find((candidate) => candidate.taskId === taskId);
|
|
1870
|
+
upsertTerminalSegmentRecord(batchState, task, segmentState, activeSegmentId, "skipped", outcome, laneByTaskId.get(taskId));
|
|
1871
|
+
}
|
|
1872
|
+
task.activeSegmentId = null;
|
|
1873
|
+
segmentState.terminalStatus = "skipped";
|
|
1874
|
+
terminalSegmentTasks.add(taskId);
|
|
1875
|
+
skippedTaskIdsThisWave.push(taskId);
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1878
|
+
// Project task-level completion/failure onto wave result arrays.
|
|
1879
|
+
waveResult.succeededTaskIds = [...new Set(completedTaskIdsThisWave)].sort();
|
|
1880
|
+
waveResult.failedTaskIds = [...new Set(failedTaskIdsThisWave)].sort();
|
|
1881
|
+
waveResult.skippedTaskIds = [...new Set(skippedTaskIdsThisWave)].sort();
|
|
1882
|
+
|
|
1416
1883
|
// Accumulate results (after retry so counts reflect recovered tasks)
|
|
1417
1884
|
batchState.succeededTasks += waveResult.succeededTaskIds.length;
|
|
1418
1885
|
batchState.failedTasks += waveResult.failedTaskIds.length;
|
|
@@ -1450,13 +1917,36 @@ export async function executeOrchBatch(
|
|
|
1450
1917
|
|
|
1451
1918
|
// ── TP-076: Emit supervisor alert for task failure ──────
|
|
1452
1919
|
const laneForTask = latestAllocatedLanes.find(l => l.tasks.some(t => t.taskId === taskId));
|
|
1920
|
+
const allocatedTask = laneForTask?.tasks.find(t => t.taskId === taskId)?.task;
|
|
1453
1921
|
const exitReason = outcome?.exitReason || "unknown";
|
|
1454
1922
|
const hasPartialProgress = (outcome?.partialProgressCommits ?? 0) > 0;
|
|
1923
|
+
const segmentFrontier = buildSupervisorSegmentFrontierSnapshot(
|
|
1924
|
+
taskId,
|
|
1925
|
+
allocatedTask?.segmentIds,
|
|
1926
|
+
allocatedTask?.activeSegmentId,
|
|
1927
|
+
batchState.segments,
|
|
1928
|
+
outcome?.segmentId,
|
|
1929
|
+
);
|
|
1930
|
+
const segmentId = outcome?.segmentId
|
|
1931
|
+
?? allocatedTask?.activeSegmentId
|
|
1932
|
+
?? segmentFrontier?.activeSegmentId
|
|
1933
|
+
?? undefined;
|
|
1934
|
+
const repoId = segmentId
|
|
1935
|
+
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ?? laneForTask?.repoId)
|
|
1936
|
+
: laneForTask?.repoId;
|
|
1937
|
+
const segmentSummary = segmentId
|
|
1938
|
+
? ` Segment: ${segmentId}${repoId ? ` (repo: ${repoId})` : ""}\n`
|
|
1939
|
+
: "";
|
|
1940
|
+
const frontierSummary = segmentFrontier
|
|
1941
|
+
? ` Segment frontier: ${segmentFrontier.terminalSegments}/${segmentFrontier.totalSegments} terminal\n`
|
|
1942
|
+
: "";
|
|
1455
1943
|
emitAlert({
|
|
1456
1944
|
category: "task-failure",
|
|
1457
1945
|
summary:
|
|
1458
1946
|
`⚠️ Task failure: ${taskId}\n` +
|
|
1459
1947
|
` Exit reason: ${exitReason}\n` +
|
|
1948
|
+
segmentSummary +
|
|
1949
|
+
frontierSummary +
|
|
1460
1950
|
` Lane: ${laneForTask?.laneId ?? "unknown"} (lane ${laneForTask?.laneNumber ?? "?"})\n` +
|
|
1461
1951
|
` Partial progress preserved: ${hasPartialProgress ? "yes" : "no"}\n` +
|
|
1462
1952
|
` Batch: wave ${waveIdx + 1}/${batchState.totalWaves}, ` +
|
|
@@ -1467,6 +1957,9 @@ export async function executeOrchBatch(
|
|
|
1467
1957
|
` - Read STATUS.md and lane logs for diagnosis`,
|
|
1468
1958
|
context: {
|
|
1469
1959
|
taskId,
|
|
1960
|
+
segmentId,
|
|
1961
|
+
repoId,
|
|
1962
|
+
segmentFrontier,
|
|
1470
1963
|
laneId: laneForTask?.laneId,
|
|
1471
1964
|
laneNumber: laneForTask?.laneNumber,
|
|
1472
1965
|
waveIndex: waveIdx,
|
|
@@ -1519,7 +2012,7 @@ export async function executeOrchBatch(
|
|
|
1519
2012
|
}
|
|
1520
2013
|
|
|
1521
2014
|
// ── Wave Merge ───────────────────────────────────────────
|
|
1522
|
-
//
|
|
2015
|
+
// Merge when at least one segment execution succeeded in this wave.
|
|
1523
2016
|
let mergeResult: MergeWaveResult | null = null;
|
|
1524
2017
|
|
|
1525
2018
|
// Build lane outcome lookup and detect mixed-outcome lanes
|
|
@@ -1536,7 +2029,7 @@ export async function executeOrchBatch(
|
|
|
1536
2029
|
return hasSucceeded && hasHardFailure;
|
|
1537
2030
|
});
|
|
1538
2031
|
|
|
1539
|
-
if (
|
|
2032
|
+
if (succeededSegmentTaskIdsForMerge.length > 0) {
|
|
1540
2033
|
const mergeableLaneCount = waveResult.allocatedLanes.filter(lane => {
|
|
1541
2034
|
const outcome = laneOutcomeByNumber.get(lane.laneNumber);
|
|
1542
2035
|
if (!outcome) return false;
|
|
@@ -1753,6 +2246,7 @@ export async function executeOrchBatch(
|
|
|
1753
2246
|
|
|
1754
2247
|
// ── TP-076: Emit supervisor alert for rollback safe-stop ──
|
|
1755
2248
|
const rollbackError = `Safe-stop at wave ${waveIdx + 1}: verification rollback failed.${persistWarning}`;
|
|
2249
|
+
const rollbackRepoId = extractFailedRepoId(mergeResult) ?? undefined;
|
|
1756
2250
|
emitAlert({
|
|
1757
2251
|
category: "merge-failure",
|
|
1758
2252
|
summary:
|
|
@@ -1766,6 +2260,7 @@ export async function executeOrchBatch(
|
|
|
1766
2260
|
context: {
|
|
1767
2261
|
waveIndex: waveIdx,
|
|
1768
2262
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2263
|
+
repoId: rollbackRepoId,
|
|
1769
2264
|
mergeError: rollbackError,
|
|
1770
2265
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1771
2266
|
},
|
|
@@ -1874,6 +2369,7 @@ export async function executeOrchBatch(
|
|
|
1874
2369
|
context: {
|
|
1875
2370
|
waveIndex: waveIdx,
|
|
1876
2371
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2372
|
+
repoId: mergeRepoId ?? undefined,
|
|
1877
2373
|
mergeError: retryOutcome.errorMessage,
|
|
1878
2374
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1879
2375
|
},
|
|
@@ -1950,6 +2446,7 @@ export async function executeOrchBatch(
|
|
|
1950
2446
|
context: {
|
|
1951
2447
|
waveIndex: waveIdx,
|
|
1952
2448
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2449
|
+
repoId: mergeRepoId ?? undefined,
|
|
1953
2450
|
mergeError: exhaustionMsg,
|
|
1954
2451
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1955
2452
|
},
|
|
@@ -1987,6 +2484,7 @@ export async function executeOrchBatch(
|
|
|
1987
2484
|
context: {
|
|
1988
2485
|
waveIndex: waveIdx,
|
|
1989
2486
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2487
|
+
repoId: mergeRepoId ?? undefined,
|
|
1990
2488
|
mergeError: mergeResult.failureReason || "unknown",
|
|
1991
2489
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1992
2490
|
},
|