taskplane 0.24.7 → 0.24.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dashboard/public/app.js +105 -4
- package/dashboard/public/style.css +36 -0
- package/extensions/taskplane/agent-bridge-extension.ts +4 -4
- package/extensions/taskplane/engine.ts +526 -21
- package/extensions/taskplane/execution.ts +18 -9
- package/extensions/taskplane/extension.ts +158 -0
- package/extensions/taskplane/lane-runner.ts +58 -18
- package/extensions/taskplane/persistence.ts +12 -0
- package/extensions/taskplane/resume.ts +267 -24
- package/extensions/taskplane/supervisor-primer.md +10 -0
- package/extensions/taskplane/supervisor.ts +97 -0
- package/extensions/taskplane/types.ts +90 -0
- package/package.json +1 -1
|
@@ -34,12 +34,12 @@ function terminateAliveV2Agents(stateRoot: string, batchId: string, sessionName:
|
|
|
34
34
|
}
|
|
35
35
|
import { getCurrentBranch, runGit } from "./git.ts";
|
|
36
36
|
import { mergeWaveByRepo } from "./merge.ts";
|
|
37
|
-
import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolicy, formatRepoMergeSummary, ORCH_MESSAGES } from "./messages.ts";
|
|
37
|
+
import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolicy, extractFailedRepoId, formatRepoMergeSummary, ORCH_MESSAGES } from "./messages.ts";
|
|
38
38
|
import type { CleanupGateRepoFailure } from "./messages.ts";
|
|
39
39
|
import { resolveOperatorId } from "./naming.ts";
|
|
40
40
|
import { applyPartialProgressToOutcomes, deleteBatchState, hasTaskDoneMarker, loadBatchState, persistRuntimeState, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
41
|
-
import { buildBatchProgressSnapshot, defaultResilienceState, StateFileError } from "./types.ts";
|
|
42
|
-
import type { AllocatedLane, AllocatedTask, LaneExecutionResult, LaneTaskOutcome, LaneTaskStatus, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, ParsedTask, PersistedBatchState, PersistedLaneRecord, ReconciledTaskState, ResumeEligibility, ResumePoint, TaskRunnerConfig, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
41
|
+
import { buildBatchProgressSnapshot, buildSupervisorSegmentFrontierSnapshot, defaultResilienceState, StateFileError } from "./types.ts";
|
|
42
|
+
import type { AllocatedLane, AllocatedTask, LaneExecutionResult, LaneTaskOutcome, LaneTaskStatus, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, ParsedTask, PersistedBatchState, PersistedLaneRecord, PersistedSegmentRecord, ReconciledTaskState, ResumeEligibility, ResumePoint, TaskRunnerConfig, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
43
43
|
import { buildDependencyGraph, resolveBaseBranch, resolveRepoRoot } from "./waves.ts";
|
|
44
44
|
import { deleteBranchBestEffort, forceCleanupWorktree, listWorktrees, preserveFailedLaneProgress, removeAllWorktrees, removeWorktree, safeResetWorktree, sleepSync } from "./worktree.ts";
|
|
45
45
|
|
|
@@ -161,6 +161,18 @@ export function reconstructAllocatedLanes(
|
|
|
161
161
|
if (persistedTask?.taskFolder) {
|
|
162
162
|
taskStub.taskFolder = persistedTask.taskFolder;
|
|
163
163
|
}
|
|
164
|
+
if ((persistedTask as any)?.packetRepoId !== undefined) {
|
|
165
|
+
(taskStub as any).packetRepoId = (persistedTask as any).packetRepoId;
|
|
166
|
+
}
|
|
167
|
+
if ((persistedTask as any)?.packetTaskPath !== undefined) {
|
|
168
|
+
(taskStub as any).packetTaskPath = (persistedTask as any).packetTaskPath;
|
|
169
|
+
}
|
|
170
|
+
if ((persistedTask as any)?.segmentIds !== undefined) {
|
|
171
|
+
(taskStub as any).segmentIds = (persistedTask as any).segmentIds;
|
|
172
|
+
}
|
|
173
|
+
if ((persistedTask as any)?.activeSegmentId !== undefined) {
|
|
174
|
+
(taskStub as any).activeSegmentId = (persistedTask as any).activeSegmentId;
|
|
175
|
+
}
|
|
164
176
|
return {
|
|
165
177
|
taskId,
|
|
166
178
|
order: 0,
|
|
@@ -211,6 +223,39 @@ export function collectAllRepoRoots(
|
|
|
211
223
|
|
|
212
224
|
// ── Resume Pure Functions ────────────────────────────────────────────
|
|
213
225
|
|
|
226
|
+
/**
|
|
227
|
+
* Collect task IDs with authoritative .DONE markers.
|
|
228
|
+
*
|
|
229
|
+
* Segment frontier state does not suppress .DONE authority. If a marker exists,
|
|
230
|
+
* resume reconciliation will mark the task complete regardless of segment state.
|
|
231
|
+
*/
|
|
232
|
+
export function collectDoneTaskIdsForResume(
|
|
233
|
+
persistedState: PersistedBatchState,
|
|
234
|
+
repoRoot: string,
|
|
235
|
+
workspaceConfig?: WorkspaceConfig | null,
|
|
236
|
+
): Set<string> {
|
|
237
|
+
const doneTaskIds = new Set<string>();
|
|
238
|
+
for (const task of persistedState.tasks) {
|
|
239
|
+
if (task.taskFolder && hasTaskDoneMarker(task.taskFolder)) {
|
|
240
|
+
doneTaskIds.add(task.taskId);
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
const laneRec = persistedState.lanes.find(l => l.taskIds.includes(task.taskId));
|
|
244
|
+
if (laneRec?.worktreePath && task.taskFolder) {
|
|
245
|
+
const resolved = resolveCanonicalTaskPaths(
|
|
246
|
+
task.taskFolder,
|
|
247
|
+
laneRec.worktreePath,
|
|
248
|
+
repoRoot,
|
|
249
|
+
!!workspaceConfig,
|
|
250
|
+
);
|
|
251
|
+
if (existsSync(resolved.donePath)) {
|
|
252
|
+
doneTaskIds.add(task.taskId);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
return doneTaskIds;
|
|
257
|
+
}
|
|
258
|
+
|
|
214
259
|
/**
|
|
215
260
|
* Check whether a persisted batch state is eligible for resume.
|
|
216
261
|
*
|
|
@@ -333,6 +378,126 @@ export function checkResumeEligibility(state: PersistedBatchState, force: boolea
|
|
|
333
378
|
}
|
|
334
379
|
}
|
|
335
380
|
|
|
381
|
+
interface SegmentFrontierResumeTaskState {
|
|
382
|
+
taskId: string;
|
|
383
|
+
completedSegmentIds: string[];
|
|
384
|
+
inFlightSegmentIds: string[];
|
|
385
|
+
pendingSegmentIds: string[];
|
|
386
|
+
failedSegmentIds: string[];
|
|
387
|
+
nextSegmentId: string | null;
|
|
388
|
+
allSucceeded: boolean;
|
|
389
|
+
dependencyBySegmentId: Map<string, string[]>;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
function classifySegmentStatus(status: PersistedSegmentRecord["status"] | undefined): "completed" | "failed" | "in-flight" | "pending" {
|
|
393
|
+
if (status === "succeeded" || status === "skipped") return "completed";
|
|
394
|
+
if (status === "failed" || status === "stalled") return "failed";
|
|
395
|
+
if (status === "running") return "in-flight";
|
|
396
|
+
return "pending";
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Reconstruct per-task segment frontier from persisted segment records.
|
|
401
|
+
*
|
|
402
|
+
* Mutates persisted task records in-place to reflect the segment frontier:
|
|
403
|
+
* - sets `activeSegmentId` to running or next pending segment
|
|
404
|
+
* - normalizes task `status` to pending/running/terminal based on segments
|
|
405
|
+
*/
|
|
406
|
+
export function reconstructSegmentFrontier(
|
|
407
|
+
persistedState: PersistedBatchState,
|
|
408
|
+
): Map<string, SegmentFrontierResumeTaskState> {
|
|
409
|
+
const byTask = new Map<string, SegmentFrontierResumeTaskState>();
|
|
410
|
+
const segmentRecordById = new Map<string, PersistedSegmentRecord>();
|
|
411
|
+
for (const segment of persistedState.segments ?? []) {
|
|
412
|
+
segmentRecordById.set(segment.segmentId, segment);
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
for (const task of persistedState.tasks) {
|
|
416
|
+
const segmentIds = task.segmentIds ?? [];
|
|
417
|
+
if (segmentIds.length === 0) continue;
|
|
418
|
+
|
|
419
|
+
const dependencyBySegmentId = new Map<string, string[]>();
|
|
420
|
+
const completedSegmentIds: string[] = [];
|
|
421
|
+
const inFlightSegmentIds: string[] = [];
|
|
422
|
+
const pendingSegmentIds: string[] = [];
|
|
423
|
+
const failedSegmentIds: string[] = [];
|
|
424
|
+
let hasConcreteSegmentRecord = false;
|
|
425
|
+
|
|
426
|
+
for (let idx = 0; idx < segmentIds.length; idx++) {
|
|
427
|
+
const segmentId = segmentIds[idx];
|
|
428
|
+
const record = segmentRecordById.get(segmentId);
|
|
429
|
+
if (record) hasConcreteSegmentRecord = true;
|
|
430
|
+
const recordDeps = record?.dependsOnSegmentIds ?? [];
|
|
431
|
+
const fallbackDeps = idx > 0 ? [segmentIds[idx - 1]] : [];
|
|
432
|
+
const deps = (recordDeps.length > 0 ? recordDeps : fallbackDeps)
|
|
433
|
+
.filter(dep => segmentIds.includes(dep));
|
|
434
|
+
dependencyBySegmentId.set(segmentId, [...new Set(deps)].sort((a, b) => a.localeCompare(b)));
|
|
435
|
+
|
|
436
|
+
switch (classifySegmentStatus(record?.status)) {
|
|
437
|
+
case "completed":
|
|
438
|
+
completedSegmentIds.push(segmentId);
|
|
439
|
+
break;
|
|
440
|
+
case "in-flight":
|
|
441
|
+
inFlightSegmentIds.push(segmentId);
|
|
442
|
+
break;
|
|
443
|
+
case "failed":
|
|
444
|
+
failedSegmentIds.push(segmentId);
|
|
445
|
+
break;
|
|
446
|
+
default:
|
|
447
|
+
pendingSegmentIds.push(segmentId);
|
|
448
|
+
break;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
const completedSet = new Set(completedSegmentIds);
|
|
453
|
+
const readyPending = pendingSegmentIds.filter((segmentId) => {
|
|
454
|
+
const deps = dependencyBySegmentId.get(segmentId) ?? [];
|
|
455
|
+
return deps.every(dep => completedSet.has(dep));
|
|
456
|
+
});
|
|
457
|
+
|
|
458
|
+
const nextSegmentId = inFlightSegmentIds[0]
|
|
459
|
+
?? readyPending[0]
|
|
460
|
+
?? pendingSegmentIds[0]
|
|
461
|
+
?? null;
|
|
462
|
+
const allSucceeded = segmentIds.every((segmentId) => {
|
|
463
|
+
const status = segmentRecordById.get(segmentId)?.status;
|
|
464
|
+
return status === "succeeded";
|
|
465
|
+
});
|
|
466
|
+
|
|
467
|
+
if (hasConcreteSegmentRecord) {
|
|
468
|
+
if (failedSegmentIds.length > 0) {
|
|
469
|
+
task.status = task.status === "skipped" ? "skipped" : "failed";
|
|
470
|
+
task.activeSegmentId = null;
|
|
471
|
+
} else if (inFlightSegmentIds.length > 0) {
|
|
472
|
+
task.status = "running";
|
|
473
|
+
task.activeSegmentId = inFlightSegmentIds[0];
|
|
474
|
+
} else if (pendingSegmentIds.length > 0) {
|
|
475
|
+
task.status = "pending";
|
|
476
|
+
task.activeSegmentId = nextSegmentId;
|
|
477
|
+
} else if (allSucceeded) {
|
|
478
|
+
task.status = "succeeded";
|
|
479
|
+
task.activeSegmentId = null;
|
|
480
|
+
} else {
|
|
481
|
+
task.status = task.status === "skipped" ? "skipped" : "failed";
|
|
482
|
+
task.activeSegmentId = null;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
byTask.set(task.taskId, {
|
|
487
|
+
taskId: task.taskId,
|
|
488
|
+
completedSegmentIds,
|
|
489
|
+
inFlightSegmentIds,
|
|
490
|
+
pendingSegmentIds,
|
|
491
|
+
failedSegmentIds,
|
|
492
|
+
nextSegmentId,
|
|
493
|
+
allSucceeded,
|
|
494
|
+
dependencyBySegmentId,
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
return byTask;
|
|
499
|
+
}
|
|
500
|
+
|
|
336
501
|
/**
|
|
337
502
|
* Reconcile persisted task states against live signals.
|
|
338
503
|
*
|
|
@@ -504,6 +669,33 @@ export function computeResumePoint(
|
|
|
504
669
|
reconciledMap.set(task.taskId, task);
|
|
505
670
|
}
|
|
506
671
|
|
|
672
|
+
const segmentStatusBySegmentId = new Map<string, PersistedSegmentRecord["status"]>();
|
|
673
|
+
for (const segment of persistedState.segments ?? []) {
|
|
674
|
+
segmentStatusBySegmentId.set(segment.segmentId, segment.status);
|
|
675
|
+
}
|
|
676
|
+
const persistedTasks = Array.isArray((persistedState as { tasks?: unknown }).tasks)
|
|
677
|
+
? persistedState.tasks
|
|
678
|
+
: [];
|
|
679
|
+
const segmentIdsByTaskId = new Map<string, string[]>();
|
|
680
|
+
for (const task of persistedTasks) {
|
|
681
|
+
if (task.segmentIds && task.segmentIds.length > 0) {
|
|
682
|
+
segmentIdsByTaskId.set(task.taskId, task.segmentIds);
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
const waveSegmentIdByTaskOccurrence = new Map<string, string>();
|
|
686
|
+
const occurrenceByTaskId = new Map<string, number>();
|
|
687
|
+
for (let waveIdx = 0; waveIdx < persistedState.wavePlan.length; waveIdx++) {
|
|
688
|
+
for (const taskId of persistedState.wavePlan[waveIdx]) {
|
|
689
|
+
const segmentIds = segmentIdsByTaskId.get(taskId);
|
|
690
|
+
if (!segmentIds || segmentIds.length === 0) continue;
|
|
691
|
+
const occurrence = occurrenceByTaskId.get(taskId) ?? 0;
|
|
692
|
+
if (occurrence < segmentIds.length) {
|
|
693
|
+
waveSegmentIdByTaskOccurrence.set(`${waveIdx}:${taskId}`, segmentIds[occurrence]);
|
|
694
|
+
}
|
|
695
|
+
occurrenceByTaskId.set(taskId, occurrence + 1);
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
|
|
507
699
|
// Categorize tasks
|
|
508
700
|
const completedTaskIds: string[] = [];
|
|
509
701
|
const pendingTaskIds: string[] = [];
|
|
@@ -551,6 +743,14 @@ export function computeResumePoint(
|
|
|
551
743
|
for (let i = 0; i < persistedState.wavePlan.length; i++) {
|
|
552
744
|
const waveTasks = persistedState.wavePlan[i];
|
|
553
745
|
const allDone = waveTasks.every((taskId) => {
|
|
746
|
+
const waveSegmentId = waveSegmentIdByTaskOccurrence.get(`${i}:${taskId}`);
|
|
747
|
+
if (waveSegmentId && segmentStatusBySegmentId.has(waveSegmentId)) {
|
|
748
|
+
const segmentStatus = segmentStatusBySegmentId.get(waveSegmentId)!;
|
|
749
|
+
return segmentStatus === "succeeded"
|
|
750
|
+
|| segmentStatus === "failed"
|
|
751
|
+
|| segmentStatus === "stalled"
|
|
752
|
+
|| segmentStatus === "skipped";
|
|
753
|
+
}
|
|
554
754
|
const reconciled = reconciledMap.get(taskId);
|
|
555
755
|
if (!reconciled) return false;
|
|
556
756
|
// A task is "done" for wave-skip purposes if it's terminal:
|
|
@@ -579,6 +779,10 @@ export function computeResumePoint(
|
|
|
579
779
|
// Only check merge status if the wave had any succeeded tasks (waves with
|
|
580
780
|
// only failures/skips don't produce merges and can be safely skipped).
|
|
581
781
|
const hasSucceededTasks = waveTasks.some((taskId) => {
|
|
782
|
+
const waveSegmentId = waveSegmentIdByTaskOccurrence.get(`${i}:${taskId}`);
|
|
783
|
+
if (waveSegmentId && segmentStatusBySegmentId.has(waveSegmentId)) {
|
|
784
|
+
return segmentStatusBySegmentId.get(waveSegmentId) === "succeeded";
|
|
785
|
+
}
|
|
582
786
|
const reconciled = reconciledMap.get(taskId);
|
|
583
787
|
if (!reconciled) return false;
|
|
584
788
|
if (reconciled.action === "mark-complete") return true;
|
|
@@ -603,6 +807,15 @@ export function computeResumePoint(
|
|
|
603
807
|
const actualPendingTaskIds: string[] = [];
|
|
604
808
|
for (let i = resumeWaveIndex; i < persistedState.wavePlan.length; i++) {
|
|
605
809
|
for (const taskId of persistedState.wavePlan[i]) {
|
|
810
|
+
const waveSegmentId = waveSegmentIdByTaskOccurrence.get(`${i}:${taskId}`);
|
|
811
|
+
if (waveSegmentId && segmentStatusBySegmentId.has(waveSegmentId)) {
|
|
812
|
+
const segmentStatus = segmentStatusBySegmentId.get(waveSegmentId)!;
|
|
813
|
+
if (segmentStatus === "running" || segmentStatus === "pending") {
|
|
814
|
+
actualPendingTaskIds.push(taskId);
|
|
815
|
+
}
|
|
816
|
+
continue;
|
|
817
|
+
}
|
|
818
|
+
|
|
606
819
|
const reconciled = reconciledMap.get(taskId);
|
|
607
820
|
if (!reconciled) {
|
|
608
821
|
actualPendingTaskIds.push(taskId); // Unknown task — treat as pending
|
|
@@ -872,6 +1085,24 @@ export async function resumeOrchBatch(
|
|
|
872
1085
|
"info",
|
|
873
1086
|
);
|
|
874
1087
|
|
|
1088
|
+
const segmentFrontierByTask = reconstructSegmentFrontier(persistedState);
|
|
1089
|
+
if (segmentFrontierByTask.size > 0) {
|
|
1090
|
+
let completedSegments = 0;
|
|
1091
|
+
let inFlightSegments = 0;
|
|
1092
|
+
let pendingSegments = 0;
|
|
1093
|
+
for (const frontier of segmentFrontierByTask.values()) {
|
|
1094
|
+
completedSegments += frontier.completedSegmentIds.length;
|
|
1095
|
+
inFlightSegments += frontier.inFlightSegmentIds.length;
|
|
1096
|
+
pendingSegments += frontier.pendingSegmentIds.length;
|
|
1097
|
+
}
|
|
1098
|
+
execLog("resume", persistedState.batchId, `segment frontier reconstructed`, {
|
|
1099
|
+
tasks: segmentFrontierByTask.size,
|
|
1100
|
+
completedSegments,
|
|
1101
|
+
inFlightSegments,
|
|
1102
|
+
pendingSegments,
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1105
|
+
|
|
875
1106
|
// TP-108/112: Runtime V2 backend selection for resumed batches.
|
|
876
1107
|
// MUST be computed before any backend-aware branch (section 3+).
|
|
877
1108
|
const resumeBackend: RuntimeBackend = selectRuntimeBackend(
|
|
@@ -903,27 +1134,7 @@ export async function resumeOrchBatch(
|
|
|
903
1134
|
// TP-109: In workspace mode or V2 execution, .DONE is written in the worktree
|
|
904
1135
|
// at the resolved packet path, not the original discovery path. Resume must
|
|
905
1136
|
// check both locations for authoritative completion detection.
|
|
906
|
-
const doneTaskIds =
|
|
907
|
-
for (const task of persistedState.tasks) {
|
|
908
|
-
// Check original task folder path
|
|
909
|
-
if (task.taskFolder && hasTaskDoneMarker(task.taskFolder)) {
|
|
910
|
-
doneTaskIds.add(task.taskId);
|
|
911
|
-
continue;
|
|
912
|
-
}
|
|
913
|
-
// Check worktree-relative path (packet-home authority)
|
|
914
|
-
const laneRec = persistedState.lanes.find(l => l.taskIds.includes(task.taskId));
|
|
915
|
-
if (laneRec?.worktreePath && task.taskFolder) {
|
|
916
|
-
const resolved = resolveCanonicalTaskPaths(
|
|
917
|
-
task.taskFolder,
|
|
918
|
-
laneRec.worktreePath,
|
|
919
|
-
repoRoot,
|
|
920
|
-
!!workspaceConfig,
|
|
921
|
-
);
|
|
922
|
-
if (existsSync(resolved.donePath)) {
|
|
923
|
-
doneTaskIds.add(task.taskId);
|
|
924
|
-
}
|
|
925
|
-
}
|
|
926
|
-
}
|
|
1137
|
+
const doneTaskIds = collectDoneTaskIdsForResume(persistedState, repoRoot, workspaceConfig);
|
|
927
1138
|
|
|
928
1139
|
// ── 3b. Detect existing worktrees ────────────────────────────
|
|
929
1140
|
const existingWorktreeTaskIds = new Set<string>();
|
|
@@ -1710,13 +1921,36 @@ export async function resumeOrchBatch(
|
|
|
1710
1921
|
for (const taskId of waveResult.failedTaskIds) {
|
|
1711
1922
|
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
1712
1923
|
const laneForTask = latestAllocatedLanes.find(l => l.tasks.some(t => t.taskId === taskId));
|
|
1924
|
+
const taskRecord = batchState.tasks.find((task) => task.taskId === taskId);
|
|
1713
1925
|
const exitReason = outcome?.exitReason || "unknown";
|
|
1714
1926
|
const hasPartialProgress = (outcome?.partialProgressCommits ?? 0) > 0;
|
|
1927
|
+
const segmentFrontier = buildSupervisorSegmentFrontierSnapshot(
|
|
1928
|
+
taskId,
|
|
1929
|
+
taskRecord?.segmentIds,
|
|
1930
|
+
taskRecord?.activeSegmentId,
|
|
1931
|
+
batchState.segments,
|
|
1932
|
+
outcome?.segmentId,
|
|
1933
|
+
);
|
|
1934
|
+
const segmentId = outcome?.segmentId
|
|
1935
|
+
?? taskRecord?.activeSegmentId
|
|
1936
|
+
?? segmentFrontier?.activeSegmentId
|
|
1937
|
+
?? undefined;
|
|
1938
|
+
const repoId = segmentId
|
|
1939
|
+
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ?? laneForTask?.repoId)
|
|
1940
|
+
: laneForTask?.repoId;
|
|
1941
|
+
const segmentSummary = segmentId
|
|
1942
|
+
? ` Segment: ${segmentId}${repoId ? ` (repo: ${repoId})` : ""}\n`
|
|
1943
|
+
: "";
|
|
1944
|
+
const frontierSummary = segmentFrontier
|
|
1945
|
+
? ` Segment frontier: ${segmentFrontier.terminalSegments}/${segmentFrontier.totalSegments} terminal\n`
|
|
1946
|
+
: "";
|
|
1715
1947
|
emitAlert({
|
|
1716
1948
|
category: "task-failure",
|
|
1717
1949
|
summary:
|
|
1718
1950
|
`⚠️ Task failure: ${taskId}\n` +
|
|
1719
1951
|
` Exit reason: ${exitReason}\n` +
|
|
1952
|
+
segmentSummary +
|
|
1953
|
+
frontierSummary +
|
|
1720
1954
|
` Lane: ${laneForTask?.laneId ?? "unknown"} (lane ${laneForTask?.laneNumber ?? "?"})\n` +
|
|
1721
1955
|
` Partial progress preserved: ${hasPartialProgress ? "yes" : "no"}\n` +
|
|
1722
1956
|
` Batch: wave ${waveIdx + 1}/${batchState.totalWaves}, ` +
|
|
@@ -1727,6 +1961,9 @@ export async function resumeOrchBatch(
|
|
|
1727
1961
|
` - Read STATUS.md and lane logs for diagnosis`,
|
|
1728
1962
|
context: {
|
|
1729
1963
|
taskId,
|
|
1964
|
+
segmentId,
|
|
1965
|
+
repoId,
|
|
1966
|
+
segmentFrontier,
|
|
1730
1967
|
laneId: laneForTask?.laneId,
|
|
1731
1968
|
laneNumber: laneForTask?.laneNumber,
|
|
1732
1969
|
waveIndex: waveIdx,
|
|
@@ -1928,6 +2165,7 @@ export async function resumeOrchBatch(
|
|
|
1928
2165
|
);
|
|
1929
2166
|
|
|
1930
2167
|
// ── TP-076: Emit supervisor alert for rollback safe-stop ──
|
|
2168
|
+
const rollbackRepoId = extractFailedRepoId(mergeResult) ?? undefined;
|
|
1931
2169
|
emitAlert({
|
|
1932
2170
|
category: "merge-failure",
|
|
1933
2171
|
summary:
|
|
@@ -1941,6 +2179,7 @@ export async function resumeOrchBatch(
|
|
|
1941
2179
|
context: {
|
|
1942
2180
|
waveIndex: waveIdx,
|
|
1943
2181
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2182
|
+
repoId: rollbackRepoId,
|
|
1944
2183
|
mergeError: `Safe-stop: verification rollback failed at wave ${waveIdx + 1}`,
|
|
1945
2184
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1946
2185
|
},
|
|
@@ -1958,6 +2197,7 @@ export async function resumeOrchBatch(
|
|
|
1958
2197
|
batchState.resilience = defaultResilienceState();
|
|
1959
2198
|
}
|
|
1960
2199
|
|
|
2200
|
+
const mergeRepoId = extractFailedRepoId(mergeResult) ?? undefined;
|
|
1961
2201
|
const retryOutcome = await applyMergeRetryLoop(
|
|
1962
2202
|
mergeResult,
|
|
1963
2203
|
waveIdx,
|
|
@@ -2017,6 +2257,7 @@ export async function resumeOrchBatch(
|
|
|
2017
2257
|
context: {
|
|
2018
2258
|
waveIndex: waveIdx,
|
|
2019
2259
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2260
|
+
repoId: mergeRepoId,
|
|
2020
2261
|
mergeError: retryOutcome.errorMessage,
|
|
2021
2262
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
2022
2263
|
},
|
|
@@ -2056,6 +2297,7 @@ export async function resumeOrchBatch(
|
|
|
2056
2297
|
context: {
|
|
2057
2298
|
waveIndex: waveIdx,
|
|
2058
2299
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2300
|
+
repoId: mergeRepoId,
|
|
2059
2301
|
mergeError: exhaustionMsg,
|
|
2060
2302
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
2061
2303
|
},
|
|
@@ -2092,6 +2334,7 @@ export async function resumeOrchBatch(
|
|
|
2092
2334
|
context: {
|
|
2093
2335
|
waveIndex: waveIdx,
|
|
2094
2336
|
laneNumber: mergeResult.failedLane ?? undefined,
|
|
2337
|
+
repoId: mergeRepoId,
|
|
2095
2338
|
mergeError: mergeResult.failureReason || "unknown",
|
|
2096
2339
|
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
2097
2340
|
},
|
|
@@ -724,6 +724,11 @@ Each alert contains:
|
|
|
724
724
|
are available. This is what you see in the conversation.
|
|
725
725
|
- **Context**: Structured data (taskId, laneId, waveIndex, exitReason,
|
|
726
726
|
batchProgress, etc.) embedded in the message for your reference.
|
|
727
|
+
- `task-failure` alerts include segment-aware fields when available:
|
|
728
|
+
`segmentId`, `repoId`, and `segmentFrontier`.
|
|
729
|
+
- `segmentFrontier` shows ordered segment status for that task
|
|
730
|
+
(`pending/running/succeeded/failed/skipped/stalled`) so you can quickly
|
|
731
|
+
tell whether the failure happened early or near completion.
|
|
727
732
|
|
|
728
733
|
### Response Protocol
|
|
729
734
|
|
|
@@ -799,6 +804,11 @@ you observe. Do not skip steps; each observation narrows the diagnosis.
|
|
|
799
804
|
**Trigger:** `task-failure` alert — a task failed after the engine exhausted
|
|
800
805
|
deterministic recovery (retries, context resets).
|
|
801
806
|
|
|
807
|
+
**Segment-aware triage:** If alert context includes `segmentId`/`repoId`, treat
|
|
808
|
+
that as the failing execution unit. Use `segmentFrontier` to decide whether to
|
|
809
|
+
retry immediately (early segment failure) or inspect downstream impact first
|
|
810
|
+
(late-segment failure after prior segments succeeded).
|
|
811
|
+
|
|
802
812
|
```
|
|
803
813
|
TASK FAILED: {taskId}
|
|
804
814
|
│
|
|
@@ -1121,6 +1121,27 @@ export interface BatchSummaryData {
|
|
|
1121
1121
|
failedLane: number | null;
|
|
1122
1122
|
failureReason: string | null;
|
|
1123
1123
|
}>;
|
|
1124
|
+
/** Segment-level outcomes (when segment tracking is available). */
|
|
1125
|
+
segmentOutcomes: {
|
|
1126
|
+
totalSegments: number;
|
|
1127
|
+
succeeded: number;
|
|
1128
|
+
failed: number;
|
|
1129
|
+
stalled: number;
|
|
1130
|
+
skipped: number;
|
|
1131
|
+
running: number;
|
|
1132
|
+
pending: number;
|
|
1133
|
+
multiSegmentTasks: Array<{
|
|
1134
|
+
taskId: string;
|
|
1135
|
+
totalSegments: number;
|
|
1136
|
+
terminalSegments: number;
|
|
1137
|
+
succeeded: number;
|
|
1138
|
+
failed: number;
|
|
1139
|
+
stalled: number;
|
|
1140
|
+
skipped: number;
|
|
1141
|
+
running: number;
|
|
1142
|
+
pending: number;
|
|
1143
|
+
}>;
|
|
1144
|
+
} | null;
|
|
1124
1145
|
/** Audit trail entries for the batch */
|
|
1125
1146
|
auditEntries: AuditTrailEntry[];
|
|
1126
1147
|
/** Tier 0 events from events.jsonl (recovery attempts, successes, exhausted, escalations) */
|
|
@@ -1311,6 +1332,51 @@ export function collectBatchSummaryData(
|
|
|
1311
1332
|
overallStatus: wr.overallStatus || "unknown",
|
|
1312
1333
|
}));
|
|
1313
1334
|
|
|
1335
|
+
const segmentRecords = batchState.segments || [];
|
|
1336
|
+
let segmentOutcomes: BatchSummaryData["segmentOutcomes"] = null;
|
|
1337
|
+
if (segmentRecords.length > 0) {
|
|
1338
|
+
const byTaskId = new Map<string, typeof segmentRecords>();
|
|
1339
|
+
for (const segment of segmentRecords) {
|
|
1340
|
+
const existing = byTaskId.get(segment.taskId) || [];
|
|
1341
|
+
existing.push(segment);
|
|
1342
|
+
byTaskId.set(segment.taskId, existing);
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
const multiSegmentTasks: NonNullable<BatchSummaryData["segmentOutcomes"]>["multiSegmentTasks"] = [];
|
|
1346
|
+
for (const [taskId, taskSegments] of [...byTaskId.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
|
|
1347
|
+
if (taskSegments.length <= 1) continue;
|
|
1348
|
+
const succeeded = taskSegments.filter((segment) => segment.status === "succeeded").length;
|
|
1349
|
+
const failed = taskSegments.filter((segment) => segment.status === "failed").length;
|
|
1350
|
+
const stalled = taskSegments.filter((segment) => segment.status === "stalled").length;
|
|
1351
|
+
const skipped = taskSegments.filter((segment) => segment.status === "skipped").length;
|
|
1352
|
+
const running = taskSegments.filter((segment) => segment.status === "running").length;
|
|
1353
|
+
const pending = taskSegments.filter((segment) => segment.status === "pending").length;
|
|
1354
|
+
const terminalSegments = succeeded + failed + stalled + skipped;
|
|
1355
|
+
multiSegmentTasks.push({
|
|
1356
|
+
taskId,
|
|
1357
|
+
totalSegments: taskSegments.length,
|
|
1358
|
+
terminalSegments,
|
|
1359
|
+
succeeded,
|
|
1360
|
+
failed,
|
|
1361
|
+
stalled,
|
|
1362
|
+
skipped,
|
|
1363
|
+
running,
|
|
1364
|
+
pending,
|
|
1365
|
+
});
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
segmentOutcomes = {
|
|
1369
|
+
totalSegments: segmentRecords.length,
|
|
1370
|
+
succeeded: segmentRecords.filter((segment) => segment.status === "succeeded").length,
|
|
1371
|
+
failed: segmentRecords.filter((segment) => segment.status === "failed").length,
|
|
1372
|
+
stalled: segmentRecords.filter((segment) => segment.status === "stalled").length,
|
|
1373
|
+
skipped: segmentRecords.filter((segment) => segment.status === "skipped").length,
|
|
1374
|
+
running: segmentRecords.filter((segment) => segment.status === "running").length,
|
|
1375
|
+
pending: segmentRecords.filter((segment) => segment.status === "pending").length,
|
|
1376
|
+
multiSegmentTasks,
|
|
1377
|
+
};
|
|
1378
|
+
}
|
|
1379
|
+
|
|
1314
1380
|
return {
|
|
1315
1381
|
batchId: batchState.batchId,
|
|
1316
1382
|
phase: batchState.phase,
|
|
@@ -1328,6 +1394,7 @@ export function collectBatchSummaryData(
|
|
|
1328
1394
|
waveResults,
|
|
1329
1395
|
taskExits: diagnostics?.taskExits ?? {},
|
|
1330
1396
|
mergeResults: mergeResults ?? [],
|
|
1397
|
+
segmentOutcomes,
|
|
1331
1398
|
auditEntries,
|
|
1332
1399
|
tier0Events,
|
|
1333
1400
|
errors: batchState.errors || [],
|
|
@@ -1421,6 +1488,36 @@ export function formatBatchSummary(data: BatchSummaryData): string {
|
|
|
1421
1488
|
}
|
|
1422
1489
|
lines.push("");
|
|
1423
1490
|
|
|
1491
|
+
// ── Segment Outcomes ─────────────────────────────────────────
|
|
1492
|
+
lines.push("## Segment Outcomes");
|
|
1493
|
+
lines.push("");
|
|
1494
|
+
if (!data.segmentOutcomes) {
|
|
1495
|
+
lines.push("Segment data not available.");
|
|
1496
|
+
} else if (data.segmentOutcomes.multiSegmentTasks.length === 0) {
|
|
1497
|
+
lines.push(`No multi-segment task outcomes recorded (${data.segmentOutcomes.totalSegments} segment record(s) total).`);
|
|
1498
|
+
} else {
|
|
1499
|
+
const statusParts = [
|
|
1500
|
+
`${data.segmentOutcomes.succeeded} succeeded`,
|
|
1501
|
+
`${data.segmentOutcomes.failed} failed`,
|
|
1502
|
+
];
|
|
1503
|
+
if (data.segmentOutcomes.running > 0) statusParts.push(`${data.segmentOutcomes.running} running`);
|
|
1504
|
+
if (data.segmentOutcomes.pending > 0) statusParts.push(`${data.segmentOutcomes.pending} pending`);
|
|
1505
|
+
if (data.segmentOutcomes.skipped > 0) statusParts.push(`${data.segmentOutcomes.skipped} skipped`);
|
|
1506
|
+
if (data.segmentOutcomes.stalled > 0) statusParts.push(`${data.segmentOutcomes.stalled} stalled`);
|
|
1507
|
+
lines.push(`- **Tracked segments:** ${data.segmentOutcomes.totalSegments}`);
|
|
1508
|
+
lines.push(`- **Status mix:** ${statusParts.join(", ")}`);
|
|
1509
|
+
lines.push(`- **Multi-segment tasks:** ${data.segmentOutcomes.multiSegmentTasks.length}`);
|
|
1510
|
+
for (const task of data.segmentOutcomes.multiSegmentTasks) {
|
|
1511
|
+
const taskParts = [`${task.succeeded}✓`, `${task.failed}✗`];
|
|
1512
|
+
if (task.running > 0) taskParts.push(`${task.running} running`);
|
|
1513
|
+
if (task.pending > 0) taskParts.push(`${task.pending} pending`);
|
|
1514
|
+
if (task.skipped > 0) taskParts.push(`${task.skipped} skipped`);
|
|
1515
|
+
if (task.stalled > 0) taskParts.push(`${task.stalled} stalled`);
|
|
1516
|
+
lines.push(` - ${task.taskId}: ${task.terminalSegments}/${task.totalSegments} terminal (${taskParts.join(", ")})`);
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
lines.push("");
|
|
1520
|
+
|
|
1424
1521
|
// ── Incidents & Recoveries ───────────────────────────────────
|
|
1425
1522
|
lines.push("## Incidents");
|
|
1426
1523
|
lines.push("");
|
|
@@ -684,6 +684,8 @@ export interface LaneTaskOutcome {
|
|
|
684
684
|
taskId: string;
|
|
685
685
|
/** Final task status */
|
|
686
686
|
status: LaneTaskStatus;
|
|
687
|
+
/** Segment identifier for segment-aware execution (null for whole-task units). */
|
|
688
|
+
segmentId?: string | null;
|
|
687
689
|
/** When execution started (epoch ms), null if never started (skipped) */
|
|
688
690
|
startTime: number | null;
|
|
689
691
|
/** When execution ended (epoch ms), null if still pending */
|
|
@@ -1949,9 +1951,31 @@ export type SupervisorAlertCategory = "task-failure" | "merge-failure" | "batch-
|
|
|
1949
1951
|
*
|
|
1950
1952
|
* @since TP-076
|
|
1951
1953
|
*/
|
|
1954
|
+
export interface SupervisorSegmentFrontierSnapshot {
|
|
1955
|
+
/** Parent task identifier */
|
|
1956
|
+
taskId: string;
|
|
1957
|
+
/** Total number of ordered segments for the task */
|
|
1958
|
+
totalSegments: number;
|
|
1959
|
+
/** Number of segments that reached a terminal status */
|
|
1960
|
+
terminalSegments: number;
|
|
1961
|
+
/** Active (or most recently active) segment ID */
|
|
1962
|
+
activeSegmentId: string | null;
|
|
1963
|
+
/** Segment-level execution snapshot in deterministic order */
|
|
1964
|
+
segments: Array<{
|
|
1965
|
+
segmentId: string;
|
|
1966
|
+
repoId: string;
|
|
1967
|
+
status: PersistedSegmentStatus;
|
|
1968
|
+
dependsOnSegmentIds: string[];
|
|
1969
|
+
}>;
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1952
1972
|
export interface SupervisorAlertContext {
|
|
1953
1973
|
/** Task ID (for task-failure alerts) */
|
|
1954
1974
|
taskId?: string;
|
|
1975
|
+
/** Segment ID (for segment-aware task-failure alerts) */
|
|
1976
|
+
segmentId?: string;
|
|
1977
|
+
/** Repo ID associated with the failure (task segment or merge target) */
|
|
1978
|
+
repoId?: string;
|
|
1955
1979
|
/** Lane ID, e.g., "lane-1" (for task-failure alerts) */
|
|
1956
1980
|
laneId?: string;
|
|
1957
1981
|
/** Lane number (for task-failure and merge-failure alerts) */
|
|
@@ -1960,6 +1984,8 @@ export interface SupervisorAlertContext {
|
|
|
1960
1984
|
waveIndex?: number;
|
|
1961
1985
|
/** Exit reason string (for task-failure alerts) */
|
|
1962
1986
|
exitReason?: string;
|
|
1987
|
+
/** Segment frontier snapshot for task-failure diagnosis */
|
|
1988
|
+
segmentFrontier?: SupervisorSegmentFrontierSnapshot;
|
|
1963
1989
|
/** Agent ID (for agent-message alerts) */
|
|
1964
1990
|
agentId?: string;
|
|
1965
1991
|
/** Mailbox message ID (for agent-message alerts) */
|
|
@@ -2041,6 +2067,70 @@ export function buildBatchProgressSnapshot(
|
|
|
2041
2067
|
};
|
|
2042
2068
|
}
|
|
2043
2069
|
|
|
2070
|
+
function repoIdFromSegmentId(segmentId: string): string {
|
|
2071
|
+
const idx = segmentId.indexOf("::");
|
|
2072
|
+
if (idx <= 0 || idx >= segmentId.length - 2) return "unknown";
|
|
2073
|
+
return segmentId.slice(idx + 2);
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
/**
|
|
2077
|
+
* Build a task-level segment frontier snapshot for supervisor failure alerts.
|
|
2078
|
+
*
|
|
2079
|
+
* Returns `undefined` when the task has no segment metadata.
|
|
2080
|
+
*/
|
|
2081
|
+
export function buildSupervisorSegmentFrontierSnapshot(
|
|
2082
|
+
taskId: string,
|
|
2083
|
+
segmentIds: string[] | undefined,
|
|
2084
|
+
activeSegmentId: string | null | undefined,
|
|
2085
|
+
persistedSegments: PersistedSegmentRecord[] | undefined,
|
|
2086
|
+
preferredSegmentId?: string | null,
|
|
2087
|
+
): SupervisorSegmentFrontierSnapshot | undefined {
|
|
2088
|
+
const orderedSegmentIds = Array.isArray(segmentIds)
|
|
2089
|
+
? segmentIds.filter((segmentId): segmentId is string => typeof segmentId === "string" && segmentId.trim().length > 0)
|
|
2090
|
+
: [];
|
|
2091
|
+
if (orderedSegmentIds.length === 0) return undefined;
|
|
2092
|
+
|
|
2093
|
+
const bySegmentId = new Map<string, PersistedSegmentRecord>();
|
|
2094
|
+
for (const segment of persistedSegments ?? []) {
|
|
2095
|
+
if (segment && segment.taskId === taskId) {
|
|
2096
|
+
bySegmentId.set(segment.segmentId, segment);
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
|
|
2100
|
+
const resolvedActiveSegmentId = (activeSegmentId && orderedSegmentIds.includes(activeSegmentId))
|
|
2101
|
+
? activeSegmentId
|
|
2102
|
+
: (preferredSegmentId && orderedSegmentIds.includes(preferredSegmentId)
|
|
2103
|
+
? preferredSegmentId
|
|
2104
|
+
: null);
|
|
2105
|
+
|
|
2106
|
+
const segments = orderedSegmentIds.map((segmentId) => {
|
|
2107
|
+
const persisted = bySegmentId.get(segmentId);
|
|
2108
|
+
const status: PersistedSegmentStatus = persisted?.status
|
|
2109
|
+
?? (resolvedActiveSegmentId === segmentId ? "running" : "pending");
|
|
2110
|
+
return {
|
|
2111
|
+
segmentId,
|
|
2112
|
+
repoId: persisted?.repoId ?? repoIdFromSegmentId(segmentId),
|
|
2113
|
+
status,
|
|
2114
|
+
dependsOnSegmentIds: persisted?.dependsOnSegmentIds ?? [],
|
|
2115
|
+
};
|
|
2116
|
+
});
|
|
2117
|
+
|
|
2118
|
+
const terminalSegments = segments.filter((segment) =>
|
|
2119
|
+
segment.status === "succeeded"
|
|
2120
|
+
|| segment.status === "failed"
|
|
2121
|
+
|| segment.status === "stalled"
|
|
2122
|
+
|| segment.status === "skipped",
|
|
2123
|
+
).length;
|
|
2124
|
+
|
|
2125
|
+
return {
|
|
2126
|
+
taskId,
|
|
2127
|
+
totalSegments: segments.length,
|
|
2128
|
+
terminalSegments,
|
|
2129
|
+
activeSegmentId: resolvedActiveSegmentId,
|
|
2130
|
+
segments,
|
|
2131
|
+
};
|
|
2132
|
+
}
|
|
2133
|
+
|
|
2044
2134
|
/**
|
|
2045
2135
|
* Build the base fields for an engine event.
|
|
2046
2136
|
*
|