taskplane 0.29.2 → 0.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/gitignore-patterns.mjs +11 -8
- package/bin/rpc-wrapper.mjs +410 -357
- package/bin/taskplane.mjs +533 -250
- package/dashboard/public/app.js +124 -15
- package/dashboard/public/style.css +83 -2
- package/extensions/reviewer-extension.ts +17 -11
- package/extensions/taskplane/abort.ts +50 -18
- package/extensions/taskplane/agent-bridge-extension.ts +232 -105
- package/extensions/taskplane/agent-host.ts +224 -97
- package/extensions/taskplane/cleanup.ts +71 -42
- package/extensions/taskplane/config-loader.ts +142 -58
- package/extensions/taskplane/config-schema.ts +6 -13
- package/extensions/taskplane/config.ts +10 -2
- package/extensions/taskplane/diagnostic-reports.ts +59 -47
- package/extensions/taskplane/diagnostics.ts +13 -13
- package/extensions/taskplane/discovery.ts +78 -63
- package/extensions/taskplane/engine-worker.ts +53 -46
- package/extensions/taskplane/engine.ts +1760 -602
- package/extensions/taskplane/execution.ts +469 -207
- package/extensions/taskplane/extension.ts +1073 -598
- package/extensions/taskplane/formatting.ts +136 -124
- package/extensions/taskplane/git.ts +0 -2
- package/extensions/taskplane/lane-runner.ts +652 -319
- package/extensions/taskplane/mailbox.ts +57 -49
- package/extensions/taskplane/merge.ts +662 -383
- package/extensions/taskplane/messages.ts +109 -51
- package/extensions/taskplane/migrations.ts +1 -1
- package/extensions/taskplane/path-resolver.ts +8 -9
- package/extensions/taskplane/persistence.ts +425 -262
- package/extensions/taskplane/process-registry.ts +36 -7
- package/extensions/taskplane/quality-gate.ts +107 -55
- package/extensions/taskplane/resume.ts +832 -280
- package/extensions/taskplane/sessions.ts +1 -1
- package/extensions/taskplane/settings-tui.ts +505 -164
- package/extensions/taskplane/sidecar-telemetry.ts +25 -10
- package/extensions/taskplane/supervisor.ts +477 -270
- package/extensions/taskplane/task-executor-core.ts +178 -53
- package/extensions/taskplane/types.ts +209 -108
- package/extensions/taskplane/verification.ts +27 -22
- package/extensions/taskplane/waves.ts +59 -43
- package/extensions/taskplane/workspace.ts +14 -12
- package/extensions/taskplane/worktree.ts +218 -196
- package/package.json +14 -2
|
@@ -7,8 +7,21 @@ import { join } from "path";
|
|
|
7
7
|
|
|
8
8
|
import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-reports.ts";
|
|
9
9
|
import { runDiscovery } from "./discovery.ts";
|
|
10
|
-
import {
|
|
11
|
-
|
|
10
|
+
import {
|
|
11
|
+
executeOrchBatch,
|
|
12
|
+
resolveDisplayWaveNumber,
|
|
13
|
+
buildSpawnFailureAlertExtras,
|
|
14
|
+
} from "./engine.ts";
|
|
15
|
+
import {
|
|
16
|
+
buildReviewerEnv,
|
|
17
|
+
buildWorkerEnv,
|
|
18
|
+
buildWorkerExcludeEnv,
|
|
19
|
+
computeTransitiveDependents,
|
|
20
|
+
execLog,
|
|
21
|
+
executeLaneV2,
|
|
22
|
+
executeWave,
|
|
23
|
+
resolveCanonicalTaskPaths,
|
|
24
|
+
} from "./execution.ts";
|
|
12
25
|
import type { MonitorUpdateCallback, RuntimeBackend } from "./execution.ts";
|
|
13
26
|
import { selectRuntimeBackend } from "./engine.ts";
|
|
14
27
|
import { readRegistrySnapshot, isTerminalStatus, isProcessAlive } from "./process-registry.ts";
|
|
@@ -28,20 +41,74 @@ function terminateAliveV2Agents(stateRoot: string, batchId: string, sessionName:
|
|
|
28
41
|
try {
|
|
29
42
|
process.kill(manifest.pid, "SIGTERM");
|
|
30
43
|
execLog("resume", key, `terminated alive V2 agent (PID ${manifest.pid}) before re-execute`);
|
|
31
|
-
} catch {
|
|
44
|
+
} catch {
|
|
45
|
+
/* already dead */
|
|
46
|
+
}
|
|
32
47
|
}
|
|
33
48
|
}
|
|
34
49
|
}
|
|
35
50
|
import { getCurrentBranch, runGit } from "./git.ts";
|
|
36
51
|
import { mergeWaveByRepo } from "./merge.ts";
|
|
37
|
-
import {
|
|
52
|
+
import {
|
|
53
|
+
applyMergeRetryLoop,
|
|
54
|
+
computeCleanupGatePolicy,
|
|
55
|
+
computeMergeFailurePolicy,
|
|
56
|
+
extractFailedRepoId,
|
|
57
|
+
formatRepoMergeSummary,
|
|
58
|
+
ORCH_MESSAGES,
|
|
59
|
+
} from "./messages.ts";
|
|
38
60
|
import type { CleanupGateRepoFailure } from "./messages.ts";
|
|
39
61
|
import { resolveOperatorId } from "./naming.ts";
|
|
40
|
-
import {
|
|
41
|
-
|
|
42
|
-
|
|
62
|
+
import {
|
|
63
|
+
applyPartialProgressToOutcomes,
|
|
64
|
+
deleteBatchState,
|
|
65
|
+
hasTaskDoneMarker,
|
|
66
|
+
loadBatchState,
|
|
67
|
+
persistRuntimeState,
|
|
68
|
+
reconstructBatchStateFromRuntime,
|
|
69
|
+
saveBatchState,
|
|
70
|
+
seedPendingOutcomesForAllocatedLanes,
|
|
71
|
+
syncTaskOutcomesFromMonitor,
|
|
72
|
+
upsertTaskOutcome,
|
|
73
|
+
} from "./persistence.ts";
|
|
74
|
+
import {
|
|
75
|
+
buildBatchProgressSnapshot,
|
|
76
|
+
buildSupervisorSegmentFrontierSnapshot,
|
|
77
|
+
defaultResilienceState,
|
|
78
|
+
StateFileError,
|
|
79
|
+
} from "./types.ts";
|
|
80
|
+
import type {
|
|
81
|
+
AllocatedLane,
|
|
82
|
+
AllocatedTask,
|
|
83
|
+
LaneExecutionResult,
|
|
84
|
+
LaneTaskOutcome,
|
|
85
|
+
LaneTaskStatus,
|
|
86
|
+
MergeWaveResult,
|
|
87
|
+
OrchBatchPhase,
|
|
88
|
+
OrchBatchRuntimeState,
|
|
89
|
+
OrchestratorConfig,
|
|
90
|
+
ParsedTask,
|
|
91
|
+
PersistedBatchState,
|
|
92
|
+
PersistedLaneRecord,
|
|
93
|
+
PersistedSegmentRecord,
|
|
94
|
+
ReconciledTaskState,
|
|
95
|
+
ResumeEligibility,
|
|
96
|
+
ResumePoint,
|
|
97
|
+
TaskRunnerConfig,
|
|
98
|
+
WaveExecutionResult,
|
|
99
|
+
WorkspaceConfig,
|
|
100
|
+
} from "./types.ts";
|
|
43
101
|
import { buildDependencyGraph, resolveBaseBranch, resolveRepoRoot } from "./waves.ts";
|
|
44
|
-
import {
|
|
102
|
+
import {
|
|
103
|
+
deleteBranchBestEffort,
|
|
104
|
+
forceCleanupWorktree,
|
|
105
|
+
listWorktrees,
|
|
106
|
+
preserveFailedLaneProgress,
|
|
107
|
+
removeAllWorktrees,
|
|
108
|
+
removeWorktree,
|
|
109
|
+
safeResetWorktree,
|
|
110
|
+
sleepSync,
|
|
111
|
+
} from "./worktree.ts";
|
|
45
112
|
|
|
46
113
|
// ── Resume Repo Helpers ──────────────────────────────────────────────
|
|
47
114
|
|
|
@@ -228,11 +295,40 @@ export function collectAllRepoRoots(
|
|
|
228
295
|
|
|
229
296
|
// ── Resume Pure Functions ────────────────────────────────────────────
|
|
230
297
|
|
|
298
|
+
/**
|
|
299
|
+
* Determine whether a multi-segment task's persisted segment frontier is
|
|
300
|
+
* complete — i.e., every segment for the task reached a terminal-success
|
|
301
|
+
* status ("succeeded" or "skipped").
|
|
302
|
+
*
|
|
303
|
+
* Returns:
|
|
304
|
+
* - `true` when the task has segments AND all of them are terminal-success.
|
|
305
|
+
* - `true` when the task has no segments recorded (single-segment / legacy
|
|
306
|
+
* tasks — the guard does not apply and `.DONE` is authoritative).
|
|
307
|
+
* - `false` when at least one segment is pending/running/failed/stalled.
|
|
308
|
+
*
|
|
309
|
+
* Used by `collectDoneTaskIdsForResume` (TP-196 / #462) to refuse a stale or
|
|
310
|
+
* premature `.DONE` from suppressing re-execution of remaining segments.
|
|
311
|
+
*/
|
|
312
|
+
function isSegmentFrontierCompleteForResume(
|
|
313
|
+
persistedState: PersistedBatchState,
|
|
314
|
+
taskId: string,
|
|
315
|
+
): boolean {
|
|
316
|
+
const segments = (persistedState.segments ?? []).filter((s) => s.taskId === taskId);
|
|
317
|
+
if (segments.length === 0) return true; // No segments recorded — guard does not apply.
|
|
318
|
+
return segments.every((s) => s.status === "succeeded" || s.status === "skipped");
|
|
319
|
+
}
|
|
320
|
+
|
|
231
321
|
/**
|
|
232
322
|
* Collect task IDs with authoritative .DONE markers.
|
|
233
323
|
*
|
|
234
|
-
* Segment frontier state does not suppress .DONE authority
|
|
235
|
-
*
|
|
324
|
+
* Segment frontier state does not suppress .DONE authority for tasks WITHOUT
|
|
325
|
+
* persisted segment records (single-segment / legacy). For tasks WITH segment
|
|
326
|
+
* records (multi-segment), TP-196 / #462 adds a resume guard: when `.DONE`
|
|
327
|
+
* exists but the segment frontier is incomplete (at least one segment is not
|
|
328
|
+
* yet succeeded/skipped), we DO NOT add the taskId to the done set — the
|
|
329
|
+
* task will be re-reconciled instead of silently marked complete. A WARN is
|
|
330
|
+
* logged so operators can spot the inconsistency. The on-disk `.DONE` marker
|
|
331
|
+
* is left alone; the engine will re-establish authoritative state.
|
|
236
332
|
*/
|
|
237
333
|
export function collectDoneTaskIdsForResume(
|
|
238
334
|
persistedState: PersistedBatchState,
|
|
@@ -241,22 +337,38 @@ export function collectDoneTaskIdsForResume(
|
|
|
241
337
|
): Set<string> {
|
|
242
338
|
const doneTaskIds = new Set<string>();
|
|
243
339
|
for (const task of persistedState.tasks) {
|
|
340
|
+
let markerFound = false;
|
|
341
|
+
let markerLocation: string | null = null;
|
|
244
342
|
if (task.taskFolder && hasTaskDoneMarker(task.taskFolder)) {
|
|
245
|
-
|
|
246
|
-
|
|
343
|
+
markerFound = true;
|
|
344
|
+
markerLocation = task.taskFolder;
|
|
345
|
+
}
|
|
346
|
+
if (!markerFound) {
|
|
347
|
+
const laneRec = persistedState.lanes.find((l) => l.taskIds.includes(task.taskId));
|
|
348
|
+
if (laneRec?.worktreePath && task.taskFolder) {
|
|
349
|
+
const resolved = resolveCanonicalTaskPaths(
|
|
350
|
+
task.taskFolder,
|
|
351
|
+
laneRec.worktreePath,
|
|
352
|
+
repoRoot,
|
|
353
|
+
!!workspaceConfig,
|
|
354
|
+
);
|
|
355
|
+
if (existsSync(resolved.donePath)) {
|
|
356
|
+
markerFound = true;
|
|
357
|
+
markerLocation = resolved.donePath;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
247
360
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
361
|
+
if (!markerFound) continue;
|
|
362
|
+
|
|
363
|
+
// TP-196 / #462: Resume guard — refuse `.DONE` authority for multi-segment
|
|
364
|
+
// tasks with an incomplete segment frontier.
|
|
365
|
+
if (!isSegmentFrontierCompleteForResume(persistedState, task.taskId)) {
|
|
366
|
+
console.warn(
|
|
367
|
+
`[resume] WARN: .DONE present for task ${task.taskId} at ${markerLocation} but segment frontier is incomplete — not marking complete (#462 guard). Task will re-reconcile.`,
|
|
255
368
|
);
|
|
256
|
-
|
|
257
|
-
doneTaskIds.add(task.taskId);
|
|
258
|
-
}
|
|
369
|
+
continue;
|
|
259
370
|
}
|
|
371
|
+
doneTaskIds.add(task.taskId);
|
|
260
372
|
}
|
|
261
373
|
return doneTaskIds;
|
|
262
374
|
}
|
|
@@ -281,7 +393,10 @@ export function collectDoneTaskIdsForResume(
|
|
|
281
393
|
* @param state - Persisted batch state to check
|
|
282
394
|
* @param force - When true, `stopped` and `failed` phases become eligible
|
|
283
395
|
*/
|
|
284
|
-
export function checkResumeEligibility(
|
|
396
|
+
export function checkResumeEligibility(
|
|
397
|
+
state: PersistedBatchState,
|
|
398
|
+
force: boolean = false,
|
|
399
|
+
): ResumeEligibility {
|
|
285
400
|
const { phase, batchId } = state;
|
|
286
401
|
|
|
287
402
|
switch (phase) {
|
|
@@ -394,7 +509,9 @@ interface SegmentFrontierResumeTaskState {
|
|
|
394
509
|
dependencyBySegmentId: Map<string, string[]>;
|
|
395
510
|
}
|
|
396
511
|
|
|
397
|
-
function classifySegmentStatus(
|
|
512
|
+
function classifySegmentStatus(
|
|
513
|
+
status: PersistedSegmentRecord["status"] | undefined,
|
|
514
|
+
): "completed" | "failed" | "in-flight" | "pending" {
|
|
398
515
|
if (status === "succeeded" || status === "skipped") return "completed";
|
|
399
516
|
if (status === "failed" || status === "stalled") return "failed";
|
|
400
517
|
if (status === "running") return "in-flight";
|
|
@@ -434,9 +551,13 @@ export function reconstructSegmentFrontier(
|
|
|
434
551
|
if (record) hasConcreteSegmentRecord = true;
|
|
435
552
|
const recordDeps = record?.dependsOnSegmentIds ?? [];
|
|
436
553
|
const fallbackDeps = idx > 0 ? [segmentIds[idx - 1]] : [];
|
|
437
|
-
const deps = (recordDeps.length > 0 ? recordDeps : fallbackDeps)
|
|
438
|
-
|
|
439
|
-
|
|
554
|
+
const deps = (recordDeps.length > 0 ? recordDeps : fallbackDeps).filter((dep) =>
|
|
555
|
+
segmentIds.includes(dep),
|
|
556
|
+
);
|
|
557
|
+
dependencyBySegmentId.set(
|
|
558
|
+
segmentId,
|
|
559
|
+
[...new Set(deps)].sort((a, b) => a.localeCompare(b)),
|
|
560
|
+
);
|
|
440
561
|
|
|
441
562
|
switch (classifySegmentStatus(record?.status)) {
|
|
442
563
|
case "completed":
|
|
@@ -457,13 +578,10 @@ export function reconstructSegmentFrontier(
|
|
|
457
578
|
const completedSet = new Set(completedSegmentIds);
|
|
458
579
|
const readyPending = pendingSegmentIds.filter((segmentId) => {
|
|
459
580
|
const deps = dependencyBySegmentId.get(segmentId) ?? [];
|
|
460
|
-
return deps.every(dep => completedSet.has(dep));
|
|
581
|
+
return deps.every((dep) => completedSet.has(dep));
|
|
461
582
|
});
|
|
462
583
|
|
|
463
|
-
const nextSegmentId = inFlightSegmentIds[0]
|
|
464
|
-
?? readyPending[0]
|
|
465
|
-
?? pendingSegmentIds[0]
|
|
466
|
-
?? null;
|
|
584
|
+
const nextSegmentId = inFlightSegmentIds[0] ?? readyPending[0] ?? pendingSegmentIds[0] ?? null;
|
|
467
585
|
const allSucceeded = segmentIds.every((segmentId) => {
|
|
468
586
|
const status = segmentRecordById.get(segmentId)?.status;
|
|
469
587
|
return status === "succeeded";
|
|
@@ -795,7 +913,12 @@ export function computeResumePoint(
|
|
|
795
913
|
case "skip":
|
|
796
914
|
if (task.liveStatus === "succeeded" || task.persistedStatus === "succeeded") {
|
|
797
915
|
completedTaskIds.push(task.taskId);
|
|
798
|
-
} else if (
|
|
916
|
+
} else if (
|
|
917
|
+
task.liveStatus === "failed" ||
|
|
918
|
+
task.liveStatus === "stalled" ||
|
|
919
|
+
task.persistedStatus === "failed" ||
|
|
920
|
+
task.persistedStatus === "stalled"
|
|
921
|
+
) {
|
|
799
922
|
failedTaskIds.push(task.taskId);
|
|
800
923
|
}
|
|
801
924
|
// persistedStatus === "skipped" → terminal but neither completed nor failed.
|
|
@@ -830,10 +953,12 @@ export function computeResumePoint(
|
|
|
830
953
|
const waveSegmentId = waveSegmentIdByTaskOccurrence.get(`${i}:${taskId}`);
|
|
831
954
|
if (waveSegmentId && segmentStatusBySegmentId.has(waveSegmentId)) {
|
|
832
955
|
const segmentStatus = segmentStatusBySegmentId.get(waveSegmentId)!;
|
|
833
|
-
return
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
956
|
+
return (
|
|
957
|
+
segmentStatus === "succeeded" ||
|
|
958
|
+
segmentStatus === "failed" ||
|
|
959
|
+
segmentStatus === "stalled" ||
|
|
960
|
+
segmentStatus === "skipped"
|
|
961
|
+
);
|
|
837
962
|
}
|
|
838
963
|
const reconciled = reconciledMap.get(taskId);
|
|
839
964
|
if (!reconciled) return false;
|
|
@@ -870,7 +995,11 @@ export function computeResumePoint(
|
|
|
870
995
|
const reconciled = reconciledMap.get(taskId);
|
|
871
996
|
if (!reconciled) return false;
|
|
872
997
|
if (reconciled.action === "mark-complete") return true;
|
|
873
|
-
if (
|
|
998
|
+
if (
|
|
999
|
+
reconciled.action === "skip" &&
|
|
1000
|
+
(reconciled.liveStatus === "succeeded" || reconciled.persistedStatus === "succeeded")
|
|
1001
|
+
)
|
|
1002
|
+
return true;
|
|
874
1003
|
return false;
|
|
875
1004
|
});
|
|
876
1005
|
|
|
@@ -935,7 +1064,6 @@ export function computeResumePoint(
|
|
|
935
1064
|
};
|
|
936
1065
|
}
|
|
937
1066
|
|
|
938
|
-
|
|
939
1067
|
// ── Pre-Resume Diagnostics ───────────────────────────────────────────
|
|
940
1068
|
|
|
941
1069
|
/**
|
|
@@ -1001,7 +1129,10 @@ export function runPreResumeDiagnostics(
|
|
|
1001
1129
|
const label = repoId ? `repo:${repoId}` : "default-repo";
|
|
1002
1130
|
|
|
1003
1131
|
if (persistedState.orchBranch) {
|
|
1004
|
-
const branchCheck = runGit(
|
|
1132
|
+
const branchCheck = runGit(
|
|
1133
|
+
["rev-parse", "--verify", `refs/heads/${persistedState.orchBranch}`],
|
|
1134
|
+
root,
|
|
1135
|
+
);
|
|
1005
1136
|
if (branchCheck.ok) {
|
|
1006
1137
|
checks.push({
|
|
1007
1138
|
check: `branch-consistency:${label}`,
|
|
@@ -1012,7 +1143,8 @@ export function runPreResumeDiagnostics(
|
|
|
1012
1143
|
checks.push({
|
|
1013
1144
|
check: `branch-consistency:${label}`,
|
|
1014
1145
|
passed: false,
|
|
1015
|
-
detail:
|
|
1146
|
+
detail:
|
|
1147
|
+
`Orch branch "${persistedState.orchBranch}" not found in ${label}. ` +
|
|
1016
1148
|
`The branch may have been deleted or the repo is in an inconsistent state.`,
|
|
1017
1149
|
});
|
|
1018
1150
|
}
|
|
@@ -1045,18 +1177,17 @@ export function runPreResumeDiagnostics(
|
|
|
1045
1177
|
}
|
|
1046
1178
|
}
|
|
1047
1179
|
|
|
1048
|
-
const failed = checks.filter(c => !c.passed);
|
|
1180
|
+
const failed = checks.filter((c) => !c.passed);
|
|
1049
1181
|
const passed = failed.length === 0;
|
|
1050
1182
|
|
|
1051
1183
|
const summary = passed
|
|
1052
1184
|
? `✅ Pre-resume diagnostics passed (${checks.length} checks)`
|
|
1053
1185
|
: `❌ Pre-resume diagnostics failed (${failed.length}/${checks.length} checks failed):\n` +
|
|
1054
|
-
|
|
1186
|
+
failed.map((c) => ` • ${c.check}: ${c.detail}`).join("\n");
|
|
1055
1187
|
|
|
1056
1188
|
return { passed, checks, summary };
|
|
1057
1189
|
}
|
|
1058
1190
|
|
|
1059
|
-
|
|
1060
1191
|
export async function resumeOrchBatch(
|
|
1061
1192
|
orchConfig: OrchestratorConfig,
|
|
1062
1193
|
runnerConfig: TaskRunnerConfig,
|
|
@@ -1108,10 +1239,7 @@ export async function resumeOrchBatch(
|
|
|
1108
1239
|
persistedState = loadBatchState(stateRoot);
|
|
1109
1240
|
} catch (err: unknown) {
|
|
1110
1241
|
if (err instanceof StateFileError) {
|
|
1111
|
-
onNotify(
|
|
1112
|
-
`❌ Cannot resume: ${err.message}`,
|
|
1113
|
-
"error",
|
|
1114
|
-
);
|
|
1242
|
+
onNotify(`❌ Cannot resume: ${err.message}`, "error");
|
|
1115
1243
|
// ── TP-040 R006: Reset phase on pre-execution early return ──
|
|
1116
1244
|
// The caller may have set batchState.phase = "launching" before
|
|
1117
1245
|
// calling this function. Since we're returning without starting
|
|
@@ -1124,10 +1252,7 @@ export async function resumeOrchBatch(
|
|
|
1124
1252
|
|
|
1125
1253
|
if (!persistedState) {
|
|
1126
1254
|
if (!force) {
|
|
1127
|
-
onNotify(
|
|
1128
|
-
ORCH_MESSAGES.resumeNoState(),
|
|
1129
|
-
"error",
|
|
1130
|
-
);
|
|
1255
|
+
onNotify(ORCH_MESSAGES.resumeNoState(), "error");
|
|
1131
1256
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
1132
1257
|
batchState.phase = "idle";
|
|
1133
1258
|
return;
|
|
@@ -1137,10 +1262,7 @@ export async function resumeOrchBatch(
|
|
|
1137
1262
|
// by `orch_abort()` even though `.pi/batch-state.json` is deleted).
|
|
1138
1263
|
const reconstruction = reconstructBatchStateFromRuntime(stateRoot);
|
|
1139
1264
|
if (!reconstruction.ok) {
|
|
1140
|
-
onNotify(
|
|
1141
|
-
ORCH_MESSAGES.resumeNoStateAfterAbort(reconstruction.error, null),
|
|
1142
|
-
"error",
|
|
1143
|
-
);
|
|
1265
|
+
onNotify(ORCH_MESSAGES.resumeNoStateAfterAbort(reconstruction.error, null), "error");
|
|
1144
1266
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
1145
1267
|
batchState.phase = "idle";
|
|
1146
1268
|
return;
|
|
@@ -1172,7 +1294,11 @@ export async function resumeOrchBatch(
|
|
|
1172
1294
|
const eligibility = checkResumeEligibility(persistedState, force);
|
|
1173
1295
|
if (!eligibility.eligible) {
|
|
1174
1296
|
onNotify(
|
|
1175
|
-
ORCH_MESSAGES.resumePhaseNotResumable(
|
|
1297
|
+
ORCH_MESSAGES.resumePhaseNotResumable(
|
|
1298
|
+
persistedState.batchId,
|
|
1299
|
+
persistedState.phase,
|
|
1300
|
+
eligibility.reason,
|
|
1301
|
+
),
|
|
1176
1302
|
"error",
|
|
1177
1303
|
);
|
|
1178
1304
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
@@ -1181,7 +1307,8 @@ export async function resumeOrchBatch(
|
|
|
1181
1307
|
}
|
|
1182
1308
|
|
|
1183
1309
|
// ── 2b. Force-resume: pre-resume diagnostics & state mutation ──
|
|
1184
|
-
const isForceResume =
|
|
1310
|
+
const isForceResume =
|
|
1311
|
+
force && (persistedState.phase === "stopped" || persistedState.phase === "failed");
|
|
1185
1312
|
if (isForceResume) {
|
|
1186
1313
|
onNotify(
|
|
1187
1314
|
ORCH_MESSAGES.forceResumeStarting(persistedState.batchId, persistedState.phase),
|
|
@@ -1193,10 +1320,7 @@ export async function resumeOrchBatch(
|
|
|
1193
1320
|
onNotify(diagnostics.summary, diagnostics.passed ? "info" : "error");
|
|
1194
1321
|
|
|
1195
1322
|
if (!diagnostics.passed) {
|
|
1196
|
-
onNotify(
|
|
1197
|
-
ORCH_MESSAGES.forceResumeDiagnosticsFailed(persistedState.batchId),
|
|
1198
|
-
"error",
|
|
1199
|
-
);
|
|
1323
|
+
onNotify(ORCH_MESSAGES.forceResumeDiagnosticsFailed(persistedState.batchId), "error");
|
|
1200
1324
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
1201
1325
|
batchState.phase = "idle";
|
|
1202
1326
|
return;
|
|
@@ -1206,17 +1330,19 @@ export async function resumeOrchBatch(
|
|
|
1206
1330
|
persistedState.resilience.resumeForced = true;
|
|
1207
1331
|
|
|
1208
1332
|
// Reset phase to paused so normal resume flow can proceed
|
|
1209
|
-
execLog(
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1333
|
+
execLog(
|
|
1334
|
+
"resume",
|
|
1335
|
+
persistedState.batchId,
|
|
1336
|
+
`force-resume: phase ${persistedState.phase} → paused`,
|
|
1337
|
+
{
|
|
1338
|
+
diagnosticChecks: diagnostics.checks.length,
|
|
1339
|
+
diagnosticsPassed: diagnostics.passed,
|
|
1340
|
+
},
|
|
1341
|
+
);
|
|
1213
1342
|
persistedState.phase = "paused";
|
|
1214
1343
|
}
|
|
1215
1344
|
|
|
1216
|
-
onNotify(
|
|
1217
|
-
ORCH_MESSAGES.resumeStarting(persistedState.batchId, persistedState.phase),
|
|
1218
|
-
"info",
|
|
1219
|
-
);
|
|
1345
|
+
onNotify(ORCH_MESSAGES.resumeStarting(persistedState.batchId, persistedState.phase), "info");
|
|
1220
1346
|
|
|
1221
1347
|
const segmentFrontierByTask = reconstructSegmentFrontier(persistedState);
|
|
1222
1348
|
if (segmentFrontierByTask.size > 0) {
|
|
@@ -1273,14 +1399,19 @@ export async function resumeOrchBatch(
|
|
|
1273
1399
|
// ── 3b. Detect existing worktrees ────────────────────────────
|
|
1274
1400
|
const existingWorktreeTaskIds = new Set<string>();
|
|
1275
1401
|
for (const task of persistedState.tasks) {
|
|
1276
|
-
const laneRecord = persistedState.lanes.find(l => l.taskIds.includes(task.taskId));
|
|
1402
|
+
const laneRecord = persistedState.lanes.find((l) => l.taskIds.includes(task.taskId));
|
|
1277
1403
|
if (laneRecord && laneRecord.worktreePath && existsSync(laneRecord.worktreePath)) {
|
|
1278
1404
|
existingWorktreeTaskIds.add(task.taskId);
|
|
1279
1405
|
}
|
|
1280
1406
|
}
|
|
1281
1407
|
|
|
1282
1408
|
// ── 4. Reconcile task states ─────────────────────────────────
|
|
1283
|
-
const reconciledTasks = reconcileTaskStates(
|
|
1409
|
+
const reconciledTasks = reconcileTaskStates(
|
|
1410
|
+
persistedState,
|
|
1411
|
+
aliveSessions,
|
|
1412
|
+
doneTaskIds,
|
|
1413
|
+
existingWorktreeTaskIds,
|
|
1414
|
+
);
|
|
1284
1415
|
|
|
1285
1416
|
// ── 4b. Clear stale session allocation for tasks reconciled as pending ──
|
|
1286
1417
|
// TP-037 (Bug #102b): Pending tasks that had a sessionName from a prior
|
|
@@ -1292,9 +1423,13 @@ export async function resumeOrchBatch(
|
|
|
1292
1423
|
const stalePendingTaskIds = new Set<string>();
|
|
1293
1424
|
for (const reconciled of reconciledTasks) {
|
|
1294
1425
|
if (reconciled.action === "pending") {
|
|
1295
|
-
const persistedTask = persistedState.tasks.find(t => t.taskId === reconciled.taskId);
|
|
1426
|
+
const persistedTask = persistedState.tasks.find((t) => t.taskId === reconciled.taskId);
|
|
1296
1427
|
if (persistedTask && persistedTask.sessionName) {
|
|
1297
|
-
execLog(
|
|
1428
|
+
execLog(
|
|
1429
|
+
"resume",
|
|
1430
|
+
persistedState.batchId,
|
|
1431
|
+
`clear-stale-session: ${reconciled.taskId} had stale session "${persistedTask.sessionName}" (lane ${persistedTask.laneNumber})`,
|
|
1432
|
+
);
|
|
1298
1433
|
stalePendingTaskIds.add(reconciled.taskId);
|
|
1299
1434
|
persistedTask.sessionName = "";
|
|
1300
1435
|
persistedTask.laneNumber = 0;
|
|
@@ -1305,7 +1440,7 @@ export async function resumeOrchBatch(
|
|
|
1305
1440
|
// (and subsequent serializeBatchState()) won't map them back to the old lane.
|
|
1306
1441
|
if (stalePendingTaskIds.size > 0) {
|
|
1307
1442
|
for (const lane of persistedState.lanes) {
|
|
1308
|
-
lane.taskIds = lane.taskIds.filter(id => !stalePendingTaskIds.has(id));
|
|
1443
|
+
lane.taskIds = lane.taskIds.filter((id) => !stalePendingTaskIds.has(id));
|
|
1309
1444
|
}
|
|
1310
1445
|
}
|
|
1311
1446
|
|
|
@@ -1329,22 +1464,16 @@ export async function resumeOrchBatch(
|
|
|
1329
1464
|
);
|
|
1330
1465
|
|
|
1331
1466
|
if (resumePoint.reconnectTaskIds.length > 0) {
|
|
1332
|
-
onNotify(
|
|
1333
|
-
ORCH_MESSAGES.resumeReconnecting(resumePoint.reconnectTaskIds.length),
|
|
1334
|
-
"info",
|
|
1335
|
-
);
|
|
1467
|
+
onNotify(ORCH_MESSAGES.resumeReconnecting(resumePoint.reconnectTaskIds.length), "info");
|
|
1336
1468
|
}
|
|
1337
1469
|
|
|
1338
1470
|
if (resumePoint.resumeWaveIndex > 0) {
|
|
1339
|
-
onNotify(
|
|
1340
|
-
ORCH_MESSAGES.resumeSkippedWaves(resumePoint.resumeWaveIndex),
|
|
1341
|
-
"info",
|
|
1342
|
-
);
|
|
1471
|
+
onNotify(ORCH_MESSAGES.resumeSkippedWaves(resumePoint.resumeWaveIndex), "info");
|
|
1343
1472
|
}
|
|
1344
1473
|
|
|
1345
1474
|
if (resumePoint.mergeRetryWaveIndexes.length > 0) {
|
|
1346
1475
|
onNotify(
|
|
1347
|
-
`🔀 ${resumePoint.mergeRetryWaveIndexes.length} wave(s) need merge retry: ${resumePoint.mergeRetryWaveIndexes.map(i => `W${i + 1}`).join(", ")}`,
|
|
1476
|
+
`🔀 ${resumePoint.mergeRetryWaveIndexes.length} wave(s) need merge retry: ${resumePoint.mergeRetryWaveIndexes.map((i) => `W${i + 1}`).join(", ")}`,
|
|
1348
1477
|
"warning",
|
|
1349
1478
|
);
|
|
1350
1479
|
}
|
|
@@ -1358,8 +1487,8 @@ export async function resumeOrchBatch(
|
|
|
1358
1487
|
if (!persistedState.orchBranch) {
|
|
1359
1488
|
onNotify(
|
|
1360
1489
|
`❌ Cannot resume batch ${persistedState.batchId}: persisted state has no orch branch. ` +
|
|
1361
|
-
|
|
1362
|
-
|
|
1490
|
+
`This batch was created before orch-branch routing was implemented. ` +
|
|
1491
|
+
`Use /orch-abort to clean up, then start a new batch.`,
|
|
1363
1492
|
"error",
|
|
1364
1493
|
);
|
|
1365
1494
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
@@ -1380,7 +1509,9 @@ export async function resumeOrchBatch(
|
|
|
1380
1509
|
// TP-166: Restore task-level wave metadata for correct display.
|
|
1381
1510
|
// Normalize: fall back to totalWaves for pre-TP-166 state files.
|
|
1382
1511
|
batchState.taskLevelWaveCount = persistedState.taskLevelWaveCount ?? persistedState.totalWaves;
|
|
1383
|
-
batchState.roundToTaskWave = persistedState.roundToTaskWave
|
|
1512
|
+
batchState.roundToTaskWave = persistedState.roundToTaskWave
|
|
1513
|
+
? [...persistedState.roundToTaskWave]
|
|
1514
|
+
: undefined;
|
|
1384
1515
|
batchState.totalTasks = persistedState.totalTasks;
|
|
1385
1516
|
batchState.succeededTasks = resumePoint.completedTaskIds.length;
|
|
1386
1517
|
batchState.failedTasks = resumePoint.failedTaskIds.length;
|
|
@@ -1410,7 +1541,11 @@ export async function resumeOrchBatch(
|
|
|
1410
1541
|
}
|
|
1411
1542
|
if (uncountedBlocked > 0) {
|
|
1412
1543
|
batchState.blockedTasks += uncountedBlocked;
|
|
1413
|
-
execLog(
|
|
1544
|
+
execLog(
|
|
1545
|
+
"resume",
|
|
1546
|
+
persistedState.batchId,
|
|
1547
|
+
`blocked counter fix: ${uncountedBlocked} persisted-blocked task(s) in unvisited waves added to blockedTasks`,
|
|
1548
|
+
);
|
|
1414
1549
|
}
|
|
1415
1550
|
}
|
|
1416
1551
|
|
|
@@ -1453,7 +1588,8 @@ export async function resumeOrchBatch(
|
|
|
1453
1588
|
"warning",
|
|
1454
1589
|
);
|
|
1455
1590
|
} else {
|
|
1456
|
-
const errMsg =
|
|
1591
|
+
const errMsg =
|
|
1592
|
+
`Failed to re-create orch branch "${batchState.orchBranch}" in repo "${repoId}": ${createRes.stderr}. ` +
|
|
1457
1593
|
`Cannot resume without orch branch isolation.`;
|
|
1458
1594
|
execLog("resume", batchState.batchId, errMsg, {
|
|
1459
1595
|
orchBranch: batchState.orchBranch,
|
|
@@ -1501,11 +1637,10 @@ export async function resumeOrchBatch(
|
|
|
1501
1637
|
}
|
|
1502
1638
|
}
|
|
1503
1639
|
|
|
1504
|
-
|
|
1505
1640
|
// ── 8. Handle alive sessions (reconnect) ─────────────────────
|
|
1506
1641
|
// For tasks with alive sessions, we need to wait for them to complete.
|
|
1507
1642
|
// We poll each alive session's .DONE file.
|
|
1508
|
-
const reconnectTasks = reconciledTasks.filter(t => t.action === "reconnect");
|
|
1643
|
+
const reconnectTasks = reconciledTasks.filter((t) => t.action === "reconnect");
|
|
1509
1644
|
const reconnectFinalStatus = new Map<string, LaneTaskStatus>();
|
|
1510
1645
|
|
|
1511
1646
|
if (reconnectTasks.length > 0) {
|
|
@@ -1515,9 +1650,7 @@ export async function resumeOrchBatch(
|
|
|
1515
1650
|
if (!parsedTask) continue;
|
|
1516
1651
|
|
|
1517
1652
|
// Find the lane info from persisted state
|
|
1518
|
-
const laneRecord = persistedState.lanes.find(
|
|
1519
|
-
l => l.taskIds.includes(task.taskId),
|
|
1520
|
-
);
|
|
1653
|
+
const laneRecord = persistedState.lanes.find((l) => l.taskIds.includes(task.taskId));
|
|
1521
1654
|
if (!laneRecord) continue;
|
|
1522
1655
|
|
|
1523
1656
|
// Build a minimal AllocatedLane for polling
|
|
@@ -1552,12 +1685,20 @@ export async function resumeOrchBatch(
|
|
|
1552
1685
|
terminateAliveV2Agents(stateRoot, persistedState.batchId, laneRecord.laneSessionId);
|
|
1553
1686
|
try {
|
|
1554
1687
|
const laneResult = await executeLaneV2(
|
|
1555
|
-
lane,
|
|
1556
|
-
|
|
1557
|
-
|
|
1688
|
+
lane,
|
|
1689
|
+
orchConfig,
|
|
1690
|
+
laneRepoRoot,
|
|
1691
|
+
batchState.pauseSignal,
|
|
1692
|
+
workspaceRoot,
|
|
1693
|
+
!!workspaceConfig,
|
|
1694
|
+
{
|
|
1695
|
+
ORCH_BATCH_ID: batchState.batchId,
|
|
1696
|
+
...buildReviewerEnv(runnerConfig.reviewer),
|
|
1697
|
+
...buildWorkerExcludeEnv(runnerConfig.workerExcludeExtensions),
|
|
1698
|
+
},
|
|
1558
1699
|
emitAlert,
|
|
1559
1700
|
);
|
|
1560
|
-
const taskResult = laneResult.tasks.find(t => t.taskId === task.taskId);
|
|
1701
|
+
const taskResult = laneResult.tasks.find((t) => t.taskId === task.taskId);
|
|
1561
1702
|
if (taskResult?.status === "succeeded") {
|
|
1562
1703
|
reconnectFinalStatus.set(task.taskId, "succeeded");
|
|
1563
1704
|
completedTaskSet.add(task.taskId);
|
|
@@ -1577,13 +1718,17 @@ export async function resumeOrchBatch(
|
|
|
1577
1718
|
completedTaskSet.delete(task.taskId);
|
|
1578
1719
|
reconnectTaskSet.delete(task.taskId);
|
|
1579
1720
|
batchState.failedTasks++;
|
|
1580
|
-
execLog(
|
|
1721
|
+
execLog(
|
|
1722
|
+
"resume",
|
|
1723
|
+
task.taskId,
|
|
1724
|
+
`V2 reconnect error: ${err instanceof Error ? err.message : String(err)}`,
|
|
1725
|
+
);
|
|
1581
1726
|
}
|
|
1582
1727
|
}
|
|
1583
1728
|
}
|
|
1584
1729
|
|
|
1585
1730
|
// ── 8b. Handle re-execute tasks (dead session + existing worktree) ──
|
|
1586
|
-
const reExecuteTasks = reconciledTasks.filter(t => t.action === "re-execute");
|
|
1731
|
+
const reExecuteTasks = reconciledTasks.filter((t) => t.action === "re-execute");
|
|
1587
1732
|
const reExecuteFinalStatus = new Map<string, LaneTaskStatus>();
|
|
1588
1733
|
const reExecAllocatedLanes: AllocatedLane[] = [];
|
|
1589
1734
|
|
|
@@ -1597,9 +1742,7 @@ export async function resumeOrchBatch(
|
|
|
1597
1742
|
const parsedTask = discovery.pending.get(task.taskId);
|
|
1598
1743
|
if (!parsedTask) continue;
|
|
1599
1744
|
|
|
1600
|
-
const laneRecord = persistedState.lanes.find(
|
|
1601
|
-
l => l.taskIds.includes(task.taskId),
|
|
1602
|
-
);
|
|
1745
|
+
const laneRecord = persistedState.lanes.find((l) => l.taskIds.includes(task.taskId));
|
|
1603
1746
|
if (!laneRecord) continue;
|
|
1604
1747
|
|
|
1605
1748
|
const allocatedTask: AllocatedTask = {
|
|
@@ -1634,12 +1777,20 @@ export async function resumeOrchBatch(
|
|
|
1634
1777
|
// TP-112: Runtime V2 re-execution.
|
|
1635
1778
|
terminateAliveV2Agents(stateRoot, batchState.batchId, laneRecord.laneSessionId);
|
|
1636
1779
|
const laneResult = await executeLaneV2(
|
|
1637
|
-
lane,
|
|
1638
|
-
|
|
1639
|
-
|
|
1780
|
+
lane,
|
|
1781
|
+
orchConfig,
|
|
1782
|
+
reExecRepoRoot,
|
|
1783
|
+
batchState.pauseSignal,
|
|
1784
|
+
workspaceRoot,
|
|
1785
|
+
!!workspaceConfig,
|
|
1786
|
+
{
|
|
1787
|
+
ORCH_BATCH_ID: batchState.batchId,
|
|
1788
|
+
...buildReviewerEnv(runnerConfig.reviewer),
|
|
1789
|
+
...buildWorkerExcludeEnv(runnerConfig.workerExcludeExtensions),
|
|
1790
|
+
},
|
|
1640
1791
|
emitAlert,
|
|
1641
1792
|
);
|
|
1642
|
-
const taskResult = laneResult.tasks.find(t => t.taskId === task.taskId);
|
|
1793
|
+
const taskResult = laneResult.tasks.find((t) => t.taskId === task.taskId);
|
|
1643
1794
|
const pollResult: { status: LaneTaskStatus; exitReason: string; doneFileFound: boolean } = {
|
|
1644
1795
|
status: taskResult?.status ?? "failed",
|
|
1645
1796
|
exitReason: taskResult?.exitReason ?? "V2 re-execution completed",
|
|
@@ -1660,7 +1811,11 @@ export async function resumeOrchBatch(
|
|
|
1660
1811
|
completedTaskSet.delete(task.taskId);
|
|
1661
1812
|
reExecuteTaskSet.delete(task.taskId);
|
|
1662
1813
|
batchState.failedTasks++;
|
|
1663
|
-
execLog(
|
|
1814
|
+
execLog(
|
|
1815
|
+
"resume",
|
|
1816
|
+
task.taskId,
|
|
1817
|
+
`re-executed task ${pollResult.status}: ${pollResult.exitReason}`,
|
|
1818
|
+
);
|
|
1664
1819
|
}
|
|
1665
1820
|
} catch (err: unknown) {
|
|
1666
1821
|
reExecuteFinalStatus.set(task.taskId, "failed");
|
|
@@ -1683,16 +1838,13 @@ export async function resumeOrchBatch(
|
|
|
1683
1838
|
.map(([taskId]) => taskId);
|
|
1684
1839
|
|
|
1685
1840
|
if (succeededReExecTaskIds.length > 0) {
|
|
1686
|
-
onNotify(
|
|
1687
|
-
`🔀 Merging ${reExecAllocatedLanes.length} re-executed lane branch(es)...`,
|
|
1688
|
-
"info",
|
|
1689
|
-
);
|
|
1841
|
+
onNotify(`🔀 Merging ${reExecAllocatedLanes.length} re-executed lane branch(es)...`, "info");
|
|
1690
1842
|
|
|
1691
1843
|
// Build synthetic WaveExecutionResult for mergeWaveByRepo()
|
|
1692
|
-
const syntheticLaneResults: LaneExecutionResult[] = reExecAllocatedLanes.map(lane => ({
|
|
1844
|
+
const syntheticLaneResults: LaneExecutionResult[] = reExecAllocatedLanes.map((lane) => ({
|
|
1693
1845
|
laneNumber: lane.laneNumber,
|
|
1694
1846
|
laneId: lane.laneId,
|
|
1695
|
-
tasks: lane.tasks.map(t => ({
|
|
1847
|
+
tasks: lane.tasks.map((t) => ({
|
|
1696
1848
|
taskId: t.taskId,
|
|
1697
1849
|
status: "succeeded" as LaneTaskStatus,
|
|
1698
1850
|
startTime: Date.now(),
|
|
@@ -1758,7 +1910,10 @@ export async function resumeOrchBatch(
|
|
|
1758
1910
|
// Clean up merged branches (resolve per-lane repo root for workspace mode)
|
|
1759
1911
|
// TP-032 R006-3: Exclude verification_new_failure lanes from branch cleanup
|
|
1760
1912
|
for (const lr of reExecMergeResult.laneResults) {
|
|
1761
|
-
if (
|
|
1913
|
+
if (
|
|
1914
|
+
!lr.error &&
|
|
1915
|
+
(lr.result?.status === "SUCCESS" || lr.result?.status === "CONFLICT_RESOLVED")
|
|
1916
|
+
) {
|
|
1762
1917
|
const laneRepoRoot = resolveRepoRoot(lr.repoId, repoRoot, workspaceConfig);
|
|
1763
1918
|
deleteBranchBestEffort(lr.sourceBranch, laneRepoRoot);
|
|
1764
1919
|
}
|
|
@@ -1788,39 +1943,53 @@ export async function resumeOrchBatch(
|
|
|
1788
1943
|
// records with repo attribution (laneNumber, laneId, branch, repoId).
|
|
1789
1944
|
// Without this, the `resume-reconciliation` checkpoint would serialize
|
|
1790
1945
|
// empty lanes[], losing all lane context until a new wave allocates.
|
|
1791
|
-
let latestAllocatedLanes: AllocatedLane[] = reconstructAllocatedLanes(
|
|
1946
|
+
let latestAllocatedLanes: AllocatedLane[] = reconstructAllocatedLanes(
|
|
1947
|
+
persistedState.lanes,
|
|
1948
|
+
persistedState.tasks,
|
|
1949
|
+
);
|
|
1792
1950
|
|
|
1793
1951
|
// Track all repo roots encountered during execution (persisted + newly allocated).
|
|
1794
1952
|
// Used by inter-wave reset and terminal cleanup to cover repos introduced
|
|
1795
1953
|
// after resume starts (not present in persisted lanes).
|
|
1796
1954
|
// Initialized from collectRepoRoots() helper for parity with other callers.
|
|
1797
|
-
const encounteredRepoRoots = new Set(
|
|
1798
|
-
collectRepoRoots(persistedState, repoRoot, workspaceConfig),
|
|
1799
|
-
);
|
|
1955
|
+
const encounteredRepoRoots = new Set(collectRepoRoots(persistedState, repoRoot, workspaceConfig));
|
|
1800
1956
|
|
|
1801
1957
|
// Build outcomes from reconciled tasks
|
|
1802
1958
|
for (const task of reconciledTasks) {
|
|
1803
|
-
const persistedTask = persistedState.tasks.find(t => t.taskId === task.taskId);
|
|
1959
|
+
const persistedTask = persistedState.tasks.find((t) => t.taskId === task.taskId);
|
|
1804
1960
|
const reconnectStatus = reconnectFinalStatus.get(task.taskId);
|
|
1805
1961
|
const reExecuteStatus = reExecuteFinalStatus.get(task.taskId);
|
|
1806
|
-
const status =
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1962
|
+
const status =
|
|
1963
|
+
task.action === "reconnect"
|
|
1964
|
+
? reconnectStatus || "running"
|
|
1965
|
+
: task.action === "re-execute"
|
|
1966
|
+
? reExecuteStatus || "pending"
|
|
1967
|
+
: task.liveStatus;
|
|
1968
|
+
const isTerminal =
|
|
1969
|
+
status === "succeeded" || status === "failed" || status === "stalled" || status === "skipped";
|
|
1812
1970
|
allTaskOutcomes.push({
|
|
1813
1971
|
taskId: task.taskId,
|
|
1814
1972
|
status,
|
|
1815
1973
|
startTime: persistedTask?.startedAt ?? null,
|
|
1816
1974
|
endTime: isTerminal ? Date.now() : null,
|
|
1817
|
-
exitReason:
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1975
|
+
exitReason:
|
|
1976
|
+
task.action === "mark-complete"
|
|
1977
|
+
? ".DONE file found on resume"
|
|
1978
|
+
: task.action === "mark-failed"
|
|
1979
|
+
? "Session dead, no .DONE file, no worktree on resume"
|
|
1980
|
+
: task.action === "reconnect"
|
|
1981
|
+
? status === "succeeded"
|
|
1982
|
+
? "Reconnected task completed"
|
|
1983
|
+
: status === "failed"
|
|
1984
|
+
? "Reconnected task failed"
|
|
1985
|
+
: "Reconnected to alive session"
|
|
1986
|
+
: task.action === "re-execute"
|
|
1987
|
+
? status === "succeeded"
|
|
1988
|
+
? "Re-executed task completed"
|
|
1989
|
+
: status === "failed"
|
|
1990
|
+
? "Re-executed task failed"
|
|
1991
|
+
: "Re-executing in existing worktree"
|
|
1992
|
+
: (persistedTask?.exitReason ?? ""),
|
|
1824
1993
|
sessionName: persistedTask?.sessionName ?? "",
|
|
1825
1994
|
doneFileFound: status === "succeeded" ? true : task.doneFileFound,
|
|
1826
1995
|
laneNumber: persistedTask?.laneNumber,
|
|
@@ -1842,14 +2011,27 @@ export async function resumeOrchBatch(
|
|
|
1842
2011
|
batchState.blockedTaskIds.add(taskId);
|
|
1843
2012
|
}
|
|
1844
2013
|
if (reconciledBlocked.size > 0) {
|
|
1845
|
-
execLog(
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
2014
|
+
execLog(
|
|
2015
|
+
"resume",
|
|
2016
|
+
batchState.batchId,
|
|
2017
|
+
`skip-dependents: ${reconciledBlocked.size} task(s) blocked from reconciled failures`,
|
|
2018
|
+
{
|
|
2019
|
+
blocked: [...reconciledBlocked].sort().join(","),
|
|
2020
|
+
sources: [...failedTaskSet].sort().join(","),
|
|
2021
|
+
},
|
|
2022
|
+
);
|
|
1849
2023
|
}
|
|
1850
2024
|
}
|
|
1851
2025
|
|
|
1852
|
-
persistRuntimeState(
|
|
2026
|
+
persistRuntimeState(
|
|
2027
|
+
"resume-reconciliation",
|
|
2028
|
+
batchState,
|
|
2029
|
+
wavePlan,
|
|
2030
|
+
latestAllocatedLanes,
|
|
2031
|
+
allTaskOutcomes,
|
|
2032
|
+
discovery ?? null,
|
|
2033
|
+
stateRoot,
|
|
2034
|
+
);
|
|
1853
2035
|
|
|
1854
2036
|
// ── 10. Continue wave execution ──────────────────────────────
|
|
1855
2037
|
// We need to execute remaining waves starting from resumeWaveIndex.
|
|
@@ -1868,33 +2050,53 @@ export async function resumeOrchBatch(
|
|
|
1868
2050
|
// Check pause signal
|
|
1869
2051
|
if (batchState.pauseSignal.paused) {
|
|
1870
2052
|
batchState.phase = "paused";
|
|
1871
|
-
persistRuntimeState(
|
|
1872
|
-
|
|
2053
|
+
persistRuntimeState(
|
|
2054
|
+
"pause-before-wave",
|
|
2055
|
+
batchState,
|
|
2056
|
+
wavePlan,
|
|
2057
|
+
latestAllocatedLanes,
|
|
2058
|
+
allTaskOutcomes,
|
|
2059
|
+
discovery,
|
|
2060
|
+
stateRoot,
|
|
2061
|
+
);
|
|
2062
|
+
const { displayWave: pauseWave } = resolveDisplayWaveNumber(
|
|
2063
|
+
waveIdx,
|
|
2064
|
+
roundToTaskWave,
|
|
2065
|
+
taskLevelWaveCount,
|
|
2066
|
+
);
|
|
1873
2067
|
onNotify(`⏸️ Batch paused before wave ${pauseWave}.`, "warning");
|
|
1874
2068
|
break;
|
|
1875
2069
|
}
|
|
1876
2070
|
|
|
1877
2071
|
batchState.currentWaveIndex = waveIdx;
|
|
1878
|
-
persistRuntimeState(
|
|
2072
|
+
persistRuntimeState(
|
|
2073
|
+
"wave-index-change",
|
|
2074
|
+
batchState,
|
|
2075
|
+
wavePlan,
|
|
2076
|
+
latestAllocatedLanes,
|
|
2077
|
+
allTaskOutcomes,
|
|
2078
|
+
discovery,
|
|
2079
|
+
stateRoot,
|
|
2080
|
+
);
|
|
1879
2081
|
|
|
1880
2082
|
// Get wave tasks, filtering out completed/failed/skipped/blocked ones.
|
|
1881
2083
|
// Persisted "skipped" tasks are terminal and must never be re-executed.
|
|
1882
2084
|
let waveTasks = wavePlan[waveIdx].filter(
|
|
1883
|
-
|
|
2085
|
+
(taskId) =>
|
|
2086
|
+
!completedTaskSet.has(taskId) &&
|
|
1884
2087
|
!failedTaskSet.has(taskId) &&
|
|
1885
2088
|
persistedStatusByTaskId.get(taskId) !== "skipped" &&
|
|
1886
2089
|
!batchState.blockedTaskIds.has(taskId),
|
|
1887
2090
|
);
|
|
1888
2091
|
|
|
1889
2092
|
// Also filter tasks where discovery doesn't have them as pending
|
|
1890
|
-
waveTasks = waveTasks.filter(taskId => discovery.pending.has(taskId));
|
|
2093
|
+
waveTasks = waveTasks.filter((taskId) => discovery.pending.has(taskId));
|
|
1891
2094
|
|
|
1892
2095
|
// Count only newly blocked tasks (not already persisted) to avoid double-counting.
|
|
1893
2096
|
// persistedState.blockedTaskIds were already counted in persistedState.blockedTasks
|
|
1894
2097
|
// which initialized batchState.blockedTasks.
|
|
1895
2098
|
const blockedInWave = wavePlan[waveIdx].filter(
|
|
1896
|
-
taskId => batchState.blockedTaskIds.has(taskId) &&
|
|
1897
|
-
!persistedBlockedTaskIds.has(taskId),
|
|
2099
|
+
(taskId) => batchState.blockedTaskIds.has(taskId) && !persistedBlockedTaskIds.has(taskId),
|
|
1898
2100
|
);
|
|
1899
2101
|
if (blockedInWave.length > 0) {
|
|
1900
2102
|
batchState.blockedTasks += blockedInWave.length;
|
|
@@ -1904,13 +2106,20 @@ export async function resumeOrchBatch(
|
|
|
1904
2106
|
// TP-037 Bug #102: Check if this wave needs merge retry.
|
|
1905
2107
|
// All tasks are terminal but the merge may have failed/been interrupted.
|
|
1906
2108
|
if (resumePoint.mergeRetryWaveIndexes.includes(waveIdx)) {
|
|
1907
|
-
execLog(
|
|
1908
|
-
|
|
2109
|
+
execLog(
|
|
2110
|
+
"resume",
|
|
2111
|
+
batchState.batchId,
|
|
2112
|
+
`wave ${waveIdx + 1}: all tasks done but merge needs retry`,
|
|
2113
|
+
);
|
|
2114
|
+
onNotify(
|
|
2115
|
+
`🔀 Wave ${resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave}: retrying merge (tasks already complete, merge was missing/failed)`,
|
|
2116
|
+
"info",
|
|
2117
|
+
);
|
|
1909
2118
|
|
|
1910
2119
|
// Reconstruct lanes for this wave from persisted state
|
|
1911
2120
|
const waveTaskIds = new Set(wavePlan[waveIdx]);
|
|
1912
|
-
const waveLaneRecords = persistedState.lanes.filter(
|
|
1913
|
-
lane
|
|
2121
|
+
const waveLaneRecords = persistedState.lanes.filter((lane) =>
|
|
2122
|
+
lane.taskIds.some((tid) => waveTaskIds.has(tid)),
|
|
1914
2123
|
);
|
|
1915
2124
|
const mergeRetryLanes = reconstructAllocatedLanes(waveLaneRecords, persistedState.tasks);
|
|
1916
2125
|
|
|
@@ -1918,18 +2127,14 @@ export async function resumeOrchBatch(
|
|
|
1918
2127
|
// Crucial for orch_force_merge: tasks intentionally marked "skipped" must
|
|
1919
2128
|
// remain skipped here (not failed), otherwise mixed-outcome detection would
|
|
1920
2129
|
// trigger again and block the forced merge recovery path.
|
|
1921
|
-
const succeededTaskIds = wavePlan[waveIdx].filter(
|
|
1922
|
-
taskId => completedTaskSet.has(taskId),
|
|
1923
|
-
);
|
|
2130
|
+
const succeededTaskIds = wavePlan[waveIdx].filter((taskId) => completedTaskSet.has(taskId));
|
|
1924
2131
|
const skippedTaskIds = wavePlan[waveIdx].filter(
|
|
1925
|
-
taskId => persistedStatusByTaskId.get(taskId) === "skipped",
|
|
1926
|
-
);
|
|
1927
|
-
const failedTaskIds = wavePlan[waveIdx].filter(
|
|
1928
|
-
taskId => {
|
|
1929
|
-
const status = persistedStatusByTaskId.get(taskId);
|
|
1930
|
-
return status === "failed" || status === "stalled";
|
|
1931
|
-
},
|
|
2132
|
+
(taskId) => persistedStatusByTaskId.get(taskId) === "skipped",
|
|
1932
2133
|
);
|
|
2134
|
+
const failedTaskIds = wavePlan[waveIdx].filter((taskId) => {
|
|
2135
|
+
const status = persistedStatusByTaskId.get(taskId);
|
|
2136
|
+
return status === "failed" || status === "stalled";
|
|
2137
|
+
});
|
|
1933
2138
|
|
|
1934
2139
|
const syntheticLaneResults: LaneExecutionResult[] = mergeRetryLanes.map((lane) => {
|
|
1935
2140
|
const laneTasks = lane.tasks.map((t) => {
|
|
@@ -1953,10 +2158,13 @@ export async function resumeOrchBatch(
|
|
|
1953
2158
|
startTime: Date.now(),
|
|
1954
2159
|
endTime: Date.now(),
|
|
1955
2160
|
exitReason:
|
|
1956
|
-
status === "succeeded"
|
|
1957
|
-
|
|
1958
|
-
: status === "
|
|
1959
|
-
|
|
2161
|
+
status === "succeeded"
|
|
2162
|
+
? "Task completed (merge retry)"
|
|
2163
|
+
: status === "skipped"
|
|
2164
|
+
? "Task skipped (merge retry)"
|
|
2165
|
+
: status === "stalled"
|
|
2166
|
+
? "Task stalled (merge retry)"
|
|
2167
|
+
: "Task failed (merge retry)",
|
|
1960
2168
|
sessionName: lane.laneSessionId,
|
|
1961
2169
|
doneFileFound: status === "succeeded",
|
|
1962
2170
|
laneNumber: lane.laneNumber,
|
|
@@ -1968,7 +2176,9 @@ export async function resumeOrchBatch(
|
|
|
1968
2176
|
);
|
|
1969
2177
|
const laneHasSucceeded = laneTasks.some((t) => t.status === "succeeded");
|
|
1970
2178
|
const overallStatus = laneHasHardFailure
|
|
1971
|
-
?
|
|
2179
|
+
? laneHasSucceeded
|
|
2180
|
+
? "partial"
|
|
2181
|
+
: "failed"
|
|
1972
2182
|
: "succeeded";
|
|
1973
2183
|
|
|
1974
2184
|
return {
|
|
@@ -1999,7 +2209,15 @@ export async function resumeOrchBatch(
|
|
|
1999
2209
|
};
|
|
2000
2210
|
|
|
2001
2211
|
batchState.phase = "merging";
|
|
2002
|
-
persistRuntimeState(
|
|
2212
|
+
persistRuntimeState(
|
|
2213
|
+
"merge-retry-start",
|
|
2214
|
+
batchState,
|
|
2215
|
+
wavePlan,
|
|
2216
|
+
latestAllocatedLanes,
|
|
2217
|
+
allTaskOutcomes,
|
|
2218
|
+
discovery,
|
|
2219
|
+
stateRoot,
|
|
2220
|
+
);
|
|
2003
2221
|
|
|
2004
2222
|
const mergeRetryResult = await mergeWaveByRepo(
|
|
2005
2223
|
mergeRetryLanes,
|
|
@@ -2020,10 +2238,16 @@ export async function resumeOrchBatch(
|
|
|
2020
2238
|
batchState.mergeResults.push(mergeRetryResult);
|
|
2021
2239
|
|
|
2022
2240
|
if (mergeRetryResult.status === "succeeded") {
|
|
2023
|
-
onNotify(
|
|
2241
|
+
onNotify(
|
|
2242
|
+
`✅ Wave ${resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave} merge retry succeeded`,
|
|
2243
|
+
"info",
|
|
2244
|
+
);
|
|
2024
2245
|
// Clean up merged branches
|
|
2025
2246
|
for (const lr of mergeRetryResult.laneResults) {
|
|
2026
|
-
if (
|
|
2247
|
+
if (
|
|
2248
|
+
!lr.error &&
|
|
2249
|
+
(lr.result?.status === "SUCCESS" || lr.result?.status === "CONFLICT_RESOLVED")
|
|
2250
|
+
) {
|
|
2027
2251
|
const laneRepoRoot = resolveRepoRoot(lr.repoId, repoRoot, workspaceConfig);
|
|
2028
2252
|
deleteBranchBestEffort(lr.sourceBranch, laneRepoRoot);
|
|
2029
2253
|
}
|
|
@@ -2035,27 +2259,61 @@ export async function resumeOrchBatch(
|
|
|
2035
2259
|
);
|
|
2036
2260
|
// Apply merge failure policy (same as normal wave merge failure)
|
|
2037
2261
|
const policyResult = computeMergeFailurePolicy(mergeRetryResult, waveIdx, orchConfig);
|
|
2038
|
-
execLog(
|
|
2262
|
+
execLog(
|
|
2263
|
+
"batch",
|
|
2264
|
+
batchState.batchId,
|
|
2265
|
+
`merge retry failure — applying ${policyResult.policy} policy`,
|
|
2266
|
+
policyResult.logDetails,
|
|
2267
|
+
);
|
|
2039
2268
|
batchState.phase = policyResult.targetPhase;
|
|
2040
2269
|
batchState.errors.push(policyResult.errorMessage);
|
|
2041
|
-
persistRuntimeState(
|
|
2270
|
+
persistRuntimeState(
|
|
2271
|
+
policyResult.persistTrigger,
|
|
2272
|
+
batchState,
|
|
2273
|
+
wavePlan,
|
|
2274
|
+
latestAllocatedLanes,
|
|
2275
|
+
allTaskOutcomes,
|
|
2276
|
+
discovery,
|
|
2277
|
+
stateRoot,
|
|
2278
|
+
);
|
|
2042
2279
|
onNotify(policyResult.notifyMessage, policyResult.notifyLevel);
|
|
2043
2280
|
preserveWorktreesForResume = true;
|
|
2044
2281
|
break;
|
|
2045
2282
|
}
|
|
2046
2283
|
|
|
2047
2284
|
batchState.phase = "executing";
|
|
2048
|
-
persistRuntimeState(
|
|
2285
|
+
persistRuntimeState(
|
|
2286
|
+
"merge-retry-complete",
|
|
2287
|
+
batchState,
|
|
2288
|
+
wavePlan,
|
|
2289
|
+
latestAllocatedLanes,
|
|
2290
|
+
allTaskOutcomes,
|
|
2291
|
+
discovery,
|
|
2292
|
+
stateRoot,
|
|
2293
|
+
);
|
|
2049
2294
|
} else {
|
|
2050
|
-
execLog(
|
|
2295
|
+
execLog(
|
|
2296
|
+
"resume",
|
|
2297
|
+
batchState.batchId,
|
|
2298
|
+
`wave ${waveIdx + 1}: no tasks to execute (all completed/blocked)`,
|
|
2299
|
+
);
|
|
2051
2300
|
}
|
|
2052
2301
|
continue;
|
|
2053
2302
|
}
|
|
2054
2303
|
|
|
2055
2304
|
{
|
|
2056
|
-
const { displayWave, displayTotal } = resolveDisplayWaveNumber(
|
|
2305
|
+
const { displayWave, displayTotal } = resolveDisplayWaveNumber(
|
|
2306
|
+
waveIdx,
|
|
2307
|
+
roundToTaskWave,
|
|
2308
|
+
taskLevelWaveCount,
|
|
2309
|
+
);
|
|
2057
2310
|
onNotify(
|
|
2058
|
-
ORCH_MESSAGES.orchWaveStart(
|
|
2311
|
+
ORCH_MESSAGES.orchWaveStart(
|
|
2312
|
+
displayWave,
|
|
2313
|
+
displayTotal,
|
|
2314
|
+
waveTasks.length,
|
|
2315
|
+
Math.min(waveTasks.length, orchConfig.orchestrator.max_lanes),
|
|
2316
|
+
),
|
|
2059
2317
|
"info",
|
|
2060
2318
|
);
|
|
2061
2319
|
}
|
|
@@ -2063,7 +2321,15 @@ export async function resumeOrchBatch(
|
|
|
2063
2321
|
const handleResumeMonitorUpdate: MonitorUpdateCallback = (monitorState) => {
|
|
2064
2322
|
const changed = syncTaskOutcomesFromMonitor(monitorState, allTaskOutcomes);
|
|
2065
2323
|
if (changed) {
|
|
2066
|
-
persistRuntimeState(
|
|
2324
|
+
persistRuntimeState(
|
|
2325
|
+
"task-transition",
|
|
2326
|
+
batchState,
|
|
2327
|
+
wavePlan,
|
|
2328
|
+
latestAllocatedLanes,
|
|
2329
|
+
allTaskOutcomes,
|
|
2330
|
+
discovery,
|
|
2331
|
+
stateRoot,
|
|
2332
|
+
);
|
|
2067
2333
|
}
|
|
2068
2334
|
onMonitorUpdate?.(monitorState);
|
|
2069
2335
|
};
|
|
@@ -2088,7 +2354,15 @@ export async function resumeOrchBatch(
|
|
|
2088
2354
|
encounteredRepoRoots.add(resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig));
|
|
2089
2355
|
}
|
|
2090
2356
|
if (seedPendingOutcomesForAllocatedLanes(lanes, allTaskOutcomes)) {
|
|
2091
|
-
persistRuntimeState(
|
|
2357
|
+
persistRuntimeState(
|
|
2358
|
+
"wave-lanes-allocated",
|
|
2359
|
+
batchState,
|
|
2360
|
+
wavePlan,
|
|
2361
|
+
latestAllocatedLanes,
|
|
2362
|
+
allTaskOutcomes,
|
|
2363
|
+
discovery,
|
|
2364
|
+
stateRoot,
|
|
2365
|
+
);
|
|
2092
2366
|
}
|
|
2093
2367
|
},
|
|
2094
2368
|
workspaceConfig,
|
|
@@ -2135,9 +2409,16 @@ export async function resumeOrchBatch(
|
|
|
2135
2409
|
|
|
2136
2410
|
// ── TP-076: Emit supervisor alerts for task failures ────
|
|
2137
2411
|
for (const taskId of waveResult.failedTaskIds) {
|
|
2138
|
-
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
2139
|
-
const laneForTask = latestAllocatedLanes.find(l => l.tasks.some(t => t.taskId === taskId));
|
|
2140
|
-
|
|
2412
|
+
const outcome = allTaskOutcomes.find((o) => o.taskId === taskId);
|
|
2413
|
+
const laneForTask = latestAllocatedLanes.find((l) => l.tasks.some((t) => t.taskId === taskId));
|
|
2414
|
+
// TP-195: corrected the lookup to the real source of segment
|
|
2415
|
+
// metadata. `batchState.tasks` does not exist on
|
|
2416
|
+
// `OrchBatchRuntimeState` (it's on `PersistedBatchState`); the
|
|
2417
|
+
// previous read would have thrown `undefined.find is not a
|
|
2418
|
+
// function` if hit at runtime. The allocated lane carries the
|
|
2419
|
+
// `ParsedTask` payload via `AllocatedTask.task`, which has
|
|
2420
|
+
// `segmentIds`/`activeSegmentId` already populated by discovery.
|
|
2421
|
+
const taskRecord = laneForTask?.tasks.find((t) => t.taskId === taskId)?.task;
|
|
2141
2422
|
const exitReason = outcome?.exitReason || "unknown";
|
|
2142
2423
|
const hasPartialProgress = (outcome?.partialProgressCommits ?? 0) > 0;
|
|
2143
2424
|
const segmentFrontier = buildSupervisorSegmentFrontierSnapshot(
|
|
@@ -2147,12 +2428,14 @@ export async function resumeOrchBatch(
|
|
|
2147
2428
|
batchState.segments,
|
|
2148
2429
|
outcome?.segmentId,
|
|
2149
2430
|
);
|
|
2150
|
-
const segmentId =
|
|
2151
|
-
??
|
|
2152
|
-
|
|
2153
|
-
??
|
|
2431
|
+
const segmentId =
|
|
2432
|
+
outcome?.segmentId ??
|
|
2433
|
+
taskRecord?.activeSegmentId ??
|
|
2434
|
+
segmentFrontier?.activeSegmentId ??
|
|
2435
|
+
undefined;
|
|
2154
2436
|
const repoId = segmentId
|
|
2155
|
-
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ??
|
|
2437
|
+
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ??
|
|
2438
|
+
laneForTask?.repoId)
|
|
2156
2439
|
: laneForTask?.repoId;
|
|
2157
2440
|
const segmentSummary = segmentId
|
|
2158
2441
|
? ` Segment: ${segmentId}${repoId ? ` (repo: ${repoId})` : ""}\n`
|
|
@@ -2197,11 +2480,23 @@ export async function resumeOrchBatch(
|
|
|
2197
2480
|
});
|
|
2198
2481
|
}
|
|
2199
2482
|
|
|
2200
|
-
persistRuntimeState(
|
|
2483
|
+
persistRuntimeState(
|
|
2484
|
+
"wave-execution-complete",
|
|
2485
|
+
batchState,
|
|
2486
|
+
wavePlan,
|
|
2487
|
+
latestAllocatedLanes,
|
|
2488
|
+
allTaskOutcomes,
|
|
2489
|
+
discovery,
|
|
2490
|
+
stateRoot,
|
|
2491
|
+
);
|
|
2201
2492
|
|
|
2202
2493
|
const elapsedSec = Math.round((waveResult.endedAt - waveResult.startedAt) / 1000);
|
|
2203
2494
|
{
|
|
2204
|
-
const { displayWave: completeDisplayWave } = resolveDisplayWaveNumber(
|
|
2495
|
+
const { displayWave: completeDisplayWave } = resolveDisplayWaveNumber(
|
|
2496
|
+
waveIdx,
|
|
2497
|
+
roundToTaskWave,
|
|
2498
|
+
taskLevelWaveCount,
|
|
2499
|
+
);
|
|
2205
2500
|
onNotify(
|
|
2206
2501
|
ORCH_MESSAGES.orchWaveComplete(
|
|
2207
2502
|
completeDisplayWave,
|
|
@@ -2218,13 +2513,29 @@ export async function resumeOrchBatch(
|
|
|
2218
2513
|
if (waveResult.stoppedEarly) {
|
|
2219
2514
|
if (waveResult.policyApplied === "stop-all") {
|
|
2220
2515
|
batchState.phase = "stopped";
|
|
2221
|
-
persistRuntimeState(
|
|
2516
|
+
persistRuntimeState(
|
|
2517
|
+
"stop-all",
|
|
2518
|
+
batchState,
|
|
2519
|
+
wavePlan,
|
|
2520
|
+
latestAllocatedLanes,
|
|
2521
|
+
allTaskOutcomes,
|
|
2522
|
+
discovery,
|
|
2523
|
+
stateRoot,
|
|
2524
|
+
);
|
|
2222
2525
|
onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-all"), "error");
|
|
2223
2526
|
break;
|
|
2224
2527
|
}
|
|
2225
2528
|
if (waveResult.policyApplied === "stop-wave") {
|
|
2226
2529
|
batchState.phase = "stopped";
|
|
2227
|
-
persistRuntimeState(
|
|
2530
|
+
persistRuntimeState(
|
|
2531
|
+
"stop-wave",
|
|
2532
|
+
batchState,
|
|
2533
|
+
wavePlan,
|
|
2534
|
+
latestAllocatedLanes,
|
|
2535
|
+
allTaskOutcomes,
|
|
2536
|
+
discovery,
|
|
2537
|
+
stateRoot,
|
|
2538
|
+
);
|
|
2228
2539
|
onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-wave"), "error");
|
|
2229
2540
|
break;
|
|
2230
2541
|
}
|
|
@@ -2237,29 +2548,41 @@ export async function resumeOrchBatch(
|
|
|
2237
2548
|
for (const lr of waveResult.laneResults) {
|
|
2238
2549
|
laneOutcomeByNumber.set(lr.laneNumber, lr);
|
|
2239
2550
|
}
|
|
2240
|
-
const mixedOutcomeLanes = waveResult.laneResults.filter(lr => {
|
|
2241
|
-
const hasSucceeded = lr.tasks.some(t => t.status === "succeeded");
|
|
2242
|
-
const hasHardFailure = lr.tasks.some(
|
|
2243
|
-
t => t.status === "failed" || t.status === "stalled",
|
|
2244
|
-
);
|
|
2551
|
+
const mixedOutcomeLanes = waveResult.laneResults.filter((lr) => {
|
|
2552
|
+
const hasSucceeded = lr.tasks.some((t) => t.status === "succeeded");
|
|
2553
|
+
const hasHardFailure = lr.tasks.some((t) => t.status === "failed" || t.status === "stalled");
|
|
2245
2554
|
return hasSucceeded && hasHardFailure;
|
|
2246
2555
|
});
|
|
2247
2556
|
|
|
2248
2557
|
if (waveResult.succeededTaskIds.length > 0) {
|
|
2249
|
-
const mergeableLaneCount = waveResult.allocatedLanes.filter(lane => {
|
|
2558
|
+
const mergeableLaneCount = waveResult.allocatedLanes.filter((lane) => {
|
|
2250
2559
|
const outcome = laneOutcomeByNumber.get(lane.laneNumber);
|
|
2251
2560
|
if (!outcome) return false;
|
|
2252
|
-
const hasSucceeded = outcome.tasks.some(t => t.status === "succeeded");
|
|
2561
|
+
const hasSucceeded = outcome.tasks.some((t) => t.status === "succeeded");
|
|
2253
2562
|
const hasHardFailure = outcome.tasks.some(
|
|
2254
|
-
t => t.status === "failed" || t.status === "stalled",
|
|
2563
|
+
(t) => t.status === "failed" || t.status === "stalled",
|
|
2255
2564
|
);
|
|
2256
2565
|
return hasSucceeded && !hasHardFailure;
|
|
2257
2566
|
}).length;
|
|
2258
2567
|
|
|
2259
2568
|
if (mergeableLaneCount > 0) {
|
|
2260
2569
|
batchState.phase = "merging";
|
|
2261
|
-
persistRuntimeState(
|
|
2262
|
-
|
|
2570
|
+
persistRuntimeState(
|
|
2571
|
+
"merge-start",
|
|
2572
|
+
batchState,
|
|
2573
|
+
wavePlan,
|
|
2574
|
+
latestAllocatedLanes,
|
|
2575
|
+
allTaskOutcomes,
|
|
2576
|
+
discovery,
|
|
2577
|
+
stateRoot,
|
|
2578
|
+
);
|
|
2579
|
+
onNotify(
|
|
2580
|
+
ORCH_MESSAGES.orchMergeStart(
|
|
2581
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2582
|
+
mergeableLaneCount,
|
|
2583
|
+
),
|
|
2584
|
+
"info",
|
|
2585
|
+
);
|
|
2263
2586
|
|
|
2264
2587
|
mergeResult = await mergeWaveByRepo(
|
|
2265
2588
|
waveResult.allocatedLanes,
|
|
@@ -2287,35 +2610,65 @@ export async function resumeOrchBatch(
|
|
|
2287
2610
|
if (lr.error) {
|
|
2288
2611
|
onNotify(ORCH_MESSAGES.orchMergeLaneFailed(lr.laneNumber, lr.error), "error");
|
|
2289
2612
|
} else if (lr.result?.status === "SUCCESS") {
|
|
2290
|
-
onNotify(
|
|
2613
|
+
onNotify(
|
|
2614
|
+
ORCH_MESSAGES.orchMergeLaneSuccess(lr.laneNumber, lr.result.merge_commit, durationSec),
|
|
2615
|
+
"info",
|
|
2616
|
+
);
|
|
2291
2617
|
} else if (lr.result?.status === "CONFLICT_RESOLVED") {
|
|
2292
|
-
onNotify(
|
|
2293
|
-
|
|
2618
|
+
onNotify(
|
|
2619
|
+
ORCH_MESSAGES.orchMergeLaneConflictResolved(
|
|
2620
|
+
lr.laneNumber,
|
|
2621
|
+
lr.result.conflicts.length,
|
|
2622
|
+
durationSec,
|
|
2623
|
+
),
|
|
2624
|
+
"info",
|
|
2625
|
+
);
|
|
2626
|
+
} else if (
|
|
2627
|
+
lr.result?.status === "CONFLICT_UNRESOLVED" ||
|
|
2628
|
+
lr.result?.status === "BUILD_FAILURE"
|
|
2629
|
+
) {
|
|
2294
2630
|
onNotify(ORCH_MESSAGES.orchMergeLaneFailed(lr.laneNumber, lr.result.status), "error");
|
|
2295
2631
|
}
|
|
2296
2632
|
}
|
|
2297
2633
|
|
|
2298
2634
|
if (mixedOutcomeLanes.length > 0) {
|
|
2299
|
-
const mixedIds = mixedOutcomeLanes.map(l => `lane-${l.laneNumber}`).join(", ");
|
|
2635
|
+
const mixedIds = mixedOutcomeLanes.map((l) => `lane-${l.laneNumber}`).join(", ");
|
|
2300
2636
|
const failureReason =
|
|
2301
2637
|
`Lane(s) ${mixedIds} contain both succeeded and failed tasks. ` +
|
|
2302
2638
|
`Automatic partial-branch merge is disabled to avoid dropping succeeded commits.`;
|
|
2303
|
-
mergeResult = {
|
|
2639
|
+
mergeResult = {
|
|
2640
|
+
...mergeResult,
|
|
2641
|
+
status: "partial",
|
|
2642
|
+
failedLane: mixedOutcomeLanes[0].laneNumber,
|
|
2643
|
+
failureReason,
|
|
2644
|
+
};
|
|
2304
2645
|
// Update the already-pushed reference so persisted state reflects "partial"
|
|
2305
2646
|
batchState.mergeResults[batchState.mergeResults.length - 1] = mergeResult;
|
|
2306
2647
|
}
|
|
2307
2648
|
|
|
2308
2649
|
// TP-032 R006-3: Exclude verification_new_failure lanes from success count
|
|
2309
2650
|
const mergedCount = mergeResult.laneResults.filter(
|
|
2310
|
-
r =>
|
|
2651
|
+
(r) =>
|
|
2652
|
+
!r.error && (r.result?.status === "SUCCESS" || r.result?.status === "CONFLICT_RESOLVED"),
|
|
2311
2653
|
).length;
|
|
2312
2654
|
const mergeTotalSec = Math.round(mergeResult.totalDurationMs / 1000);
|
|
2313
2655
|
|
|
2314
2656
|
if (mergeResult.status === "succeeded") {
|
|
2315
|
-
onNotify(
|
|
2657
|
+
onNotify(
|
|
2658
|
+
ORCH_MESSAGES.orchMergeComplete(
|
|
2659
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2660
|
+
mergedCount,
|
|
2661
|
+
mergeTotalSec,
|
|
2662
|
+
),
|
|
2663
|
+
"info",
|
|
2664
|
+
);
|
|
2316
2665
|
} else {
|
|
2317
2666
|
onNotify(
|
|
2318
|
-
ORCH_MESSAGES.orchMergeFailed(
|
|
2667
|
+
ORCH_MESSAGES.orchMergeFailed(
|
|
2668
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2669
|
+
mergeResult.failedLane ?? 0,
|
|
2670
|
+
mergeResult.failureReason || "unknown",
|
|
2671
|
+
),
|
|
2319
2672
|
"error",
|
|
2320
2673
|
);
|
|
2321
2674
|
|
|
@@ -2329,9 +2682,17 @@ export async function resumeOrchBatch(
|
|
|
2329
2682
|
}
|
|
2330
2683
|
|
|
2331
2684
|
batchState.phase = "executing";
|
|
2332
|
-
persistRuntimeState(
|
|
2685
|
+
persistRuntimeState(
|
|
2686
|
+
"merge-complete",
|
|
2687
|
+
batchState,
|
|
2688
|
+
wavePlan,
|
|
2689
|
+
latestAllocatedLanes,
|
|
2690
|
+
allTaskOutcomes,
|
|
2691
|
+
discovery,
|
|
2692
|
+
stateRoot,
|
|
2693
|
+
);
|
|
2333
2694
|
} else if (mixedOutcomeLanes.length > 0) {
|
|
2334
|
-
const mixedIds = mixedOutcomeLanes.map(l => `lane-${l.laneNumber}`).join(", ");
|
|
2695
|
+
const mixedIds = mixedOutcomeLanes.map((l) => `lane-${l.laneNumber}`).join(", ");
|
|
2335
2696
|
mergeResult = {
|
|
2336
2697
|
waveIndex: waveIdx + 1,
|
|
2337
2698
|
status: "partial",
|
|
@@ -2346,14 +2707,28 @@ export async function resumeOrchBatch(
|
|
|
2346
2707
|
// Downstream retry/update paths assume the current wave has an entry.
|
|
2347
2708
|
batchState.mergeResults.push(mergeResult);
|
|
2348
2709
|
onNotify(
|
|
2349
|
-
ORCH_MESSAGES.orchMergeFailed(
|
|
2710
|
+
ORCH_MESSAGES.orchMergeFailed(
|
|
2711
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2712
|
+
mergeResult.failedLane,
|
|
2713
|
+
mergeResult.failureReason || "unknown",
|
|
2714
|
+
),
|
|
2350
2715
|
"error",
|
|
2351
2716
|
);
|
|
2352
2717
|
} else {
|
|
2353
|
-
onNotify(
|
|
2718
|
+
onNotify(
|
|
2719
|
+
ORCH_MESSAGES.orchMergeSkipped(
|
|
2720
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2721
|
+
),
|
|
2722
|
+
"info",
|
|
2723
|
+
);
|
|
2354
2724
|
}
|
|
2355
2725
|
} else {
|
|
2356
|
-
onNotify(
|
|
2726
|
+
onNotify(
|
|
2727
|
+
ORCH_MESSAGES.orchMergeSkipped(
|
|
2728
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2729
|
+
),
|
|
2730
|
+
"info",
|
|
2731
|
+
);
|
|
2357
2732
|
}
|
|
2358
2733
|
|
|
2359
2734
|
// ── TP-033: Safe-stop on rollback failure ─────────────────
|
|
@@ -2363,30 +2738,44 @@ export async function resumeOrchBatch(
|
|
|
2363
2738
|
if (mergeResult?.rollbackFailed) {
|
|
2364
2739
|
// TP-033 R004-2: Include persistence error warning when transaction
|
|
2365
2740
|
// record files may be missing, so operator knows to inspect manually
|
|
2366
|
-
const hasPersistErrors =
|
|
2741
|
+
const hasPersistErrors =
|
|
2742
|
+
mergeResult.persistenceErrors && mergeResult.persistenceErrors.length > 0;
|
|
2367
2743
|
const persistWarning = hasPersistErrors
|
|
2368
2744
|
? ` WARNING: ${mergeResult.persistenceErrors!.length} transaction record(s) failed to persist — recovery file(s) may be missing.`
|
|
2369
2745
|
: "";
|
|
2370
2746
|
|
|
2371
|
-
execLog(
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2747
|
+
execLog(
|
|
2748
|
+
"batch",
|
|
2749
|
+
batchState.batchId,
|
|
2750
|
+
"SAFE-STOP: verification rollback failed — forcing paused regardless of policy",
|
|
2751
|
+
{
|
|
2752
|
+
waveIndex: waveIdx,
|
|
2753
|
+
configPolicy: orchConfig.failure.on_merge_failure,
|
|
2754
|
+
...(hasPersistErrors ? { persistenceErrors: mergeResult.persistenceErrors } : {}),
|
|
2755
|
+
},
|
|
2756
|
+
);
|
|
2376
2757
|
|
|
2377
2758
|
batchState.phase = "paused";
|
|
2378
2759
|
batchState.errors.push(
|
|
2379
2760
|
`Safe-stop at wave ${waveIdx + 1}: verification rollback failed. ` +
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2761
|
+
`Merge worktree and temp branch preserved for recovery. ` +
|
|
2762
|
+
`Check transaction records in .pi/verification/ for recovery commands.` +
|
|
2763
|
+
persistWarning,
|
|
2764
|
+
);
|
|
2765
|
+
persistRuntimeState(
|
|
2766
|
+
"merge-rollback-safe-stop",
|
|
2767
|
+
batchState,
|
|
2768
|
+
wavePlan,
|
|
2769
|
+
latestAllocatedLanes,
|
|
2770
|
+
allTaskOutcomes,
|
|
2771
|
+
discovery,
|
|
2772
|
+
stateRoot,
|
|
2383
2773
|
);
|
|
2384
|
-
persistRuntimeState("merge-rollback-safe-stop", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discovery, stateRoot);
|
|
2385
2774
|
onNotify(
|
|
2386
2775
|
`🛑 Safe-stop: verification rollback failed at wave ${waveIdx + 1}. ` +
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2776
|
+
`Batch force-paused. Merge worktree preserved for manual recovery. ` +
|
|
2777
|
+
`See .pi/verification/ transaction records for recovery commands.` +
|
|
2778
|
+
persistWarning,
|
|
2390
2779
|
"error",
|
|
2391
2780
|
);
|
|
2392
2781
|
|
|
@@ -2448,7 +2837,16 @@ export async function resumeOrchBatch(
|
|
|
2448
2837
|
resumeBackend,
|
|
2449
2838
|
);
|
|
2450
2839
|
},
|
|
2451
|
-
persist: (trigger) =>
|
|
2840
|
+
persist: (trigger) =>
|
|
2841
|
+
persistRuntimeState(
|
|
2842
|
+
trigger,
|
|
2843
|
+
batchState,
|
|
2844
|
+
wavePlan,
|
|
2845
|
+
latestAllocatedLanes,
|
|
2846
|
+
allTaskOutcomes,
|
|
2847
|
+
discovery,
|
|
2848
|
+
stateRoot,
|
|
2849
|
+
),
|
|
2452
2850
|
log: (message, details) => execLog("batch", batchState.batchId, message, details),
|
|
2453
2851
|
notify: (message, level) => onNotify(message, level),
|
|
2454
2852
|
updateMergeResult: (result) => {
|
|
@@ -2462,13 +2860,29 @@ export async function resumeOrchBatch(
|
|
|
2462
2860
|
if (retryOutcome.kind === "retry_succeeded") {
|
|
2463
2861
|
mergeResult = retryOutcome.mergeResult;
|
|
2464
2862
|
batchState.phase = "executing";
|
|
2465
|
-
persistRuntimeState(
|
|
2863
|
+
persistRuntimeState(
|
|
2864
|
+
"merge-retry-succeeded",
|
|
2865
|
+
batchState,
|
|
2866
|
+
wavePlan,
|
|
2867
|
+
latestAllocatedLanes,
|
|
2868
|
+
allTaskOutcomes,
|
|
2869
|
+
discovery,
|
|
2870
|
+
stateRoot,
|
|
2871
|
+
);
|
|
2466
2872
|
// Fall through to normal post-merge flow
|
|
2467
2873
|
} else if (retryOutcome.kind === "safe_stop") {
|
|
2468
2874
|
mergeResult = retryOutcome.mergeResult;
|
|
2469
2875
|
batchState.phase = "paused";
|
|
2470
2876
|
batchState.errors.push(retryOutcome.errorMessage);
|
|
2471
|
-
persistRuntimeState(
|
|
2877
|
+
persistRuntimeState(
|
|
2878
|
+
"merge-rollback-safe-stop",
|
|
2879
|
+
batchState,
|
|
2880
|
+
wavePlan,
|
|
2881
|
+
latestAllocatedLanes,
|
|
2882
|
+
allTaskOutcomes,
|
|
2883
|
+
discovery,
|
|
2884
|
+
stateRoot,
|
|
2885
|
+
);
|
|
2472
2886
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
2473
2887
|
|
|
2474
2888
|
// ── TP-076: Emit supervisor alert for merge safe-stop ──
|
|
@@ -2494,7 +2908,8 @@ export async function resumeOrchBatch(
|
|
|
2494
2908
|
} else if (retryOutcome.kind === "exhausted") {
|
|
2495
2909
|
// TP-033 R006-2: Force paused regardless of on_merge_failure config.
|
|
2496
2910
|
mergeResult = retryOutcome.mergeResult;
|
|
2497
|
-
const exhaustionMsg =
|
|
2911
|
+
const exhaustionMsg =
|
|
2912
|
+
retryOutcome.errorMessage +
|
|
2498
2913
|
` [${retryOutcome.classification ?? "unknown"} ${retryOutcome.lastDecision.currentAttempt}/${retryOutcome.lastDecision.maxAttempts}, scope=${retryOutcome.scopeKey}]`;
|
|
2499
2914
|
|
|
2500
2915
|
execLog("batch", batchState.batchId, `merge retry exhausted — forcing paused`, {
|
|
@@ -2506,7 +2921,15 @@ export async function resumeOrchBatch(
|
|
|
2506
2921
|
|
|
2507
2922
|
batchState.phase = "paused";
|
|
2508
2923
|
batchState.errors.push(exhaustionMsg);
|
|
2509
|
-
persistRuntimeState(
|
|
2924
|
+
persistRuntimeState(
|
|
2925
|
+
"merge-retry-exhausted",
|
|
2926
|
+
batchState,
|
|
2927
|
+
wavePlan,
|
|
2928
|
+
latestAllocatedLanes,
|
|
2929
|
+
allTaskOutcomes,
|
|
2930
|
+
discovery,
|
|
2931
|
+
stateRoot,
|
|
2932
|
+
);
|
|
2510
2933
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
2511
2934
|
|
|
2512
2935
|
// ── TP-076: Emit supervisor alert for merge retry exhausted ──
|
|
@@ -2539,11 +2962,24 @@ export async function resumeOrchBatch(
|
|
|
2539
2962
|
? ` [not retriable: ${retryOutcome.classification}, scope=${retryOutcome.scopeKey}]`
|
|
2540
2963
|
: "";
|
|
2541
2964
|
|
|
2542
|
-
execLog(
|
|
2965
|
+
execLog(
|
|
2966
|
+
"batch",
|
|
2967
|
+
batchState.batchId,
|
|
2968
|
+
`merge failure — applying ${policyResult.policy} policy${classNote}`,
|
|
2969
|
+
policyResult.logDetails,
|
|
2970
|
+
);
|
|
2543
2971
|
|
|
2544
2972
|
batchState.phase = policyResult.targetPhase;
|
|
2545
2973
|
batchState.errors.push(policyResult.errorMessage + classNote);
|
|
2546
|
-
persistRuntimeState(
|
|
2974
|
+
persistRuntimeState(
|
|
2975
|
+
policyResult.persistTrigger,
|
|
2976
|
+
batchState,
|
|
2977
|
+
wavePlan,
|
|
2978
|
+
latestAllocatedLanes,
|
|
2979
|
+
allTaskOutcomes,
|
|
2980
|
+
discovery,
|
|
2981
|
+
stateRoot,
|
|
2982
|
+
);
|
|
2547
2983
|
onNotify(policyResult.notifyMessage + classNote, policyResult.notifyLevel);
|
|
2548
2984
|
|
|
2549
2985
|
// ── TP-076: Emit supervisor alert for merge failure (no-retry policy) ──
|
|
@@ -2575,9 +3011,15 @@ export async function resumeOrchBatch(
|
|
|
2575
3011
|
// TP-032 R006-3: Exclude verification_new_failure lanes from branch cleanup
|
|
2576
3012
|
if (mergeResult && mergeResult.status === "succeeded") {
|
|
2577
3013
|
for (const lr of mergeResult.laneResults) {
|
|
2578
|
-
if (
|
|
3014
|
+
if (
|
|
3015
|
+
!lr.error &&
|
|
3016
|
+
(lr.result?.status === "SUCCESS" || lr.result?.status === "CONFLICT_RESOLVED")
|
|
3017
|
+
) {
|
|
2579
3018
|
const laneRepoRoot = resolveRepoRoot(lr.repoId, repoRoot, workspaceConfig);
|
|
2580
|
-
const ancestorCheck = runGit(
|
|
3019
|
+
const ancestorCheck = runGit(
|
|
3020
|
+
["merge-base", "--is-ancestor", lr.sourceBranch, lr.targetBranch],
|
|
3021
|
+
laneRepoRoot,
|
|
3022
|
+
);
|
|
2581
3023
|
if (ancestorCheck.ok) {
|
|
2582
3024
|
deleteBranchBestEffort(lr.sourceBranch, laneRepoRoot);
|
|
2583
3025
|
}
|
|
@@ -2601,30 +3043,46 @@ export async function resumeOrchBatch(
|
|
|
2601
3043
|
let targetBranch = batchState.orchBranch;
|
|
2602
3044
|
if (repoId && perRepoRoot !== repoRoot) {
|
|
2603
3045
|
try {
|
|
2604
|
-
targetBranch = resolveBaseBranch(
|
|
2605
|
-
|
|
3046
|
+
targetBranch = resolveBaseBranch(
|
|
3047
|
+
repoId,
|
|
3048
|
+
perRepoRoot,
|
|
3049
|
+
batchState.orchBranch,
|
|
3050
|
+
workspaceConfig,
|
|
3051
|
+
);
|
|
3052
|
+
} catch {
|
|
3053
|
+
/* fall back to orchBranch */
|
|
3054
|
+
}
|
|
2606
3055
|
}
|
|
2607
3056
|
return { repoRoot: perRepoRoot, targetBranch };
|
|
2608
3057
|
},
|
|
2609
3058
|
);
|
|
2610
3059
|
ppUnsafeBranches = ppResult.unsafeBranches;
|
|
2611
|
-
if (ppResult.results.some(r => r.saved)) {
|
|
2612
|
-
execLog(
|
|
2613
|
-
|
|
3060
|
+
if (ppResult.results.some((r) => r.saved)) {
|
|
3061
|
+
execLog(
|
|
3062
|
+
"batch",
|
|
3063
|
+
batchState.batchId,
|
|
3064
|
+
`preserved partial progress for ${ppResult.results.filter((r) => r.saved).length} failed task(s) before inter-wave reset`,
|
|
3065
|
+
);
|
|
2614
3066
|
}
|
|
2615
3067
|
// Log per-task warnings for failed preservation attempts
|
|
2616
3068
|
for (const r of ppResult.results) {
|
|
2617
3069
|
if (!r.saved && (r.commitCount > 0 || r.error)) {
|
|
2618
|
-
execLog(
|
|
3070
|
+
execLog(
|
|
3071
|
+
"batch",
|
|
3072
|
+
batchState.batchId,
|
|
2619
3073
|
`WARNING: Failed to preserve partial progress for task ${r.taskId} ` +
|
|
2620
|
-
|
|
2621
|
-
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" }
|
|
3074
|
+
`(${r.commitCount} commit(s) at risk on lane branch)`,
|
|
3075
|
+
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" },
|
|
3076
|
+
);
|
|
2622
3077
|
}
|
|
2623
3078
|
}
|
|
2624
3079
|
if (ppUnsafeBranches.size > 0) {
|
|
2625
|
-
execLog(
|
|
3080
|
+
execLog(
|
|
3081
|
+
"batch",
|
|
3082
|
+
batchState.batchId,
|
|
2626
3083
|
`WARNING: ${ppUnsafeBranches.size} lane branch(es) could not be preserved — skipping reset for those lanes to prevent commit loss`,
|
|
2627
|
-
{ unsafeBranches: [...ppUnsafeBranches] }
|
|
3084
|
+
{ unsafeBranches: [...ppUnsafeBranches] },
|
|
3085
|
+
);
|
|
2628
3086
|
}
|
|
2629
3087
|
// TP-028: Stamp task outcomes with partial progress data for persistence
|
|
2630
3088
|
applyPartialProgressToOutcomes(ppResult, allTaskOutcomes);
|
|
@@ -2636,7 +3094,10 @@ export async function resumeOrchBatch(
|
|
|
2636
3094
|
// TP-029 R006: Track worktrees that failed reset AND removal
|
|
2637
3095
|
// so the cleanup gate only fires on true stale state, not
|
|
2638
3096
|
// successfully-reset reusable worktrees. (Parity with engine.ts)
|
|
2639
|
-
const failedRemovalWorktrees = new Map<
|
|
3097
|
+
const failedRemovalWorktrees = new Map<
|
|
3098
|
+
string,
|
|
3099
|
+
{ repoId: string | undefined; paths: string[] }
|
|
3100
|
+
>();
|
|
2640
3101
|
|
|
2641
3102
|
// Use encounteredRepoRoots which includes both persisted lanes
|
|
2642
3103
|
// AND newly allocated lanes from resumed waves, ensuring repos
|
|
@@ -2652,7 +3113,12 @@ export async function resumeOrchBatch(
|
|
|
2652
3113
|
} else {
|
|
2653
3114
|
const repoId = resolveRepoIdFromRoot(perRepoRoot, workspaceConfig);
|
|
2654
3115
|
try {
|
|
2655
|
-
targetBranch = resolveBaseBranch(
|
|
3116
|
+
targetBranch = resolveBaseBranch(
|
|
3117
|
+
repoId,
|
|
3118
|
+
perRepoRoot,
|
|
3119
|
+
batchState.orchBranch,
|
|
3120
|
+
workspaceConfig,
|
|
3121
|
+
);
|
|
2656
3122
|
} catch {
|
|
2657
3123
|
// If resolution fails, fall back to orchBranch (reset will
|
|
2658
3124
|
// fail gracefully and trigger worktree removal)
|
|
@@ -2663,9 +3129,12 @@ export async function resumeOrchBatch(
|
|
|
2663
3129
|
// TP-028: Skip reset for worktrees whose lane branch has
|
|
2664
3130
|
// unsaved partial progress (preservation failed with commits)
|
|
2665
3131
|
if (ppUnsafeBranches.has(wt.branch)) {
|
|
2666
|
-
execLog(
|
|
3132
|
+
execLog(
|
|
3133
|
+
"batch",
|
|
3134
|
+
batchState.batchId,
|
|
2667
3135
|
`skipping worktree reset for lane ${wt.laneNumber} — branch "${wt.branch}" has unsaved partial progress`,
|
|
2668
|
-
{ path: wt.path, branch: wt.branch }
|
|
3136
|
+
{ path: wt.path, branch: wt.branch },
|
|
3137
|
+
);
|
|
2669
3138
|
continue;
|
|
2670
3139
|
}
|
|
2671
3140
|
|
|
@@ -2676,9 +3145,8 @@ export async function resumeOrchBatch(
|
|
|
2676
3145
|
} catch {
|
|
2677
3146
|
forceCleanupWorktree(wt, perRepoRoot, batchState.batchId);
|
|
2678
3147
|
// Track this worktree for the cleanup gate — it may still be registered
|
|
2679
|
-
const perRepoId =
|
|
2680
|
-
? undefined
|
|
2681
|
-
: resolveRepoIdFromRoot(perRepoRoot, workspaceConfig);
|
|
3148
|
+
const perRepoId =
|
|
3149
|
+
perRepoRoot === repoRoot ? undefined : resolveRepoIdFromRoot(perRepoRoot, workspaceConfig);
|
|
2682
3150
|
if (!failedRemovalWorktrees.has(perRepoRoot)) {
|
|
2683
3151
|
failedRemovalWorktrees.set(perRepoRoot, { repoId: perRepoId, paths: [] });
|
|
2684
3152
|
}
|
|
@@ -2699,9 +3167,9 @@ export async function resumeOrchBatch(
|
|
|
2699
3167
|
if (failedRemovalWorktrees.size > 0) {
|
|
2700
3168
|
for (const [perRepoRoot, { repoId: perRepoId, paths: failedPaths }] of failedRemovalWorktrees) {
|
|
2701
3169
|
const remaining = listWorktrees(wtPrefix, perRepoRoot, resetOpId, batchState.batchId);
|
|
2702
|
-
const remainingPaths = new Set(remaining.map(wt => wt.path));
|
|
3170
|
+
const remainingPaths = new Set(remaining.map((wt) => wt.path));
|
|
2703
3171
|
// Only report worktrees that were targeted for removal but are still registered
|
|
2704
|
-
const stale = failedPaths.filter(p => remainingPaths.has(p));
|
|
3172
|
+
const stale = failedPaths.filter((p) => remainingPaths.has(p));
|
|
2705
3173
|
if (stale.length > 0) {
|
|
2706
3174
|
cleanupGateFailures.push({
|
|
2707
3175
|
repoRoot: perRepoRoot,
|
|
@@ -2715,11 +3183,24 @@ export async function resumeOrchBatch(
|
|
|
2715
3183
|
if (cleanupGateFailures.length > 0) {
|
|
2716
3184
|
const gatePolicyResult = computeCleanupGatePolicy(waveIdx, cleanupGateFailures);
|
|
2717
3185
|
|
|
2718
|
-
execLog(
|
|
3186
|
+
execLog(
|
|
3187
|
+
"batch",
|
|
3188
|
+
batchState.batchId,
|
|
3189
|
+
`cleanup gate failed — pausing batch`,
|
|
3190
|
+
gatePolicyResult.logDetails,
|
|
3191
|
+
);
|
|
2719
3192
|
|
|
2720
3193
|
batchState.phase = gatePolicyResult.targetPhase;
|
|
2721
3194
|
batchState.errors.push(gatePolicyResult.errorMessage);
|
|
2722
|
-
persistRuntimeState(
|
|
3195
|
+
persistRuntimeState(
|
|
3196
|
+
gatePolicyResult.persistTrigger,
|
|
3197
|
+
batchState,
|
|
3198
|
+
wavePlan,
|
|
3199
|
+
latestAllocatedLanes,
|
|
3200
|
+
allTaskOutcomes,
|
|
3201
|
+
discovery,
|
|
3202
|
+
stateRoot,
|
|
3203
|
+
);
|
|
2723
3204
|
onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
|
|
2724
3205
|
preserveWorktreesForResume = true;
|
|
2725
3206
|
break;
|
|
@@ -2731,11 +3212,18 @@ export async function resumeOrchBatch(
|
|
|
2731
3212
|
// TP-031 (R006): Parity with engine.ts — this check MUST run before cleanup
|
|
2732
3213
|
// so that worktrees survive when failedTasks > 0. Without this, cleanup
|
|
2733
3214
|
// deletes worktrees before the batch is marked "paused", breaking resumability.
|
|
2734
|
-
if (
|
|
2735
|
-
|
|
2736
|
-
batchState.
|
|
3215
|
+
if (
|
|
3216
|
+
!preserveWorktreesForResume &&
|
|
3217
|
+
((batchState.phase as OrchBatchPhase) === "executing" ||
|
|
3218
|
+
(batchState.phase as OrchBatchPhase) === "merging") &&
|
|
3219
|
+
batchState.failedTasks > 0
|
|
3220
|
+
) {
|
|
2737
3221
|
preserveWorktreesForResume = true;
|
|
2738
|
-
execLog(
|
|
3222
|
+
execLog(
|
|
3223
|
+
"resume",
|
|
3224
|
+
batchState.batchId,
|
|
3225
|
+
"pre-cleanup: failedTasks > 0 detected, preserving worktrees for resume",
|
|
3226
|
+
);
|
|
2739
3227
|
}
|
|
2740
3228
|
|
|
2741
3229
|
// ── 11. Cleanup and terminal state ───────────────────────────
|
|
@@ -2754,24 +3242,32 @@ export async function resumeOrchBatch(
|
|
|
2754
3242
|
if (repoId && perRepoRoot !== repoRoot) {
|
|
2755
3243
|
try {
|
|
2756
3244
|
targetBranch = resolveBaseBranch(repoId, perRepoRoot, batchState.orchBranch, workspaceConfig);
|
|
2757
|
-
} catch {
|
|
3245
|
+
} catch {
|
|
3246
|
+
/* fall back to orchBranch */
|
|
3247
|
+
}
|
|
2758
3248
|
}
|
|
2759
3249
|
return { repoRoot: perRepoRoot, targetBranch };
|
|
2760
3250
|
},
|
|
2761
3251
|
);
|
|
2762
|
-
if (ppResult.results.some(r => r.saved)) {
|
|
2763
|
-
execLog(
|
|
2764
|
-
|
|
3252
|
+
if (ppResult.results.some((r) => r.saved)) {
|
|
3253
|
+
execLog(
|
|
3254
|
+
"batch",
|
|
3255
|
+
batchState.batchId,
|
|
3256
|
+
`preserved partial progress for ${ppResult.results.filter((r) => r.saved).length} failed task(s) before terminal cleanup`,
|
|
3257
|
+
);
|
|
2765
3258
|
}
|
|
2766
3259
|
// Log warnings for failed preservation attempts — at terminal cleanup
|
|
2767
3260
|
// we cannot skip deletion (batch is ending), but operators need to know
|
|
2768
3261
|
// that commits may become unreachable via reflog only.
|
|
2769
3262
|
for (const r of ppResult.results) {
|
|
2770
3263
|
if (!r.saved && (r.commitCount > 0 || r.error)) {
|
|
2771
|
-
execLog(
|
|
3264
|
+
execLog(
|
|
3265
|
+
"batch",
|
|
3266
|
+
batchState.batchId,
|
|
2772
3267
|
`WARNING: Failed to preserve partial progress for task ${r.taskId} ` +
|
|
2773
|
-
|
|
2774
|
-
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" }
|
|
3268
|
+
`(${r.commitCount} commit(s) may become unreachable after cleanup)`,
|
|
3269
|
+
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" },
|
|
3270
|
+
);
|
|
2775
3271
|
}
|
|
2776
3272
|
}
|
|
2777
3273
|
// TP-028: Stamp task outcomes with partial progress data for persistence
|
|
@@ -2813,14 +3309,24 @@ export async function resumeOrchBatch(
|
|
|
2813
3309
|
targetBranch = undefined;
|
|
2814
3310
|
}
|
|
2815
3311
|
}
|
|
2816
|
-
removeAllWorktrees(
|
|
3312
|
+
removeAllWorktrees(
|
|
3313
|
+
wtPrefix,
|
|
3314
|
+
perRepoRoot,
|
|
3315
|
+
cleanupOpId,
|
|
3316
|
+
targetBranch,
|
|
3317
|
+
batchState.batchId,
|
|
3318
|
+
orchConfig,
|
|
3319
|
+
);
|
|
2817
3320
|
}
|
|
2818
3321
|
}
|
|
2819
3322
|
|
|
2820
3323
|
batchState.endedAt = Date.now();
|
|
2821
3324
|
const totalElapsedSec = Math.round((batchState.endedAt - batchState.startedAt) / 1000);
|
|
2822
3325
|
|
|
2823
|
-
if (
|
|
3326
|
+
if (
|
|
3327
|
+
(batchState.phase as OrchBatchPhase) === "executing" ||
|
|
3328
|
+
(batchState.phase as OrchBatchPhase) === "merging"
|
|
3329
|
+
) {
|
|
2824
3330
|
if (batchState.failedTasks > 0) {
|
|
2825
3331
|
// TP-031: Parity with engine.ts — default to "paused" so the batch is
|
|
2826
3332
|
// resumable without --force. "failed" is reserved for unrecoverable
|
|
@@ -2842,28 +3348,65 @@ export async function resumeOrchBatch(
|
|
|
2842
3348
|
// supervisor agent. Legacy engine fast-forward is removed — supervisor
|
|
2843
3349
|
// handles all non-manual integration after batch_complete event.
|
|
2844
3350
|
const mergedTaskCount = batchState.succeededTasks;
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
3351
|
+
// TP-195: hoist `batchState.phase` to a fresh local with the wide
|
|
3352
|
+
// `OrchBatchPhase` type. TypeScript's narrowing-on-property semantics
|
|
3353
|
+
// under `strict: false` carries assignments forward through the
|
|
3354
|
+
// function (visible in the `(batchState.phase as OrchBatchPhase) === ...`
|
|
3355
|
+
// pattern already used at lines ~3366/~3476 above), which here narrows
|
|
3356
|
+
// `batchState.phase` to a subtype that excludes `"completed"` and
|
|
3357
|
+
// `"failed"`. Hoisting to a typed local breaks the narrowing chain so
|
|
3358
|
+
// the comparisons typecheck without a per-call cast. Runtime
|
|
3359
|
+
// evaluation is identical.
|
|
3360
|
+
const phaseAtTerminal = batchState.phase as OrchBatchPhase;
|
|
3361
|
+
const isTerminalPhase = phaseAtTerminal === "completed" || phaseAtTerminal === "failed";
|
|
3362
|
+
if (
|
|
3363
|
+
isTerminalPhase &&
|
|
3364
|
+
!preserveWorktreesForResume &&
|
|
3365
|
+
batchState.orchBranch &&
|
|
3366
|
+
mergedTaskCount > 0
|
|
3367
|
+
) {
|
|
3368
|
+
if (
|
|
3369
|
+
orchConfig.orchestrator.integration === "supervised" ||
|
|
3370
|
+
orchConfig.orchestrator.integration === "auto"
|
|
3371
|
+
) {
|
|
2848
3372
|
// TP-043: Supervisor-managed integration modes. Defer to supervisor.
|
|
2849
|
-
execLog(
|
|
3373
|
+
execLog(
|
|
3374
|
+
"resume",
|
|
3375
|
+
batchState.batchId,
|
|
3376
|
+
`integration deferred to supervisor (mode: ${orchConfig.orchestrator.integration})`,
|
|
3377
|
+
);
|
|
2850
3378
|
} else {
|
|
2851
3379
|
// Manual mode (default): show integration guidance
|
|
2852
3380
|
onNotify(
|
|
2853
|
-
ORCH_MESSAGES.orchIntegrationManual(
|
|
3381
|
+
ORCH_MESSAGES.orchIntegrationManual(
|
|
3382
|
+
batchState.orchBranch,
|
|
3383
|
+
batchState.baseBranch,
|
|
3384
|
+
mergedTaskCount,
|
|
3385
|
+
),
|
|
2854
3386
|
"info",
|
|
2855
3387
|
);
|
|
2856
3388
|
}
|
|
2857
3389
|
}
|
|
2858
3390
|
|
|
2859
|
-
persistRuntimeState(
|
|
3391
|
+
persistRuntimeState(
|
|
3392
|
+
"batch-terminal",
|
|
3393
|
+
batchState,
|
|
3394
|
+
wavePlan,
|
|
3395
|
+
latestAllocatedLanes,
|
|
3396
|
+
allTaskOutcomes,
|
|
3397
|
+
discovery,
|
|
3398
|
+
stateRoot,
|
|
3399
|
+
);
|
|
2860
3400
|
|
|
2861
3401
|
// ── TP-076: Emit supervisor alert for batch completion ──────
|
|
2862
|
-
|
|
3402
|
+
// TP-195: reuse the hoisted-typed phase to avoid the same narrowing
|
|
3403
|
+
// artifact as the `isTerminalPhase` check above.
|
|
3404
|
+
if (phaseAtTerminal === "completed" || phaseAtTerminal === "failed") {
|
|
2863
3405
|
const batchDurationMs = batchState.endedAt ? batchState.endedAt - batchState.startedAt : 0;
|
|
2864
|
-
const durationStr =
|
|
2865
|
-
|
|
2866
|
-
|
|
3406
|
+
const durationStr =
|
|
3407
|
+
batchDurationMs > 0
|
|
3408
|
+
? `${Math.floor(batchDurationMs / 60000)}m ${Math.round((batchDurationMs % 60000) / 1000)}s`
|
|
3409
|
+
: "unknown";
|
|
2867
3410
|
if (batchState.phase === "completed" && batchState.failedTasks === 0) {
|
|
2868
3411
|
emitAlert({
|
|
2869
3412
|
category: "batch-complete",
|
|
@@ -2900,10 +3443,21 @@ export async function resumeOrchBatch(
|
|
|
2900
3443
|
|
|
2901
3444
|
// ── TP-031: Emit diagnostic reports (JSONL + markdown) ──
|
|
2902
3445
|
// Non-fatal: errors are logged but never crash batch finalization.
|
|
2903
|
-
emitDiagnosticReports(
|
|
3446
|
+
emitDiagnosticReports(
|
|
3447
|
+
assembleDiagnosticInput(
|
|
3448
|
+
orchConfig,
|
|
3449
|
+
batchState,
|
|
3450
|
+
wavePlan,
|
|
3451
|
+
latestAllocatedLanes,
|
|
3452
|
+
allTaskOutcomes,
|
|
3453
|
+
stateRoot,
|
|
3454
|
+
),
|
|
3455
|
+
);
|
|
2904
3456
|
|
|
2905
3457
|
if (batchState.phase === "paused" || batchState.phase === "stopped") {
|
|
2906
|
-
execLog("resume", batchState.batchId, "resumed batch ended in non-terminal state", {
|
|
3458
|
+
execLog("resume", batchState.batchId, "resumed batch ended in non-terminal state", {
|
|
3459
|
+
phase: batchState.phase,
|
|
3460
|
+
});
|
|
2907
3461
|
} else {
|
|
2908
3462
|
onNotify(
|
|
2909
3463
|
ORCH_MESSAGES.resumeComplete(
|
|
@@ -2928,9 +3482,7 @@ export async function resumeOrchBatch(
|
|
|
2928
3482
|
}
|
|
2929
3483
|
}
|
|
2930
3484
|
|
|
2931
|
-
|
|
2932
3485
|
// TP-043: attemptAutoIntegration is no longer called from engine.ts or resume.ts.
|
|
2933
3486
|
// Supervisor-managed integration ("supervised" and "auto" modes) is handled by
|
|
2934
3487
|
// the supervisor agent after batch_complete. The helper remains in merge.ts for
|
|
2935
3488
|
// use by the supervisor's integration flow.
|
|
2936
|
-
|