taskplane 0.29.2 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/gitignore-patterns.mjs +11 -8
- package/bin/rpc-wrapper.mjs +410 -357
- package/bin/taskplane.mjs +533 -250
- package/extensions/reviewer-extension.ts +17 -11
- package/extensions/taskplane/abort.ts +50 -18
- package/extensions/taskplane/agent-bridge-extension.ts +232 -105
- package/extensions/taskplane/agent-host.ts +224 -97
- package/extensions/taskplane/cleanup.ts +71 -42
- package/extensions/taskplane/config-loader.ts +142 -58
- package/extensions/taskplane/config-schema.ts +6 -13
- package/extensions/taskplane/config.ts +10 -2
- package/extensions/taskplane/diagnostic-reports.ts +59 -47
- package/extensions/taskplane/diagnostics.ts +13 -13
- package/extensions/taskplane/discovery.ts +35 -61
- package/extensions/taskplane/engine-worker.ts +53 -46
- package/extensions/taskplane/engine.ts +1760 -602
- package/extensions/taskplane/execution.ts +426 -206
- package/extensions/taskplane/extension.ts +1073 -598
- package/extensions/taskplane/formatting.ts +136 -124
- package/extensions/taskplane/git.ts +0 -2
- package/extensions/taskplane/lane-runner.ts +542 -311
- package/extensions/taskplane/mailbox.ts +57 -49
- package/extensions/taskplane/merge.ts +662 -383
- package/extensions/taskplane/messages.ts +109 -51
- package/extensions/taskplane/migrations.ts +1 -1
- package/extensions/taskplane/path-resolver.ts +8 -9
- package/extensions/taskplane/persistence.ts +425 -262
- package/extensions/taskplane/process-registry.ts +36 -7
- package/extensions/taskplane/quality-gate.ts +107 -55
- package/extensions/taskplane/resume.ts +774 -267
- package/extensions/taskplane/sessions.ts +1 -1
- package/extensions/taskplane/settings-tui.ts +505 -164
- package/extensions/taskplane/sidecar-telemetry.ts +25 -10
- package/extensions/taskplane/supervisor.ts +477 -270
- package/extensions/taskplane/task-executor-core.ts +178 -53
- package/extensions/taskplane/types.ts +186 -108
- package/extensions/taskplane/verification.ts +27 -22
- package/extensions/taskplane/waves.ts +59 -43
- package/extensions/taskplane/workspace.ts +14 -12
- package/extensions/taskplane/worktree.ts +218 -196
- package/package.json +14 -2
|
@@ -7,8 +7,21 @@ import { join } from "path";
|
|
|
7
7
|
|
|
8
8
|
import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-reports.ts";
|
|
9
9
|
import { runDiscovery } from "./discovery.ts";
|
|
10
|
-
import {
|
|
11
|
-
|
|
10
|
+
import {
|
|
11
|
+
executeOrchBatch,
|
|
12
|
+
resolveDisplayWaveNumber,
|
|
13
|
+
buildSpawnFailureAlertExtras,
|
|
14
|
+
} from "./engine.ts";
|
|
15
|
+
import {
|
|
16
|
+
buildReviewerEnv,
|
|
17
|
+
buildWorkerEnv,
|
|
18
|
+
buildWorkerExcludeEnv,
|
|
19
|
+
computeTransitiveDependents,
|
|
20
|
+
execLog,
|
|
21
|
+
executeLaneV2,
|
|
22
|
+
executeWave,
|
|
23
|
+
resolveCanonicalTaskPaths,
|
|
24
|
+
} from "./execution.ts";
|
|
12
25
|
import type { MonitorUpdateCallback, RuntimeBackend } from "./execution.ts";
|
|
13
26
|
import { selectRuntimeBackend } from "./engine.ts";
|
|
14
27
|
import { readRegistrySnapshot, isTerminalStatus, isProcessAlive } from "./process-registry.ts";
|
|
@@ -28,20 +41,74 @@ function terminateAliveV2Agents(stateRoot: string, batchId: string, sessionName:
|
|
|
28
41
|
try {
|
|
29
42
|
process.kill(manifest.pid, "SIGTERM");
|
|
30
43
|
execLog("resume", key, `terminated alive V2 agent (PID ${manifest.pid}) before re-execute`);
|
|
31
|
-
} catch {
|
|
44
|
+
} catch {
|
|
45
|
+
/* already dead */
|
|
46
|
+
}
|
|
32
47
|
}
|
|
33
48
|
}
|
|
34
49
|
}
|
|
35
50
|
import { getCurrentBranch, runGit } from "./git.ts";
|
|
36
51
|
import { mergeWaveByRepo } from "./merge.ts";
|
|
37
|
-
import {
|
|
52
|
+
import {
|
|
53
|
+
applyMergeRetryLoop,
|
|
54
|
+
computeCleanupGatePolicy,
|
|
55
|
+
computeMergeFailurePolicy,
|
|
56
|
+
extractFailedRepoId,
|
|
57
|
+
formatRepoMergeSummary,
|
|
58
|
+
ORCH_MESSAGES,
|
|
59
|
+
} from "./messages.ts";
|
|
38
60
|
import type { CleanupGateRepoFailure } from "./messages.ts";
|
|
39
61
|
import { resolveOperatorId } from "./naming.ts";
|
|
40
|
-
import {
|
|
41
|
-
|
|
42
|
-
|
|
62
|
+
import {
|
|
63
|
+
applyPartialProgressToOutcomes,
|
|
64
|
+
deleteBatchState,
|
|
65
|
+
hasTaskDoneMarker,
|
|
66
|
+
loadBatchState,
|
|
67
|
+
persistRuntimeState,
|
|
68
|
+
reconstructBatchStateFromRuntime,
|
|
69
|
+
saveBatchState,
|
|
70
|
+
seedPendingOutcomesForAllocatedLanes,
|
|
71
|
+
syncTaskOutcomesFromMonitor,
|
|
72
|
+
upsertTaskOutcome,
|
|
73
|
+
} from "./persistence.ts";
|
|
74
|
+
import {
|
|
75
|
+
buildBatchProgressSnapshot,
|
|
76
|
+
buildSupervisorSegmentFrontierSnapshot,
|
|
77
|
+
defaultResilienceState,
|
|
78
|
+
StateFileError,
|
|
79
|
+
} from "./types.ts";
|
|
80
|
+
import type {
|
|
81
|
+
AllocatedLane,
|
|
82
|
+
AllocatedTask,
|
|
83
|
+
LaneExecutionResult,
|
|
84
|
+
LaneTaskOutcome,
|
|
85
|
+
LaneTaskStatus,
|
|
86
|
+
MergeWaveResult,
|
|
87
|
+
OrchBatchPhase,
|
|
88
|
+
OrchBatchRuntimeState,
|
|
89
|
+
OrchestratorConfig,
|
|
90
|
+
ParsedTask,
|
|
91
|
+
PersistedBatchState,
|
|
92
|
+
PersistedLaneRecord,
|
|
93
|
+
PersistedSegmentRecord,
|
|
94
|
+
ReconciledTaskState,
|
|
95
|
+
ResumeEligibility,
|
|
96
|
+
ResumePoint,
|
|
97
|
+
TaskRunnerConfig,
|
|
98
|
+
WaveExecutionResult,
|
|
99
|
+
WorkspaceConfig,
|
|
100
|
+
} from "./types.ts";
|
|
43
101
|
import { buildDependencyGraph, resolveBaseBranch, resolveRepoRoot } from "./waves.ts";
|
|
44
|
-
import {
|
|
102
|
+
import {
|
|
103
|
+
deleteBranchBestEffort,
|
|
104
|
+
forceCleanupWorktree,
|
|
105
|
+
listWorktrees,
|
|
106
|
+
preserveFailedLaneProgress,
|
|
107
|
+
removeAllWorktrees,
|
|
108
|
+
removeWorktree,
|
|
109
|
+
safeResetWorktree,
|
|
110
|
+
sleepSync,
|
|
111
|
+
} from "./worktree.ts";
|
|
45
112
|
|
|
46
113
|
// ── Resume Repo Helpers ──────────────────────────────────────────────
|
|
47
114
|
|
|
@@ -245,7 +312,7 @@ export function collectDoneTaskIdsForResume(
|
|
|
245
312
|
doneTaskIds.add(task.taskId);
|
|
246
313
|
continue;
|
|
247
314
|
}
|
|
248
|
-
const laneRec = persistedState.lanes.find(l => l.taskIds.includes(task.taskId));
|
|
315
|
+
const laneRec = persistedState.lanes.find((l) => l.taskIds.includes(task.taskId));
|
|
249
316
|
if (laneRec?.worktreePath && task.taskFolder) {
|
|
250
317
|
const resolved = resolveCanonicalTaskPaths(
|
|
251
318
|
task.taskFolder,
|
|
@@ -281,7 +348,10 @@ export function collectDoneTaskIdsForResume(
|
|
|
281
348
|
* @param state - Persisted batch state to check
|
|
282
349
|
* @param force - When true, `stopped` and `failed` phases become eligible
|
|
283
350
|
*/
|
|
284
|
-
export function checkResumeEligibility(
|
|
351
|
+
export function checkResumeEligibility(
|
|
352
|
+
state: PersistedBatchState,
|
|
353
|
+
force: boolean = false,
|
|
354
|
+
): ResumeEligibility {
|
|
285
355
|
const { phase, batchId } = state;
|
|
286
356
|
|
|
287
357
|
switch (phase) {
|
|
@@ -394,7 +464,9 @@ interface SegmentFrontierResumeTaskState {
|
|
|
394
464
|
dependencyBySegmentId: Map<string, string[]>;
|
|
395
465
|
}
|
|
396
466
|
|
|
397
|
-
function classifySegmentStatus(
|
|
467
|
+
function classifySegmentStatus(
|
|
468
|
+
status: PersistedSegmentRecord["status"] | undefined,
|
|
469
|
+
): "completed" | "failed" | "in-flight" | "pending" {
|
|
398
470
|
if (status === "succeeded" || status === "skipped") return "completed";
|
|
399
471
|
if (status === "failed" || status === "stalled") return "failed";
|
|
400
472
|
if (status === "running") return "in-flight";
|
|
@@ -434,9 +506,13 @@ export function reconstructSegmentFrontier(
|
|
|
434
506
|
if (record) hasConcreteSegmentRecord = true;
|
|
435
507
|
const recordDeps = record?.dependsOnSegmentIds ?? [];
|
|
436
508
|
const fallbackDeps = idx > 0 ? [segmentIds[idx - 1]] : [];
|
|
437
|
-
const deps = (recordDeps.length > 0 ? recordDeps : fallbackDeps)
|
|
438
|
-
|
|
439
|
-
|
|
509
|
+
const deps = (recordDeps.length > 0 ? recordDeps : fallbackDeps).filter((dep) =>
|
|
510
|
+
segmentIds.includes(dep),
|
|
511
|
+
);
|
|
512
|
+
dependencyBySegmentId.set(
|
|
513
|
+
segmentId,
|
|
514
|
+
[...new Set(deps)].sort((a, b) => a.localeCompare(b)),
|
|
515
|
+
);
|
|
440
516
|
|
|
441
517
|
switch (classifySegmentStatus(record?.status)) {
|
|
442
518
|
case "completed":
|
|
@@ -457,13 +533,10 @@ export function reconstructSegmentFrontier(
|
|
|
457
533
|
const completedSet = new Set(completedSegmentIds);
|
|
458
534
|
const readyPending = pendingSegmentIds.filter((segmentId) => {
|
|
459
535
|
const deps = dependencyBySegmentId.get(segmentId) ?? [];
|
|
460
|
-
return deps.every(dep => completedSet.has(dep));
|
|
536
|
+
return deps.every((dep) => completedSet.has(dep));
|
|
461
537
|
});
|
|
462
538
|
|
|
463
|
-
const nextSegmentId = inFlightSegmentIds[0]
|
|
464
|
-
?? readyPending[0]
|
|
465
|
-
?? pendingSegmentIds[0]
|
|
466
|
-
?? null;
|
|
539
|
+
const nextSegmentId = inFlightSegmentIds[0] ?? readyPending[0] ?? pendingSegmentIds[0] ?? null;
|
|
467
540
|
const allSucceeded = segmentIds.every((segmentId) => {
|
|
468
541
|
const status = segmentRecordById.get(segmentId)?.status;
|
|
469
542
|
return status === "succeeded";
|
|
@@ -795,7 +868,12 @@ export function computeResumePoint(
|
|
|
795
868
|
case "skip":
|
|
796
869
|
if (task.liveStatus === "succeeded" || task.persistedStatus === "succeeded") {
|
|
797
870
|
completedTaskIds.push(task.taskId);
|
|
798
|
-
} else if (
|
|
871
|
+
} else if (
|
|
872
|
+
task.liveStatus === "failed" ||
|
|
873
|
+
task.liveStatus === "stalled" ||
|
|
874
|
+
task.persistedStatus === "failed" ||
|
|
875
|
+
task.persistedStatus === "stalled"
|
|
876
|
+
) {
|
|
799
877
|
failedTaskIds.push(task.taskId);
|
|
800
878
|
}
|
|
801
879
|
// persistedStatus === "skipped" → terminal but neither completed nor failed.
|
|
@@ -830,10 +908,12 @@ export function computeResumePoint(
|
|
|
830
908
|
const waveSegmentId = waveSegmentIdByTaskOccurrence.get(`${i}:${taskId}`);
|
|
831
909
|
if (waveSegmentId && segmentStatusBySegmentId.has(waveSegmentId)) {
|
|
832
910
|
const segmentStatus = segmentStatusBySegmentId.get(waveSegmentId)!;
|
|
833
|
-
return
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
911
|
+
return (
|
|
912
|
+
segmentStatus === "succeeded" ||
|
|
913
|
+
segmentStatus === "failed" ||
|
|
914
|
+
segmentStatus === "stalled" ||
|
|
915
|
+
segmentStatus === "skipped"
|
|
916
|
+
);
|
|
837
917
|
}
|
|
838
918
|
const reconciled = reconciledMap.get(taskId);
|
|
839
919
|
if (!reconciled) return false;
|
|
@@ -870,7 +950,11 @@ export function computeResumePoint(
|
|
|
870
950
|
const reconciled = reconciledMap.get(taskId);
|
|
871
951
|
if (!reconciled) return false;
|
|
872
952
|
if (reconciled.action === "mark-complete") return true;
|
|
873
|
-
if (
|
|
953
|
+
if (
|
|
954
|
+
reconciled.action === "skip" &&
|
|
955
|
+
(reconciled.liveStatus === "succeeded" || reconciled.persistedStatus === "succeeded")
|
|
956
|
+
)
|
|
957
|
+
return true;
|
|
874
958
|
return false;
|
|
875
959
|
});
|
|
876
960
|
|
|
@@ -935,7 +1019,6 @@ export function computeResumePoint(
|
|
|
935
1019
|
};
|
|
936
1020
|
}
|
|
937
1021
|
|
|
938
|
-
|
|
939
1022
|
// ── Pre-Resume Diagnostics ───────────────────────────────────────────
|
|
940
1023
|
|
|
941
1024
|
/**
|
|
@@ -1001,7 +1084,10 @@ export function runPreResumeDiagnostics(
|
|
|
1001
1084
|
const label = repoId ? `repo:${repoId}` : "default-repo";
|
|
1002
1085
|
|
|
1003
1086
|
if (persistedState.orchBranch) {
|
|
1004
|
-
const branchCheck = runGit(
|
|
1087
|
+
const branchCheck = runGit(
|
|
1088
|
+
["rev-parse", "--verify", `refs/heads/${persistedState.orchBranch}`],
|
|
1089
|
+
root,
|
|
1090
|
+
);
|
|
1005
1091
|
if (branchCheck.ok) {
|
|
1006
1092
|
checks.push({
|
|
1007
1093
|
check: `branch-consistency:${label}`,
|
|
@@ -1012,7 +1098,8 @@ export function runPreResumeDiagnostics(
|
|
|
1012
1098
|
checks.push({
|
|
1013
1099
|
check: `branch-consistency:${label}`,
|
|
1014
1100
|
passed: false,
|
|
1015
|
-
detail:
|
|
1101
|
+
detail:
|
|
1102
|
+
`Orch branch "${persistedState.orchBranch}" not found in ${label}. ` +
|
|
1016
1103
|
`The branch may have been deleted or the repo is in an inconsistent state.`,
|
|
1017
1104
|
});
|
|
1018
1105
|
}
|
|
@@ -1045,18 +1132,17 @@ export function runPreResumeDiagnostics(
|
|
|
1045
1132
|
}
|
|
1046
1133
|
}
|
|
1047
1134
|
|
|
1048
|
-
const failed = checks.filter(c => !c.passed);
|
|
1135
|
+
const failed = checks.filter((c) => !c.passed);
|
|
1049
1136
|
const passed = failed.length === 0;
|
|
1050
1137
|
|
|
1051
1138
|
const summary = passed
|
|
1052
1139
|
? `✅ Pre-resume diagnostics passed (${checks.length} checks)`
|
|
1053
1140
|
: `❌ Pre-resume diagnostics failed (${failed.length}/${checks.length} checks failed):\n` +
|
|
1054
|
-
|
|
1141
|
+
failed.map((c) => ` • ${c.check}: ${c.detail}`).join("\n");
|
|
1055
1142
|
|
|
1056
1143
|
return { passed, checks, summary };
|
|
1057
1144
|
}
|
|
1058
1145
|
|
|
1059
|
-
|
|
1060
1146
|
export async function resumeOrchBatch(
|
|
1061
1147
|
orchConfig: OrchestratorConfig,
|
|
1062
1148
|
runnerConfig: TaskRunnerConfig,
|
|
@@ -1108,10 +1194,7 @@ export async function resumeOrchBatch(
|
|
|
1108
1194
|
persistedState = loadBatchState(stateRoot);
|
|
1109
1195
|
} catch (err: unknown) {
|
|
1110
1196
|
if (err instanceof StateFileError) {
|
|
1111
|
-
onNotify(
|
|
1112
|
-
`❌ Cannot resume: ${err.message}`,
|
|
1113
|
-
"error",
|
|
1114
|
-
);
|
|
1197
|
+
onNotify(`❌ Cannot resume: ${err.message}`, "error");
|
|
1115
1198
|
// ── TP-040 R006: Reset phase on pre-execution early return ──
|
|
1116
1199
|
// The caller may have set batchState.phase = "launching" before
|
|
1117
1200
|
// calling this function. Since we're returning without starting
|
|
@@ -1124,10 +1207,7 @@ export async function resumeOrchBatch(
|
|
|
1124
1207
|
|
|
1125
1208
|
if (!persistedState) {
|
|
1126
1209
|
if (!force) {
|
|
1127
|
-
onNotify(
|
|
1128
|
-
ORCH_MESSAGES.resumeNoState(),
|
|
1129
|
-
"error",
|
|
1130
|
-
);
|
|
1210
|
+
onNotify(ORCH_MESSAGES.resumeNoState(), "error");
|
|
1131
1211
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
1132
1212
|
batchState.phase = "idle";
|
|
1133
1213
|
return;
|
|
@@ -1137,10 +1217,7 @@ export async function resumeOrchBatch(
|
|
|
1137
1217
|
// by `orch_abort()` even though `.pi/batch-state.json` is deleted).
|
|
1138
1218
|
const reconstruction = reconstructBatchStateFromRuntime(stateRoot);
|
|
1139
1219
|
if (!reconstruction.ok) {
|
|
1140
|
-
onNotify(
|
|
1141
|
-
ORCH_MESSAGES.resumeNoStateAfterAbort(reconstruction.error, null),
|
|
1142
|
-
"error",
|
|
1143
|
-
);
|
|
1220
|
+
onNotify(ORCH_MESSAGES.resumeNoStateAfterAbort(reconstruction.error, null), "error");
|
|
1144
1221
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
1145
1222
|
batchState.phase = "idle";
|
|
1146
1223
|
return;
|
|
@@ -1172,7 +1249,11 @@ export async function resumeOrchBatch(
|
|
|
1172
1249
|
const eligibility = checkResumeEligibility(persistedState, force);
|
|
1173
1250
|
if (!eligibility.eligible) {
|
|
1174
1251
|
onNotify(
|
|
1175
|
-
ORCH_MESSAGES.resumePhaseNotResumable(
|
|
1252
|
+
ORCH_MESSAGES.resumePhaseNotResumable(
|
|
1253
|
+
persistedState.batchId,
|
|
1254
|
+
persistedState.phase,
|
|
1255
|
+
eligibility.reason,
|
|
1256
|
+
),
|
|
1176
1257
|
"error",
|
|
1177
1258
|
);
|
|
1178
1259
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
@@ -1181,7 +1262,8 @@ export async function resumeOrchBatch(
|
|
|
1181
1262
|
}
|
|
1182
1263
|
|
|
1183
1264
|
// ── 2b. Force-resume: pre-resume diagnostics & state mutation ──
|
|
1184
|
-
const isForceResume =
|
|
1265
|
+
const isForceResume =
|
|
1266
|
+
force && (persistedState.phase === "stopped" || persistedState.phase === "failed");
|
|
1185
1267
|
if (isForceResume) {
|
|
1186
1268
|
onNotify(
|
|
1187
1269
|
ORCH_MESSAGES.forceResumeStarting(persistedState.batchId, persistedState.phase),
|
|
@@ -1193,10 +1275,7 @@ export async function resumeOrchBatch(
|
|
|
1193
1275
|
onNotify(diagnostics.summary, diagnostics.passed ? "info" : "error");
|
|
1194
1276
|
|
|
1195
1277
|
if (!diagnostics.passed) {
|
|
1196
|
-
onNotify(
|
|
1197
|
-
ORCH_MESSAGES.forceResumeDiagnosticsFailed(persistedState.batchId),
|
|
1198
|
-
"error",
|
|
1199
|
-
);
|
|
1278
|
+
onNotify(ORCH_MESSAGES.forceResumeDiagnosticsFailed(persistedState.batchId), "error");
|
|
1200
1279
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
1201
1280
|
batchState.phase = "idle";
|
|
1202
1281
|
return;
|
|
@@ -1206,17 +1285,19 @@ export async function resumeOrchBatch(
|
|
|
1206
1285
|
persistedState.resilience.resumeForced = true;
|
|
1207
1286
|
|
|
1208
1287
|
// Reset phase to paused so normal resume flow can proceed
|
|
1209
|
-
execLog(
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1288
|
+
execLog(
|
|
1289
|
+
"resume",
|
|
1290
|
+
persistedState.batchId,
|
|
1291
|
+
`force-resume: phase ${persistedState.phase} → paused`,
|
|
1292
|
+
{
|
|
1293
|
+
diagnosticChecks: diagnostics.checks.length,
|
|
1294
|
+
diagnosticsPassed: diagnostics.passed,
|
|
1295
|
+
},
|
|
1296
|
+
);
|
|
1213
1297
|
persistedState.phase = "paused";
|
|
1214
1298
|
}
|
|
1215
1299
|
|
|
1216
|
-
onNotify(
|
|
1217
|
-
ORCH_MESSAGES.resumeStarting(persistedState.batchId, persistedState.phase),
|
|
1218
|
-
"info",
|
|
1219
|
-
);
|
|
1300
|
+
onNotify(ORCH_MESSAGES.resumeStarting(persistedState.batchId, persistedState.phase), "info");
|
|
1220
1301
|
|
|
1221
1302
|
const segmentFrontierByTask = reconstructSegmentFrontier(persistedState);
|
|
1222
1303
|
if (segmentFrontierByTask.size > 0) {
|
|
@@ -1273,14 +1354,19 @@ export async function resumeOrchBatch(
|
|
|
1273
1354
|
// ── 3b. Detect existing worktrees ────────────────────────────
|
|
1274
1355
|
const existingWorktreeTaskIds = new Set<string>();
|
|
1275
1356
|
for (const task of persistedState.tasks) {
|
|
1276
|
-
const laneRecord = persistedState.lanes.find(l => l.taskIds.includes(task.taskId));
|
|
1357
|
+
const laneRecord = persistedState.lanes.find((l) => l.taskIds.includes(task.taskId));
|
|
1277
1358
|
if (laneRecord && laneRecord.worktreePath && existsSync(laneRecord.worktreePath)) {
|
|
1278
1359
|
existingWorktreeTaskIds.add(task.taskId);
|
|
1279
1360
|
}
|
|
1280
1361
|
}
|
|
1281
1362
|
|
|
1282
1363
|
// ── 4. Reconcile task states ─────────────────────────────────
|
|
1283
|
-
const reconciledTasks = reconcileTaskStates(
|
|
1364
|
+
const reconciledTasks = reconcileTaskStates(
|
|
1365
|
+
persistedState,
|
|
1366
|
+
aliveSessions,
|
|
1367
|
+
doneTaskIds,
|
|
1368
|
+
existingWorktreeTaskIds,
|
|
1369
|
+
);
|
|
1284
1370
|
|
|
1285
1371
|
// ── 4b. Clear stale session allocation for tasks reconciled as pending ──
|
|
1286
1372
|
// TP-037 (Bug #102b): Pending tasks that had a sessionName from a prior
|
|
@@ -1292,9 +1378,13 @@ export async function resumeOrchBatch(
|
|
|
1292
1378
|
const stalePendingTaskIds = new Set<string>();
|
|
1293
1379
|
for (const reconciled of reconciledTasks) {
|
|
1294
1380
|
if (reconciled.action === "pending") {
|
|
1295
|
-
const persistedTask = persistedState.tasks.find(t => t.taskId === reconciled.taskId);
|
|
1381
|
+
const persistedTask = persistedState.tasks.find((t) => t.taskId === reconciled.taskId);
|
|
1296
1382
|
if (persistedTask && persistedTask.sessionName) {
|
|
1297
|
-
execLog(
|
|
1383
|
+
execLog(
|
|
1384
|
+
"resume",
|
|
1385
|
+
persistedState.batchId,
|
|
1386
|
+
`clear-stale-session: ${reconciled.taskId} had stale session "${persistedTask.sessionName}" (lane ${persistedTask.laneNumber})`,
|
|
1387
|
+
);
|
|
1298
1388
|
stalePendingTaskIds.add(reconciled.taskId);
|
|
1299
1389
|
persistedTask.sessionName = "";
|
|
1300
1390
|
persistedTask.laneNumber = 0;
|
|
@@ -1305,7 +1395,7 @@ export async function resumeOrchBatch(
|
|
|
1305
1395
|
// (and subsequent serializeBatchState()) won't map them back to the old lane.
|
|
1306
1396
|
if (stalePendingTaskIds.size > 0) {
|
|
1307
1397
|
for (const lane of persistedState.lanes) {
|
|
1308
|
-
lane.taskIds = lane.taskIds.filter(id => !stalePendingTaskIds.has(id));
|
|
1398
|
+
lane.taskIds = lane.taskIds.filter((id) => !stalePendingTaskIds.has(id));
|
|
1309
1399
|
}
|
|
1310
1400
|
}
|
|
1311
1401
|
|
|
@@ -1329,22 +1419,16 @@ export async function resumeOrchBatch(
|
|
|
1329
1419
|
);
|
|
1330
1420
|
|
|
1331
1421
|
if (resumePoint.reconnectTaskIds.length > 0) {
|
|
1332
|
-
onNotify(
|
|
1333
|
-
ORCH_MESSAGES.resumeReconnecting(resumePoint.reconnectTaskIds.length),
|
|
1334
|
-
"info",
|
|
1335
|
-
);
|
|
1422
|
+
onNotify(ORCH_MESSAGES.resumeReconnecting(resumePoint.reconnectTaskIds.length), "info");
|
|
1336
1423
|
}
|
|
1337
1424
|
|
|
1338
1425
|
if (resumePoint.resumeWaveIndex > 0) {
|
|
1339
|
-
onNotify(
|
|
1340
|
-
ORCH_MESSAGES.resumeSkippedWaves(resumePoint.resumeWaveIndex),
|
|
1341
|
-
"info",
|
|
1342
|
-
);
|
|
1426
|
+
onNotify(ORCH_MESSAGES.resumeSkippedWaves(resumePoint.resumeWaveIndex), "info");
|
|
1343
1427
|
}
|
|
1344
1428
|
|
|
1345
1429
|
if (resumePoint.mergeRetryWaveIndexes.length > 0) {
|
|
1346
1430
|
onNotify(
|
|
1347
|
-
`🔀 ${resumePoint.mergeRetryWaveIndexes.length} wave(s) need merge retry: ${resumePoint.mergeRetryWaveIndexes.map(i => `W${i + 1}`).join(", ")}`,
|
|
1431
|
+
`🔀 ${resumePoint.mergeRetryWaveIndexes.length} wave(s) need merge retry: ${resumePoint.mergeRetryWaveIndexes.map((i) => `W${i + 1}`).join(", ")}`,
|
|
1348
1432
|
"warning",
|
|
1349
1433
|
);
|
|
1350
1434
|
}
|
|
@@ -1358,8 +1442,8 @@ export async function resumeOrchBatch(
|
|
|
1358
1442
|
if (!persistedState.orchBranch) {
|
|
1359
1443
|
onNotify(
|
|
1360
1444
|
`❌ Cannot resume batch ${persistedState.batchId}: persisted state has no orch branch. ` +
|
|
1361
|
-
|
|
1362
|
-
|
|
1445
|
+
`This batch was created before orch-branch routing was implemented. ` +
|
|
1446
|
+
`Use /orch-abort to clean up, then start a new batch.`,
|
|
1363
1447
|
"error",
|
|
1364
1448
|
);
|
|
1365
1449
|
// TP-040 R006: Reset phase on pre-execution early return
|
|
@@ -1380,7 +1464,9 @@ export async function resumeOrchBatch(
|
|
|
1380
1464
|
// TP-166: Restore task-level wave metadata for correct display.
|
|
1381
1465
|
// Normalize: fall back to totalWaves for pre-TP-166 state files.
|
|
1382
1466
|
batchState.taskLevelWaveCount = persistedState.taskLevelWaveCount ?? persistedState.totalWaves;
|
|
1383
|
-
batchState.roundToTaskWave = persistedState.roundToTaskWave
|
|
1467
|
+
batchState.roundToTaskWave = persistedState.roundToTaskWave
|
|
1468
|
+
? [...persistedState.roundToTaskWave]
|
|
1469
|
+
: undefined;
|
|
1384
1470
|
batchState.totalTasks = persistedState.totalTasks;
|
|
1385
1471
|
batchState.succeededTasks = resumePoint.completedTaskIds.length;
|
|
1386
1472
|
batchState.failedTasks = resumePoint.failedTaskIds.length;
|
|
@@ -1410,7 +1496,11 @@ export async function resumeOrchBatch(
|
|
|
1410
1496
|
}
|
|
1411
1497
|
if (uncountedBlocked > 0) {
|
|
1412
1498
|
batchState.blockedTasks += uncountedBlocked;
|
|
1413
|
-
execLog(
|
|
1499
|
+
execLog(
|
|
1500
|
+
"resume",
|
|
1501
|
+
persistedState.batchId,
|
|
1502
|
+
`blocked counter fix: ${uncountedBlocked} persisted-blocked task(s) in unvisited waves added to blockedTasks`,
|
|
1503
|
+
);
|
|
1414
1504
|
}
|
|
1415
1505
|
}
|
|
1416
1506
|
|
|
@@ -1453,7 +1543,8 @@ export async function resumeOrchBatch(
|
|
|
1453
1543
|
"warning",
|
|
1454
1544
|
);
|
|
1455
1545
|
} else {
|
|
1456
|
-
const errMsg =
|
|
1546
|
+
const errMsg =
|
|
1547
|
+
`Failed to re-create orch branch "${batchState.orchBranch}" in repo "${repoId}": ${createRes.stderr}. ` +
|
|
1457
1548
|
`Cannot resume without orch branch isolation.`;
|
|
1458
1549
|
execLog("resume", batchState.batchId, errMsg, {
|
|
1459
1550
|
orchBranch: batchState.orchBranch,
|
|
@@ -1501,11 +1592,10 @@ export async function resumeOrchBatch(
|
|
|
1501
1592
|
}
|
|
1502
1593
|
}
|
|
1503
1594
|
|
|
1504
|
-
|
|
1505
1595
|
// ── 8. Handle alive sessions (reconnect) ─────────────────────
|
|
1506
1596
|
// For tasks with alive sessions, we need to wait for them to complete.
|
|
1507
1597
|
// We poll each alive session's .DONE file.
|
|
1508
|
-
const reconnectTasks = reconciledTasks.filter(t => t.action === "reconnect");
|
|
1598
|
+
const reconnectTasks = reconciledTasks.filter((t) => t.action === "reconnect");
|
|
1509
1599
|
const reconnectFinalStatus = new Map<string, LaneTaskStatus>();
|
|
1510
1600
|
|
|
1511
1601
|
if (reconnectTasks.length > 0) {
|
|
@@ -1515,9 +1605,7 @@ export async function resumeOrchBatch(
|
|
|
1515
1605
|
if (!parsedTask) continue;
|
|
1516
1606
|
|
|
1517
1607
|
// Find the lane info from persisted state
|
|
1518
|
-
const laneRecord = persistedState.lanes.find(
|
|
1519
|
-
l => l.taskIds.includes(task.taskId),
|
|
1520
|
-
);
|
|
1608
|
+
const laneRecord = persistedState.lanes.find((l) => l.taskIds.includes(task.taskId));
|
|
1521
1609
|
if (!laneRecord) continue;
|
|
1522
1610
|
|
|
1523
1611
|
// Build a minimal AllocatedLane for polling
|
|
@@ -1552,12 +1640,20 @@ export async function resumeOrchBatch(
|
|
|
1552
1640
|
terminateAliveV2Agents(stateRoot, persistedState.batchId, laneRecord.laneSessionId);
|
|
1553
1641
|
try {
|
|
1554
1642
|
const laneResult = await executeLaneV2(
|
|
1555
|
-
lane,
|
|
1556
|
-
|
|
1557
|
-
|
|
1643
|
+
lane,
|
|
1644
|
+
orchConfig,
|
|
1645
|
+
laneRepoRoot,
|
|
1646
|
+
batchState.pauseSignal,
|
|
1647
|
+
workspaceRoot,
|
|
1648
|
+
!!workspaceConfig,
|
|
1649
|
+
{
|
|
1650
|
+
ORCH_BATCH_ID: batchState.batchId,
|
|
1651
|
+
...buildReviewerEnv(runnerConfig.reviewer),
|
|
1652
|
+
...buildWorkerExcludeEnv(runnerConfig.workerExcludeExtensions),
|
|
1653
|
+
},
|
|
1558
1654
|
emitAlert,
|
|
1559
1655
|
);
|
|
1560
|
-
const taskResult = laneResult.tasks.find(t => t.taskId === task.taskId);
|
|
1656
|
+
const taskResult = laneResult.tasks.find((t) => t.taskId === task.taskId);
|
|
1561
1657
|
if (taskResult?.status === "succeeded") {
|
|
1562
1658
|
reconnectFinalStatus.set(task.taskId, "succeeded");
|
|
1563
1659
|
completedTaskSet.add(task.taskId);
|
|
@@ -1577,13 +1673,17 @@ export async function resumeOrchBatch(
|
|
|
1577
1673
|
completedTaskSet.delete(task.taskId);
|
|
1578
1674
|
reconnectTaskSet.delete(task.taskId);
|
|
1579
1675
|
batchState.failedTasks++;
|
|
1580
|
-
execLog(
|
|
1676
|
+
execLog(
|
|
1677
|
+
"resume",
|
|
1678
|
+
task.taskId,
|
|
1679
|
+
`V2 reconnect error: ${err instanceof Error ? err.message : String(err)}`,
|
|
1680
|
+
);
|
|
1581
1681
|
}
|
|
1582
1682
|
}
|
|
1583
1683
|
}
|
|
1584
1684
|
|
|
1585
1685
|
// ── 8b. Handle re-execute tasks (dead session + existing worktree) ──
|
|
1586
|
-
const reExecuteTasks = reconciledTasks.filter(t => t.action === "re-execute");
|
|
1686
|
+
const reExecuteTasks = reconciledTasks.filter((t) => t.action === "re-execute");
|
|
1587
1687
|
const reExecuteFinalStatus = new Map<string, LaneTaskStatus>();
|
|
1588
1688
|
const reExecAllocatedLanes: AllocatedLane[] = [];
|
|
1589
1689
|
|
|
@@ -1597,9 +1697,7 @@ export async function resumeOrchBatch(
|
|
|
1597
1697
|
const parsedTask = discovery.pending.get(task.taskId);
|
|
1598
1698
|
if (!parsedTask) continue;
|
|
1599
1699
|
|
|
1600
|
-
const laneRecord = persistedState.lanes.find(
|
|
1601
|
-
l => l.taskIds.includes(task.taskId),
|
|
1602
|
-
);
|
|
1700
|
+
const laneRecord = persistedState.lanes.find((l) => l.taskIds.includes(task.taskId));
|
|
1603
1701
|
if (!laneRecord) continue;
|
|
1604
1702
|
|
|
1605
1703
|
const allocatedTask: AllocatedTask = {
|
|
@@ -1634,12 +1732,20 @@ export async function resumeOrchBatch(
|
|
|
1634
1732
|
// TP-112: Runtime V2 re-execution.
|
|
1635
1733
|
terminateAliveV2Agents(stateRoot, batchState.batchId, laneRecord.laneSessionId);
|
|
1636
1734
|
const laneResult = await executeLaneV2(
|
|
1637
|
-
lane,
|
|
1638
|
-
|
|
1639
|
-
|
|
1735
|
+
lane,
|
|
1736
|
+
orchConfig,
|
|
1737
|
+
reExecRepoRoot,
|
|
1738
|
+
batchState.pauseSignal,
|
|
1739
|
+
workspaceRoot,
|
|
1740
|
+
!!workspaceConfig,
|
|
1741
|
+
{
|
|
1742
|
+
ORCH_BATCH_ID: batchState.batchId,
|
|
1743
|
+
...buildReviewerEnv(runnerConfig.reviewer),
|
|
1744
|
+
...buildWorkerExcludeEnv(runnerConfig.workerExcludeExtensions),
|
|
1745
|
+
},
|
|
1640
1746
|
emitAlert,
|
|
1641
1747
|
);
|
|
1642
|
-
const taskResult = laneResult.tasks.find(t => t.taskId === task.taskId);
|
|
1748
|
+
const taskResult = laneResult.tasks.find((t) => t.taskId === task.taskId);
|
|
1643
1749
|
const pollResult: { status: LaneTaskStatus; exitReason: string; doneFileFound: boolean } = {
|
|
1644
1750
|
status: taskResult?.status ?? "failed",
|
|
1645
1751
|
exitReason: taskResult?.exitReason ?? "V2 re-execution completed",
|
|
@@ -1660,7 +1766,11 @@ export async function resumeOrchBatch(
|
|
|
1660
1766
|
completedTaskSet.delete(task.taskId);
|
|
1661
1767
|
reExecuteTaskSet.delete(task.taskId);
|
|
1662
1768
|
batchState.failedTasks++;
|
|
1663
|
-
execLog(
|
|
1769
|
+
execLog(
|
|
1770
|
+
"resume",
|
|
1771
|
+
task.taskId,
|
|
1772
|
+
`re-executed task ${pollResult.status}: ${pollResult.exitReason}`,
|
|
1773
|
+
);
|
|
1664
1774
|
}
|
|
1665
1775
|
} catch (err: unknown) {
|
|
1666
1776
|
reExecuteFinalStatus.set(task.taskId, "failed");
|
|
@@ -1683,16 +1793,13 @@ export async function resumeOrchBatch(
|
|
|
1683
1793
|
.map(([taskId]) => taskId);
|
|
1684
1794
|
|
|
1685
1795
|
if (succeededReExecTaskIds.length > 0) {
|
|
1686
|
-
onNotify(
|
|
1687
|
-
`🔀 Merging ${reExecAllocatedLanes.length} re-executed lane branch(es)...`,
|
|
1688
|
-
"info",
|
|
1689
|
-
);
|
|
1796
|
+
onNotify(`🔀 Merging ${reExecAllocatedLanes.length} re-executed lane branch(es)...`, "info");
|
|
1690
1797
|
|
|
1691
1798
|
// Build synthetic WaveExecutionResult for mergeWaveByRepo()
|
|
1692
|
-
const syntheticLaneResults: LaneExecutionResult[] = reExecAllocatedLanes.map(lane => ({
|
|
1799
|
+
const syntheticLaneResults: LaneExecutionResult[] = reExecAllocatedLanes.map((lane) => ({
|
|
1693
1800
|
laneNumber: lane.laneNumber,
|
|
1694
1801
|
laneId: lane.laneId,
|
|
1695
|
-
tasks: lane.tasks.map(t => ({
|
|
1802
|
+
tasks: lane.tasks.map((t) => ({
|
|
1696
1803
|
taskId: t.taskId,
|
|
1697
1804
|
status: "succeeded" as LaneTaskStatus,
|
|
1698
1805
|
startTime: Date.now(),
|
|
@@ -1758,7 +1865,10 @@ export async function resumeOrchBatch(
|
|
|
1758
1865
|
// Clean up merged branches (resolve per-lane repo root for workspace mode)
|
|
1759
1866
|
// TP-032 R006-3: Exclude verification_new_failure lanes from branch cleanup
|
|
1760
1867
|
for (const lr of reExecMergeResult.laneResults) {
|
|
1761
|
-
if (
|
|
1868
|
+
if (
|
|
1869
|
+
!lr.error &&
|
|
1870
|
+
(lr.result?.status === "SUCCESS" || lr.result?.status === "CONFLICT_RESOLVED")
|
|
1871
|
+
) {
|
|
1762
1872
|
const laneRepoRoot = resolveRepoRoot(lr.repoId, repoRoot, workspaceConfig);
|
|
1763
1873
|
deleteBranchBestEffort(lr.sourceBranch, laneRepoRoot);
|
|
1764
1874
|
}
|
|
@@ -1788,39 +1898,53 @@ export async function resumeOrchBatch(
|
|
|
1788
1898
|
// records with repo attribution (laneNumber, laneId, branch, repoId).
|
|
1789
1899
|
// Without this, the `resume-reconciliation` checkpoint would serialize
|
|
1790
1900
|
// empty lanes[], losing all lane context until a new wave allocates.
|
|
1791
|
-
let latestAllocatedLanes: AllocatedLane[] = reconstructAllocatedLanes(
|
|
1901
|
+
let latestAllocatedLanes: AllocatedLane[] = reconstructAllocatedLanes(
|
|
1902
|
+
persistedState.lanes,
|
|
1903
|
+
persistedState.tasks,
|
|
1904
|
+
);
|
|
1792
1905
|
|
|
1793
1906
|
// Track all repo roots encountered during execution (persisted + newly allocated).
|
|
1794
1907
|
// Used by inter-wave reset and terminal cleanup to cover repos introduced
|
|
1795
1908
|
// after resume starts (not present in persisted lanes).
|
|
1796
1909
|
// Initialized from collectRepoRoots() helper for parity with other callers.
|
|
1797
|
-
const encounteredRepoRoots = new Set(
|
|
1798
|
-
collectRepoRoots(persistedState, repoRoot, workspaceConfig),
|
|
1799
|
-
);
|
|
1910
|
+
const encounteredRepoRoots = new Set(collectRepoRoots(persistedState, repoRoot, workspaceConfig));
|
|
1800
1911
|
|
|
1801
1912
|
// Build outcomes from reconciled tasks
|
|
1802
1913
|
for (const task of reconciledTasks) {
|
|
1803
|
-
const persistedTask = persistedState.tasks.find(t => t.taskId === task.taskId);
|
|
1914
|
+
const persistedTask = persistedState.tasks.find((t) => t.taskId === task.taskId);
|
|
1804
1915
|
const reconnectStatus = reconnectFinalStatus.get(task.taskId);
|
|
1805
1916
|
const reExecuteStatus = reExecuteFinalStatus.get(task.taskId);
|
|
1806
|
-
const status =
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1917
|
+
const status =
|
|
1918
|
+
task.action === "reconnect"
|
|
1919
|
+
? reconnectStatus || "running"
|
|
1920
|
+
: task.action === "re-execute"
|
|
1921
|
+
? reExecuteStatus || "pending"
|
|
1922
|
+
: task.liveStatus;
|
|
1923
|
+
const isTerminal =
|
|
1924
|
+
status === "succeeded" || status === "failed" || status === "stalled" || status === "skipped";
|
|
1812
1925
|
allTaskOutcomes.push({
|
|
1813
1926
|
taskId: task.taskId,
|
|
1814
1927
|
status,
|
|
1815
1928
|
startTime: persistedTask?.startedAt ?? null,
|
|
1816
1929
|
endTime: isTerminal ? Date.now() : null,
|
|
1817
|
-
exitReason:
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1930
|
+
exitReason:
|
|
1931
|
+
task.action === "mark-complete"
|
|
1932
|
+
? ".DONE file found on resume"
|
|
1933
|
+
: task.action === "mark-failed"
|
|
1934
|
+
? "Session dead, no .DONE file, no worktree on resume"
|
|
1935
|
+
: task.action === "reconnect"
|
|
1936
|
+
? status === "succeeded"
|
|
1937
|
+
? "Reconnected task completed"
|
|
1938
|
+
: status === "failed"
|
|
1939
|
+
? "Reconnected task failed"
|
|
1940
|
+
: "Reconnected to alive session"
|
|
1941
|
+
: task.action === "re-execute"
|
|
1942
|
+
? status === "succeeded"
|
|
1943
|
+
? "Re-executed task completed"
|
|
1944
|
+
: status === "failed"
|
|
1945
|
+
? "Re-executed task failed"
|
|
1946
|
+
: "Re-executing in existing worktree"
|
|
1947
|
+
: (persistedTask?.exitReason ?? ""),
|
|
1824
1948
|
sessionName: persistedTask?.sessionName ?? "",
|
|
1825
1949
|
doneFileFound: status === "succeeded" ? true : task.doneFileFound,
|
|
1826
1950
|
laneNumber: persistedTask?.laneNumber,
|
|
@@ -1842,14 +1966,27 @@ export async function resumeOrchBatch(
|
|
|
1842
1966
|
batchState.blockedTaskIds.add(taskId);
|
|
1843
1967
|
}
|
|
1844
1968
|
if (reconciledBlocked.size > 0) {
|
|
1845
|
-
execLog(
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1969
|
+
execLog(
|
|
1970
|
+
"resume",
|
|
1971
|
+
batchState.batchId,
|
|
1972
|
+
`skip-dependents: ${reconciledBlocked.size} task(s) blocked from reconciled failures`,
|
|
1973
|
+
{
|
|
1974
|
+
blocked: [...reconciledBlocked].sort().join(","),
|
|
1975
|
+
sources: [...failedTaskSet].sort().join(","),
|
|
1976
|
+
},
|
|
1977
|
+
);
|
|
1849
1978
|
}
|
|
1850
1979
|
}
|
|
1851
1980
|
|
|
1852
|
-
persistRuntimeState(
|
|
1981
|
+
persistRuntimeState(
|
|
1982
|
+
"resume-reconciliation",
|
|
1983
|
+
batchState,
|
|
1984
|
+
wavePlan,
|
|
1985
|
+
latestAllocatedLanes,
|
|
1986
|
+
allTaskOutcomes,
|
|
1987
|
+
discovery ?? null,
|
|
1988
|
+
stateRoot,
|
|
1989
|
+
);
|
|
1853
1990
|
|
|
1854
1991
|
// ── 10. Continue wave execution ──────────────────────────────
|
|
1855
1992
|
// We need to execute remaining waves starting from resumeWaveIndex.
|
|
@@ -1868,33 +2005,53 @@ export async function resumeOrchBatch(
|
|
|
1868
2005
|
// Check pause signal
|
|
1869
2006
|
if (batchState.pauseSignal.paused) {
|
|
1870
2007
|
batchState.phase = "paused";
|
|
1871
|
-
persistRuntimeState(
|
|
1872
|
-
|
|
2008
|
+
persistRuntimeState(
|
|
2009
|
+
"pause-before-wave",
|
|
2010
|
+
batchState,
|
|
2011
|
+
wavePlan,
|
|
2012
|
+
latestAllocatedLanes,
|
|
2013
|
+
allTaskOutcomes,
|
|
2014
|
+
discovery,
|
|
2015
|
+
stateRoot,
|
|
2016
|
+
);
|
|
2017
|
+
const { displayWave: pauseWave } = resolveDisplayWaveNumber(
|
|
2018
|
+
waveIdx,
|
|
2019
|
+
roundToTaskWave,
|
|
2020
|
+
taskLevelWaveCount,
|
|
2021
|
+
);
|
|
1873
2022
|
onNotify(`⏸️ Batch paused before wave ${pauseWave}.`, "warning");
|
|
1874
2023
|
break;
|
|
1875
2024
|
}
|
|
1876
2025
|
|
|
1877
2026
|
batchState.currentWaveIndex = waveIdx;
|
|
1878
|
-
persistRuntimeState(
|
|
2027
|
+
persistRuntimeState(
|
|
2028
|
+
"wave-index-change",
|
|
2029
|
+
batchState,
|
|
2030
|
+
wavePlan,
|
|
2031
|
+
latestAllocatedLanes,
|
|
2032
|
+
allTaskOutcomes,
|
|
2033
|
+
discovery,
|
|
2034
|
+
stateRoot,
|
|
2035
|
+
);
|
|
1879
2036
|
|
|
1880
2037
|
// Get wave tasks, filtering out completed/failed/skipped/blocked ones.
|
|
1881
2038
|
// Persisted "skipped" tasks are terminal and must never be re-executed.
|
|
1882
2039
|
let waveTasks = wavePlan[waveIdx].filter(
|
|
1883
|
-
|
|
2040
|
+
(taskId) =>
|
|
2041
|
+
!completedTaskSet.has(taskId) &&
|
|
1884
2042
|
!failedTaskSet.has(taskId) &&
|
|
1885
2043
|
persistedStatusByTaskId.get(taskId) !== "skipped" &&
|
|
1886
2044
|
!batchState.blockedTaskIds.has(taskId),
|
|
1887
2045
|
);
|
|
1888
2046
|
|
|
1889
2047
|
// Also filter tasks where discovery doesn't have them as pending
|
|
1890
|
-
waveTasks = waveTasks.filter(taskId => discovery.pending.has(taskId));
|
|
2048
|
+
waveTasks = waveTasks.filter((taskId) => discovery.pending.has(taskId));
|
|
1891
2049
|
|
|
1892
2050
|
// Count only newly blocked tasks (not already persisted) to avoid double-counting.
|
|
1893
2051
|
// persistedState.blockedTaskIds were already counted in persistedState.blockedTasks
|
|
1894
2052
|
// which initialized batchState.blockedTasks.
|
|
1895
2053
|
const blockedInWave = wavePlan[waveIdx].filter(
|
|
1896
|
-
taskId => batchState.blockedTaskIds.has(taskId) &&
|
|
1897
|
-
!persistedBlockedTaskIds.has(taskId),
|
|
2054
|
+
(taskId) => batchState.blockedTaskIds.has(taskId) && !persistedBlockedTaskIds.has(taskId),
|
|
1898
2055
|
);
|
|
1899
2056
|
if (blockedInWave.length > 0) {
|
|
1900
2057
|
batchState.blockedTasks += blockedInWave.length;
|
|
@@ -1904,13 +2061,20 @@ export async function resumeOrchBatch(
|
|
|
1904
2061
|
// TP-037 Bug #102: Check if this wave needs merge retry.
|
|
1905
2062
|
// All tasks are terminal but the merge may have failed/been interrupted.
|
|
1906
2063
|
if (resumePoint.mergeRetryWaveIndexes.includes(waveIdx)) {
|
|
1907
|
-
execLog(
|
|
1908
|
-
|
|
2064
|
+
execLog(
|
|
2065
|
+
"resume",
|
|
2066
|
+
batchState.batchId,
|
|
2067
|
+
`wave ${waveIdx + 1}: all tasks done but merge needs retry`,
|
|
2068
|
+
);
|
|
2069
|
+
onNotify(
|
|
2070
|
+
`🔀 Wave ${resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave}: retrying merge (tasks already complete, merge was missing/failed)`,
|
|
2071
|
+
"info",
|
|
2072
|
+
);
|
|
1909
2073
|
|
|
1910
2074
|
// Reconstruct lanes for this wave from persisted state
|
|
1911
2075
|
const waveTaskIds = new Set(wavePlan[waveIdx]);
|
|
1912
|
-
const waveLaneRecords = persistedState.lanes.filter(
|
|
1913
|
-
lane
|
|
2076
|
+
const waveLaneRecords = persistedState.lanes.filter((lane) =>
|
|
2077
|
+
lane.taskIds.some((tid) => waveTaskIds.has(tid)),
|
|
1914
2078
|
);
|
|
1915
2079
|
const mergeRetryLanes = reconstructAllocatedLanes(waveLaneRecords, persistedState.tasks);
|
|
1916
2080
|
|
|
@@ -1918,18 +2082,14 @@ export async function resumeOrchBatch(
|
|
|
1918
2082
|
// Crucial for orch_force_merge: tasks intentionally marked "skipped" must
|
|
1919
2083
|
// remain skipped here (not failed), otherwise mixed-outcome detection would
|
|
1920
2084
|
// trigger again and block the forced merge recovery path.
|
|
1921
|
-
const succeededTaskIds = wavePlan[waveIdx].filter(
|
|
1922
|
-
taskId => completedTaskSet.has(taskId),
|
|
1923
|
-
);
|
|
2085
|
+
const succeededTaskIds = wavePlan[waveIdx].filter((taskId) => completedTaskSet.has(taskId));
|
|
1924
2086
|
const skippedTaskIds = wavePlan[waveIdx].filter(
|
|
1925
|
-
taskId => persistedStatusByTaskId.get(taskId) === "skipped",
|
|
1926
|
-
);
|
|
1927
|
-
const failedTaskIds = wavePlan[waveIdx].filter(
|
|
1928
|
-
taskId => {
|
|
1929
|
-
const status = persistedStatusByTaskId.get(taskId);
|
|
1930
|
-
return status === "failed" || status === "stalled";
|
|
1931
|
-
},
|
|
2087
|
+
(taskId) => persistedStatusByTaskId.get(taskId) === "skipped",
|
|
1932
2088
|
);
|
|
2089
|
+
const failedTaskIds = wavePlan[waveIdx].filter((taskId) => {
|
|
2090
|
+
const status = persistedStatusByTaskId.get(taskId);
|
|
2091
|
+
return status === "failed" || status === "stalled";
|
|
2092
|
+
});
|
|
1933
2093
|
|
|
1934
2094
|
const syntheticLaneResults: LaneExecutionResult[] = mergeRetryLanes.map((lane) => {
|
|
1935
2095
|
const laneTasks = lane.tasks.map((t) => {
|
|
@@ -1953,10 +2113,13 @@ export async function resumeOrchBatch(
|
|
|
1953
2113
|
startTime: Date.now(),
|
|
1954
2114
|
endTime: Date.now(),
|
|
1955
2115
|
exitReason:
|
|
1956
|
-
status === "succeeded"
|
|
1957
|
-
|
|
1958
|
-
: status === "
|
|
1959
|
-
|
|
2116
|
+
status === "succeeded"
|
|
2117
|
+
? "Task completed (merge retry)"
|
|
2118
|
+
: status === "skipped"
|
|
2119
|
+
? "Task skipped (merge retry)"
|
|
2120
|
+
: status === "stalled"
|
|
2121
|
+
? "Task stalled (merge retry)"
|
|
2122
|
+
: "Task failed (merge retry)",
|
|
1960
2123
|
sessionName: lane.laneSessionId,
|
|
1961
2124
|
doneFileFound: status === "succeeded",
|
|
1962
2125
|
laneNumber: lane.laneNumber,
|
|
@@ -1968,7 +2131,9 @@ export async function resumeOrchBatch(
|
|
|
1968
2131
|
);
|
|
1969
2132
|
const laneHasSucceeded = laneTasks.some((t) => t.status === "succeeded");
|
|
1970
2133
|
const overallStatus = laneHasHardFailure
|
|
1971
|
-
?
|
|
2134
|
+
? laneHasSucceeded
|
|
2135
|
+
? "partial"
|
|
2136
|
+
: "failed"
|
|
1972
2137
|
: "succeeded";
|
|
1973
2138
|
|
|
1974
2139
|
return {
|
|
@@ -1999,7 +2164,15 @@ export async function resumeOrchBatch(
|
|
|
1999
2164
|
};
|
|
2000
2165
|
|
|
2001
2166
|
batchState.phase = "merging";
|
|
2002
|
-
persistRuntimeState(
|
|
2167
|
+
persistRuntimeState(
|
|
2168
|
+
"merge-retry-start",
|
|
2169
|
+
batchState,
|
|
2170
|
+
wavePlan,
|
|
2171
|
+
latestAllocatedLanes,
|
|
2172
|
+
allTaskOutcomes,
|
|
2173
|
+
discovery,
|
|
2174
|
+
stateRoot,
|
|
2175
|
+
);
|
|
2003
2176
|
|
|
2004
2177
|
const mergeRetryResult = await mergeWaveByRepo(
|
|
2005
2178
|
mergeRetryLanes,
|
|
@@ -2020,10 +2193,16 @@ export async function resumeOrchBatch(
|
|
|
2020
2193
|
batchState.mergeResults.push(mergeRetryResult);
|
|
2021
2194
|
|
|
2022
2195
|
if (mergeRetryResult.status === "succeeded") {
|
|
2023
|
-
onNotify(
|
|
2196
|
+
onNotify(
|
|
2197
|
+
`✅ Wave ${resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave} merge retry succeeded`,
|
|
2198
|
+
"info",
|
|
2199
|
+
);
|
|
2024
2200
|
// Clean up merged branches
|
|
2025
2201
|
for (const lr of mergeRetryResult.laneResults) {
|
|
2026
|
-
if (
|
|
2202
|
+
if (
|
|
2203
|
+
!lr.error &&
|
|
2204
|
+
(lr.result?.status === "SUCCESS" || lr.result?.status === "CONFLICT_RESOLVED")
|
|
2205
|
+
) {
|
|
2027
2206
|
const laneRepoRoot = resolveRepoRoot(lr.repoId, repoRoot, workspaceConfig);
|
|
2028
2207
|
deleteBranchBestEffort(lr.sourceBranch, laneRepoRoot);
|
|
2029
2208
|
}
|
|
@@ -2035,27 +2214,61 @@ export async function resumeOrchBatch(
|
|
|
2035
2214
|
);
|
|
2036
2215
|
// Apply merge failure policy (same as normal wave merge failure)
|
|
2037
2216
|
const policyResult = computeMergeFailurePolicy(mergeRetryResult, waveIdx, orchConfig);
|
|
2038
|
-
execLog(
|
|
2217
|
+
execLog(
|
|
2218
|
+
"batch",
|
|
2219
|
+
batchState.batchId,
|
|
2220
|
+
`merge retry failure — applying ${policyResult.policy} policy`,
|
|
2221
|
+
policyResult.logDetails,
|
|
2222
|
+
);
|
|
2039
2223
|
batchState.phase = policyResult.targetPhase;
|
|
2040
2224
|
batchState.errors.push(policyResult.errorMessage);
|
|
2041
|
-
persistRuntimeState(
|
|
2225
|
+
persistRuntimeState(
|
|
2226
|
+
policyResult.persistTrigger,
|
|
2227
|
+
batchState,
|
|
2228
|
+
wavePlan,
|
|
2229
|
+
latestAllocatedLanes,
|
|
2230
|
+
allTaskOutcomes,
|
|
2231
|
+
discovery,
|
|
2232
|
+
stateRoot,
|
|
2233
|
+
);
|
|
2042
2234
|
onNotify(policyResult.notifyMessage, policyResult.notifyLevel);
|
|
2043
2235
|
preserveWorktreesForResume = true;
|
|
2044
2236
|
break;
|
|
2045
2237
|
}
|
|
2046
2238
|
|
|
2047
2239
|
batchState.phase = "executing";
|
|
2048
|
-
persistRuntimeState(
|
|
2240
|
+
persistRuntimeState(
|
|
2241
|
+
"merge-retry-complete",
|
|
2242
|
+
batchState,
|
|
2243
|
+
wavePlan,
|
|
2244
|
+
latestAllocatedLanes,
|
|
2245
|
+
allTaskOutcomes,
|
|
2246
|
+
discovery,
|
|
2247
|
+
stateRoot,
|
|
2248
|
+
);
|
|
2049
2249
|
} else {
|
|
2050
|
-
execLog(
|
|
2250
|
+
execLog(
|
|
2251
|
+
"resume",
|
|
2252
|
+
batchState.batchId,
|
|
2253
|
+
`wave ${waveIdx + 1}: no tasks to execute (all completed/blocked)`,
|
|
2254
|
+
);
|
|
2051
2255
|
}
|
|
2052
2256
|
continue;
|
|
2053
2257
|
}
|
|
2054
2258
|
|
|
2055
2259
|
{
|
|
2056
|
-
const { displayWave, displayTotal } = resolveDisplayWaveNumber(
|
|
2260
|
+
const { displayWave, displayTotal } = resolveDisplayWaveNumber(
|
|
2261
|
+
waveIdx,
|
|
2262
|
+
roundToTaskWave,
|
|
2263
|
+
taskLevelWaveCount,
|
|
2264
|
+
);
|
|
2057
2265
|
onNotify(
|
|
2058
|
-
ORCH_MESSAGES.orchWaveStart(
|
|
2266
|
+
ORCH_MESSAGES.orchWaveStart(
|
|
2267
|
+
displayWave,
|
|
2268
|
+
displayTotal,
|
|
2269
|
+
waveTasks.length,
|
|
2270
|
+
Math.min(waveTasks.length, orchConfig.orchestrator.max_lanes),
|
|
2271
|
+
),
|
|
2059
2272
|
"info",
|
|
2060
2273
|
);
|
|
2061
2274
|
}
|
|
@@ -2063,7 +2276,15 @@ export async function resumeOrchBatch(
|
|
|
2063
2276
|
const handleResumeMonitorUpdate: MonitorUpdateCallback = (monitorState) => {
|
|
2064
2277
|
const changed = syncTaskOutcomesFromMonitor(monitorState, allTaskOutcomes);
|
|
2065
2278
|
if (changed) {
|
|
2066
|
-
persistRuntimeState(
|
|
2279
|
+
persistRuntimeState(
|
|
2280
|
+
"task-transition",
|
|
2281
|
+
batchState,
|
|
2282
|
+
wavePlan,
|
|
2283
|
+
latestAllocatedLanes,
|
|
2284
|
+
allTaskOutcomes,
|
|
2285
|
+
discovery,
|
|
2286
|
+
stateRoot,
|
|
2287
|
+
);
|
|
2067
2288
|
}
|
|
2068
2289
|
onMonitorUpdate?.(monitorState);
|
|
2069
2290
|
};
|
|
@@ -2088,7 +2309,15 @@ export async function resumeOrchBatch(
|
|
|
2088
2309
|
encounteredRepoRoots.add(resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig));
|
|
2089
2310
|
}
|
|
2090
2311
|
if (seedPendingOutcomesForAllocatedLanes(lanes, allTaskOutcomes)) {
|
|
2091
|
-
persistRuntimeState(
|
|
2312
|
+
persistRuntimeState(
|
|
2313
|
+
"wave-lanes-allocated",
|
|
2314
|
+
batchState,
|
|
2315
|
+
wavePlan,
|
|
2316
|
+
latestAllocatedLanes,
|
|
2317
|
+
allTaskOutcomes,
|
|
2318
|
+
discovery,
|
|
2319
|
+
stateRoot,
|
|
2320
|
+
);
|
|
2092
2321
|
}
|
|
2093
2322
|
},
|
|
2094
2323
|
workspaceConfig,
|
|
@@ -2135,9 +2364,16 @@ export async function resumeOrchBatch(
|
|
|
2135
2364
|
|
|
2136
2365
|
// ── TP-076: Emit supervisor alerts for task failures ────
|
|
2137
2366
|
for (const taskId of waveResult.failedTaskIds) {
|
|
2138
|
-
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
2139
|
-
const laneForTask = latestAllocatedLanes.find(l => l.tasks.some(t => t.taskId === taskId));
|
|
2140
|
-
|
|
2367
|
+
const outcome = allTaskOutcomes.find((o) => o.taskId === taskId);
|
|
2368
|
+
const laneForTask = latestAllocatedLanes.find((l) => l.tasks.some((t) => t.taskId === taskId));
|
|
2369
|
+
// TP-195: corrected the lookup to the real source of segment
|
|
2370
|
+
// metadata. `batchState.tasks` does not exist on
|
|
2371
|
+
// `OrchBatchRuntimeState` (it's on `PersistedBatchState`); the
|
|
2372
|
+
// previous read would have thrown `undefined.find is not a
|
|
2373
|
+
// function` if hit at runtime. The allocated lane carries the
|
|
2374
|
+
// `ParsedTask` payload via `AllocatedTask.task`, which has
|
|
2375
|
+
// `segmentIds`/`activeSegmentId` already populated by discovery.
|
|
2376
|
+
const taskRecord = laneForTask?.tasks.find((t) => t.taskId === taskId)?.task;
|
|
2141
2377
|
const exitReason = outcome?.exitReason || "unknown";
|
|
2142
2378
|
const hasPartialProgress = (outcome?.partialProgressCommits ?? 0) > 0;
|
|
2143
2379
|
const segmentFrontier = buildSupervisorSegmentFrontierSnapshot(
|
|
@@ -2147,12 +2383,14 @@ export async function resumeOrchBatch(
|
|
|
2147
2383
|
batchState.segments,
|
|
2148
2384
|
outcome?.segmentId,
|
|
2149
2385
|
);
|
|
2150
|
-
const segmentId =
|
|
2151
|
-
??
|
|
2152
|
-
|
|
2153
|
-
??
|
|
2386
|
+
const segmentId =
|
|
2387
|
+
outcome?.segmentId ??
|
|
2388
|
+
taskRecord?.activeSegmentId ??
|
|
2389
|
+
segmentFrontier?.activeSegmentId ??
|
|
2390
|
+
undefined;
|
|
2154
2391
|
const repoId = segmentId
|
|
2155
|
-
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ??
|
|
2392
|
+
? (segmentFrontier?.segments.find((segment) => segment.segmentId === segmentId)?.repoId ??
|
|
2393
|
+
laneForTask?.repoId)
|
|
2156
2394
|
: laneForTask?.repoId;
|
|
2157
2395
|
const segmentSummary = segmentId
|
|
2158
2396
|
? ` Segment: ${segmentId}${repoId ? ` (repo: ${repoId})` : ""}\n`
|
|
@@ -2197,11 +2435,23 @@ export async function resumeOrchBatch(
|
|
|
2197
2435
|
});
|
|
2198
2436
|
}
|
|
2199
2437
|
|
|
2200
|
-
persistRuntimeState(
|
|
2438
|
+
persistRuntimeState(
|
|
2439
|
+
"wave-execution-complete",
|
|
2440
|
+
batchState,
|
|
2441
|
+
wavePlan,
|
|
2442
|
+
latestAllocatedLanes,
|
|
2443
|
+
allTaskOutcomes,
|
|
2444
|
+
discovery,
|
|
2445
|
+
stateRoot,
|
|
2446
|
+
);
|
|
2201
2447
|
|
|
2202
2448
|
const elapsedSec = Math.round((waveResult.endedAt - waveResult.startedAt) / 1000);
|
|
2203
2449
|
{
|
|
2204
|
-
const { displayWave: completeDisplayWave } = resolveDisplayWaveNumber(
|
|
2450
|
+
const { displayWave: completeDisplayWave } = resolveDisplayWaveNumber(
|
|
2451
|
+
waveIdx,
|
|
2452
|
+
roundToTaskWave,
|
|
2453
|
+
taskLevelWaveCount,
|
|
2454
|
+
);
|
|
2205
2455
|
onNotify(
|
|
2206
2456
|
ORCH_MESSAGES.orchWaveComplete(
|
|
2207
2457
|
completeDisplayWave,
|
|
@@ -2218,13 +2468,29 @@ export async function resumeOrchBatch(
|
|
|
2218
2468
|
if (waveResult.stoppedEarly) {
|
|
2219
2469
|
if (waveResult.policyApplied === "stop-all") {
|
|
2220
2470
|
batchState.phase = "stopped";
|
|
2221
|
-
persistRuntimeState(
|
|
2471
|
+
persistRuntimeState(
|
|
2472
|
+
"stop-all",
|
|
2473
|
+
batchState,
|
|
2474
|
+
wavePlan,
|
|
2475
|
+
latestAllocatedLanes,
|
|
2476
|
+
allTaskOutcomes,
|
|
2477
|
+
discovery,
|
|
2478
|
+
stateRoot,
|
|
2479
|
+
);
|
|
2222
2480
|
onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-all"), "error");
|
|
2223
2481
|
break;
|
|
2224
2482
|
}
|
|
2225
2483
|
if (waveResult.policyApplied === "stop-wave") {
|
|
2226
2484
|
batchState.phase = "stopped";
|
|
2227
|
-
persistRuntimeState(
|
|
2485
|
+
persistRuntimeState(
|
|
2486
|
+
"stop-wave",
|
|
2487
|
+
batchState,
|
|
2488
|
+
wavePlan,
|
|
2489
|
+
latestAllocatedLanes,
|
|
2490
|
+
allTaskOutcomes,
|
|
2491
|
+
discovery,
|
|
2492
|
+
stateRoot,
|
|
2493
|
+
);
|
|
2228
2494
|
onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-wave"), "error");
|
|
2229
2495
|
break;
|
|
2230
2496
|
}
|
|
@@ -2237,29 +2503,41 @@ export async function resumeOrchBatch(
|
|
|
2237
2503
|
for (const lr of waveResult.laneResults) {
|
|
2238
2504
|
laneOutcomeByNumber.set(lr.laneNumber, lr);
|
|
2239
2505
|
}
|
|
2240
|
-
const mixedOutcomeLanes = waveResult.laneResults.filter(lr => {
|
|
2241
|
-
const hasSucceeded = lr.tasks.some(t => t.status === "succeeded");
|
|
2242
|
-
const hasHardFailure = lr.tasks.some(
|
|
2243
|
-
t => t.status === "failed" || t.status === "stalled",
|
|
2244
|
-
);
|
|
2506
|
+
const mixedOutcomeLanes = waveResult.laneResults.filter((lr) => {
|
|
2507
|
+
const hasSucceeded = lr.tasks.some((t) => t.status === "succeeded");
|
|
2508
|
+
const hasHardFailure = lr.tasks.some((t) => t.status === "failed" || t.status === "stalled");
|
|
2245
2509
|
return hasSucceeded && hasHardFailure;
|
|
2246
2510
|
});
|
|
2247
2511
|
|
|
2248
2512
|
if (waveResult.succeededTaskIds.length > 0) {
|
|
2249
|
-
const mergeableLaneCount = waveResult.allocatedLanes.filter(lane => {
|
|
2513
|
+
const mergeableLaneCount = waveResult.allocatedLanes.filter((lane) => {
|
|
2250
2514
|
const outcome = laneOutcomeByNumber.get(lane.laneNumber);
|
|
2251
2515
|
if (!outcome) return false;
|
|
2252
|
-
const hasSucceeded = outcome.tasks.some(t => t.status === "succeeded");
|
|
2516
|
+
const hasSucceeded = outcome.tasks.some((t) => t.status === "succeeded");
|
|
2253
2517
|
const hasHardFailure = outcome.tasks.some(
|
|
2254
|
-
t => t.status === "failed" || t.status === "stalled",
|
|
2518
|
+
(t) => t.status === "failed" || t.status === "stalled",
|
|
2255
2519
|
);
|
|
2256
2520
|
return hasSucceeded && !hasHardFailure;
|
|
2257
2521
|
}).length;
|
|
2258
2522
|
|
|
2259
2523
|
if (mergeableLaneCount > 0) {
|
|
2260
2524
|
batchState.phase = "merging";
|
|
2261
|
-
persistRuntimeState(
|
|
2262
|
-
|
|
2525
|
+
persistRuntimeState(
|
|
2526
|
+
"merge-start",
|
|
2527
|
+
batchState,
|
|
2528
|
+
wavePlan,
|
|
2529
|
+
latestAllocatedLanes,
|
|
2530
|
+
allTaskOutcomes,
|
|
2531
|
+
discovery,
|
|
2532
|
+
stateRoot,
|
|
2533
|
+
);
|
|
2534
|
+
onNotify(
|
|
2535
|
+
ORCH_MESSAGES.orchMergeStart(
|
|
2536
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2537
|
+
mergeableLaneCount,
|
|
2538
|
+
),
|
|
2539
|
+
"info",
|
|
2540
|
+
);
|
|
2263
2541
|
|
|
2264
2542
|
mergeResult = await mergeWaveByRepo(
|
|
2265
2543
|
waveResult.allocatedLanes,
|
|
@@ -2287,35 +2565,65 @@ export async function resumeOrchBatch(
|
|
|
2287
2565
|
if (lr.error) {
|
|
2288
2566
|
onNotify(ORCH_MESSAGES.orchMergeLaneFailed(lr.laneNumber, lr.error), "error");
|
|
2289
2567
|
} else if (lr.result?.status === "SUCCESS") {
|
|
2290
|
-
onNotify(
|
|
2568
|
+
onNotify(
|
|
2569
|
+
ORCH_MESSAGES.orchMergeLaneSuccess(lr.laneNumber, lr.result.merge_commit, durationSec),
|
|
2570
|
+
"info",
|
|
2571
|
+
);
|
|
2291
2572
|
} else if (lr.result?.status === "CONFLICT_RESOLVED") {
|
|
2292
|
-
onNotify(
|
|
2293
|
-
|
|
2573
|
+
onNotify(
|
|
2574
|
+
ORCH_MESSAGES.orchMergeLaneConflictResolved(
|
|
2575
|
+
lr.laneNumber,
|
|
2576
|
+
lr.result.conflicts.length,
|
|
2577
|
+
durationSec,
|
|
2578
|
+
),
|
|
2579
|
+
"info",
|
|
2580
|
+
);
|
|
2581
|
+
} else if (
|
|
2582
|
+
lr.result?.status === "CONFLICT_UNRESOLVED" ||
|
|
2583
|
+
lr.result?.status === "BUILD_FAILURE"
|
|
2584
|
+
) {
|
|
2294
2585
|
onNotify(ORCH_MESSAGES.orchMergeLaneFailed(lr.laneNumber, lr.result.status), "error");
|
|
2295
2586
|
}
|
|
2296
2587
|
}
|
|
2297
2588
|
|
|
2298
2589
|
if (mixedOutcomeLanes.length > 0) {
|
|
2299
|
-
const mixedIds = mixedOutcomeLanes.map(l => `lane-${l.laneNumber}`).join(", ");
|
|
2590
|
+
const mixedIds = mixedOutcomeLanes.map((l) => `lane-${l.laneNumber}`).join(", ");
|
|
2300
2591
|
const failureReason =
|
|
2301
2592
|
`Lane(s) ${mixedIds} contain both succeeded and failed tasks. ` +
|
|
2302
2593
|
`Automatic partial-branch merge is disabled to avoid dropping succeeded commits.`;
|
|
2303
|
-
mergeResult = {
|
|
2594
|
+
mergeResult = {
|
|
2595
|
+
...mergeResult,
|
|
2596
|
+
status: "partial",
|
|
2597
|
+
failedLane: mixedOutcomeLanes[0].laneNumber,
|
|
2598
|
+
failureReason,
|
|
2599
|
+
};
|
|
2304
2600
|
// Update the already-pushed reference so persisted state reflects "partial"
|
|
2305
2601
|
batchState.mergeResults[batchState.mergeResults.length - 1] = mergeResult;
|
|
2306
2602
|
}
|
|
2307
2603
|
|
|
2308
2604
|
// TP-032 R006-3: Exclude verification_new_failure lanes from success count
|
|
2309
2605
|
const mergedCount = mergeResult.laneResults.filter(
|
|
2310
|
-
r =>
|
|
2606
|
+
(r) =>
|
|
2607
|
+
!r.error && (r.result?.status === "SUCCESS" || r.result?.status === "CONFLICT_RESOLVED"),
|
|
2311
2608
|
).length;
|
|
2312
2609
|
const mergeTotalSec = Math.round(mergeResult.totalDurationMs / 1000);
|
|
2313
2610
|
|
|
2314
2611
|
if (mergeResult.status === "succeeded") {
|
|
2315
|
-
onNotify(
|
|
2612
|
+
onNotify(
|
|
2613
|
+
ORCH_MESSAGES.orchMergeComplete(
|
|
2614
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2615
|
+
mergedCount,
|
|
2616
|
+
mergeTotalSec,
|
|
2617
|
+
),
|
|
2618
|
+
"info",
|
|
2619
|
+
);
|
|
2316
2620
|
} else {
|
|
2317
2621
|
onNotify(
|
|
2318
|
-
ORCH_MESSAGES.orchMergeFailed(
|
|
2622
|
+
ORCH_MESSAGES.orchMergeFailed(
|
|
2623
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2624
|
+
mergeResult.failedLane ?? 0,
|
|
2625
|
+
mergeResult.failureReason || "unknown",
|
|
2626
|
+
),
|
|
2319
2627
|
"error",
|
|
2320
2628
|
);
|
|
2321
2629
|
|
|
@@ -2329,9 +2637,17 @@ export async function resumeOrchBatch(
|
|
|
2329
2637
|
}
|
|
2330
2638
|
|
|
2331
2639
|
batchState.phase = "executing";
|
|
2332
|
-
persistRuntimeState(
|
|
2640
|
+
persistRuntimeState(
|
|
2641
|
+
"merge-complete",
|
|
2642
|
+
batchState,
|
|
2643
|
+
wavePlan,
|
|
2644
|
+
latestAllocatedLanes,
|
|
2645
|
+
allTaskOutcomes,
|
|
2646
|
+
discovery,
|
|
2647
|
+
stateRoot,
|
|
2648
|
+
);
|
|
2333
2649
|
} else if (mixedOutcomeLanes.length > 0) {
|
|
2334
|
-
const mixedIds = mixedOutcomeLanes.map(l => `lane-${l.laneNumber}`).join(", ");
|
|
2650
|
+
const mixedIds = mixedOutcomeLanes.map((l) => `lane-${l.laneNumber}`).join(", ");
|
|
2335
2651
|
mergeResult = {
|
|
2336
2652
|
waveIndex: waveIdx + 1,
|
|
2337
2653
|
status: "partial",
|
|
@@ -2346,14 +2662,28 @@ export async function resumeOrchBatch(
|
|
|
2346
2662
|
// Downstream retry/update paths assume the current wave has an entry.
|
|
2347
2663
|
batchState.mergeResults.push(mergeResult);
|
|
2348
2664
|
onNotify(
|
|
2349
|
-
ORCH_MESSAGES.orchMergeFailed(
|
|
2665
|
+
ORCH_MESSAGES.orchMergeFailed(
|
|
2666
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2667
|
+
mergeResult.failedLane,
|
|
2668
|
+
mergeResult.failureReason || "unknown",
|
|
2669
|
+
),
|
|
2350
2670
|
"error",
|
|
2351
2671
|
);
|
|
2352
2672
|
} else {
|
|
2353
|
-
onNotify(
|
|
2673
|
+
onNotify(
|
|
2674
|
+
ORCH_MESSAGES.orchMergeSkipped(
|
|
2675
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2676
|
+
),
|
|
2677
|
+
"info",
|
|
2678
|
+
);
|
|
2354
2679
|
}
|
|
2355
2680
|
} else {
|
|
2356
|
-
onNotify(
|
|
2681
|
+
onNotify(
|
|
2682
|
+
ORCH_MESSAGES.orchMergeSkipped(
|
|
2683
|
+
resolveDisplayWaveNumber(waveIdx, roundToTaskWave, taskLevelWaveCount).displayWave,
|
|
2684
|
+
),
|
|
2685
|
+
"info",
|
|
2686
|
+
);
|
|
2357
2687
|
}
|
|
2358
2688
|
|
|
2359
2689
|
// ── TP-033: Safe-stop on rollback failure ─────────────────
|
|
@@ -2363,30 +2693,44 @@ export async function resumeOrchBatch(
|
|
|
2363
2693
|
if (mergeResult?.rollbackFailed) {
|
|
2364
2694
|
// TP-033 R004-2: Include persistence error warning when transaction
|
|
2365
2695
|
// record files may be missing, so operator knows to inspect manually
|
|
2366
|
-
const hasPersistErrors =
|
|
2696
|
+
const hasPersistErrors =
|
|
2697
|
+
mergeResult.persistenceErrors && mergeResult.persistenceErrors.length > 0;
|
|
2367
2698
|
const persistWarning = hasPersistErrors
|
|
2368
2699
|
? ` WARNING: ${mergeResult.persistenceErrors!.length} transaction record(s) failed to persist — recovery file(s) may be missing.`
|
|
2369
2700
|
: "";
|
|
2370
2701
|
|
|
2371
|
-
execLog(
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2702
|
+
execLog(
|
|
2703
|
+
"batch",
|
|
2704
|
+
batchState.batchId,
|
|
2705
|
+
"SAFE-STOP: verification rollback failed — forcing paused regardless of policy",
|
|
2706
|
+
{
|
|
2707
|
+
waveIndex: waveIdx,
|
|
2708
|
+
configPolicy: orchConfig.failure.on_merge_failure,
|
|
2709
|
+
...(hasPersistErrors ? { persistenceErrors: mergeResult.persistenceErrors } : {}),
|
|
2710
|
+
},
|
|
2711
|
+
);
|
|
2376
2712
|
|
|
2377
2713
|
batchState.phase = "paused";
|
|
2378
2714
|
batchState.errors.push(
|
|
2379
2715
|
`Safe-stop at wave ${waveIdx + 1}: verification rollback failed. ` +
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2716
|
+
`Merge worktree and temp branch preserved for recovery. ` +
|
|
2717
|
+
`Check transaction records in .pi/verification/ for recovery commands.` +
|
|
2718
|
+
persistWarning,
|
|
2719
|
+
);
|
|
2720
|
+
persistRuntimeState(
|
|
2721
|
+
"merge-rollback-safe-stop",
|
|
2722
|
+
batchState,
|
|
2723
|
+
wavePlan,
|
|
2724
|
+
latestAllocatedLanes,
|
|
2725
|
+
allTaskOutcomes,
|
|
2726
|
+
discovery,
|
|
2727
|
+
stateRoot,
|
|
2383
2728
|
);
|
|
2384
|
-
persistRuntimeState("merge-rollback-safe-stop", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discovery, stateRoot);
|
|
2385
2729
|
onNotify(
|
|
2386
2730
|
`🛑 Safe-stop: verification rollback failed at wave ${waveIdx + 1}. ` +
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2731
|
+
`Batch force-paused. Merge worktree preserved for manual recovery. ` +
|
|
2732
|
+
`See .pi/verification/ transaction records for recovery commands.` +
|
|
2733
|
+
persistWarning,
|
|
2390
2734
|
"error",
|
|
2391
2735
|
);
|
|
2392
2736
|
|
|
@@ -2448,7 +2792,16 @@ export async function resumeOrchBatch(
|
|
|
2448
2792
|
resumeBackend,
|
|
2449
2793
|
);
|
|
2450
2794
|
},
|
|
2451
|
-
persist: (trigger) =>
|
|
2795
|
+
persist: (trigger) =>
|
|
2796
|
+
persistRuntimeState(
|
|
2797
|
+
trigger,
|
|
2798
|
+
batchState,
|
|
2799
|
+
wavePlan,
|
|
2800
|
+
latestAllocatedLanes,
|
|
2801
|
+
allTaskOutcomes,
|
|
2802
|
+
discovery,
|
|
2803
|
+
stateRoot,
|
|
2804
|
+
),
|
|
2452
2805
|
log: (message, details) => execLog("batch", batchState.batchId, message, details),
|
|
2453
2806
|
notify: (message, level) => onNotify(message, level),
|
|
2454
2807
|
updateMergeResult: (result) => {
|
|
@@ -2462,13 +2815,29 @@ export async function resumeOrchBatch(
|
|
|
2462
2815
|
if (retryOutcome.kind === "retry_succeeded") {
|
|
2463
2816
|
mergeResult = retryOutcome.mergeResult;
|
|
2464
2817
|
batchState.phase = "executing";
|
|
2465
|
-
persistRuntimeState(
|
|
2818
|
+
persistRuntimeState(
|
|
2819
|
+
"merge-retry-succeeded",
|
|
2820
|
+
batchState,
|
|
2821
|
+
wavePlan,
|
|
2822
|
+
latestAllocatedLanes,
|
|
2823
|
+
allTaskOutcomes,
|
|
2824
|
+
discovery,
|
|
2825
|
+
stateRoot,
|
|
2826
|
+
);
|
|
2466
2827
|
// Fall through to normal post-merge flow
|
|
2467
2828
|
} else if (retryOutcome.kind === "safe_stop") {
|
|
2468
2829
|
mergeResult = retryOutcome.mergeResult;
|
|
2469
2830
|
batchState.phase = "paused";
|
|
2470
2831
|
batchState.errors.push(retryOutcome.errorMessage);
|
|
2471
|
-
persistRuntimeState(
|
|
2832
|
+
persistRuntimeState(
|
|
2833
|
+
"merge-rollback-safe-stop",
|
|
2834
|
+
batchState,
|
|
2835
|
+
wavePlan,
|
|
2836
|
+
latestAllocatedLanes,
|
|
2837
|
+
allTaskOutcomes,
|
|
2838
|
+
discovery,
|
|
2839
|
+
stateRoot,
|
|
2840
|
+
);
|
|
2472
2841
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
2473
2842
|
|
|
2474
2843
|
// ── TP-076: Emit supervisor alert for merge safe-stop ──
|
|
@@ -2494,7 +2863,8 @@ export async function resumeOrchBatch(
|
|
|
2494
2863
|
} else if (retryOutcome.kind === "exhausted") {
|
|
2495
2864
|
// TP-033 R006-2: Force paused regardless of on_merge_failure config.
|
|
2496
2865
|
mergeResult = retryOutcome.mergeResult;
|
|
2497
|
-
const exhaustionMsg =
|
|
2866
|
+
const exhaustionMsg =
|
|
2867
|
+
retryOutcome.errorMessage +
|
|
2498
2868
|
` [${retryOutcome.classification ?? "unknown"} ${retryOutcome.lastDecision.currentAttempt}/${retryOutcome.lastDecision.maxAttempts}, scope=${retryOutcome.scopeKey}]`;
|
|
2499
2869
|
|
|
2500
2870
|
execLog("batch", batchState.batchId, `merge retry exhausted — forcing paused`, {
|
|
@@ -2506,7 +2876,15 @@ export async function resumeOrchBatch(
|
|
|
2506
2876
|
|
|
2507
2877
|
batchState.phase = "paused";
|
|
2508
2878
|
batchState.errors.push(exhaustionMsg);
|
|
2509
|
-
persistRuntimeState(
|
|
2879
|
+
persistRuntimeState(
|
|
2880
|
+
"merge-retry-exhausted",
|
|
2881
|
+
batchState,
|
|
2882
|
+
wavePlan,
|
|
2883
|
+
latestAllocatedLanes,
|
|
2884
|
+
allTaskOutcomes,
|
|
2885
|
+
discovery,
|
|
2886
|
+
stateRoot,
|
|
2887
|
+
);
|
|
2510
2888
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
2511
2889
|
|
|
2512
2890
|
// ── TP-076: Emit supervisor alert for merge retry exhausted ──
|
|
@@ -2539,11 +2917,24 @@ export async function resumeOrchBatch(
|
|
|
2539
2917
|
? ` [not retriable: ${retryOutcome.classification}, scope=${retryOutcome.scopeKey}]`
|
|
2540
2918
|
: "";
|
|
2541
2919
|
|
|
2542
|
-
execLog(
|
|
2920
|
+
execLog(
|
|
2921
|
+
"batch",
|
|
2922
|
+
batchState.batchId,
|
|
2923
|
+
`merge failure — applying ${policyResult.policy} policy${classNote}`,
|
|
2924
|
+
policyResult.logDetails,
|
|
2925
|
+
);
|
|
2543
2926
|
|
|
2544
2927
|
batchState.phase = policyResult.targetPhase;
|
|
2545
2928
|
batchState.errors.push(policyResult.errorMessage + classNote);
|
|
2546
|
-
persistRuntimeState(
|
|
2929
|
+
persistRuntimeState(
|
|
2930
|
+
policyResult.persistTrigger,
|
|
2931
|
+
batchState,
|
|
2932
|
+
wavePlan,
|
|
2933
|
+
latestAllocatedLanes,
|
|
2934
|
+
allTaskOutcomes,
|
|
2935
|
+
discovery,
|
|
2936
|
+
stateRoot,
|
|
2937
|
+
);
|
|
2547
2938
|
onNotify(policyResult.notifyMessage + classNote, policyResult.notifyLevel);
|
|
2548
2939
|
|
|
2549
2940
|
// ── TP-076: Emit supervisor alert for merge failure (no-retry policy) ──
|
|
@@ -2575,9 +2966,15 @@ export async function resumeOrchBatch(
|
|
|
2575
2966
|
// TP-032 R006-3: Exclude verification_new_failure lanes from branch cleanup
|
|
2576
2967
|
if (mergeResult && mergeResult.status === "succeeded") {
|
|
2577
2968
|
for (const lr of mergeResult.laneResults) {
|
|
2578
|
-
if (
|
|
2969
|
+
if (
|
|
2970
|
+
!lr.error &&
|
|
2971
|
+
(lr.result?.status === "SUCCESS" || lr.result?.status === "CONFLICT_RESOLVED")
|
|
2972
|
+
) {
|
|
2579
2973
|
const laneRepoRoot = resolveRepoRoot(lr.repoId, repoRoot, workspaceConfig);
|
|
2580
|
-
const ancestorCheck = runGit(
|
|
2974
|
+
const ancestorCheck = runGit(
|
|
2975
|
+
["merge-base", "--is-ancestor", lr.sourceBranch, lr.targetBranch],
|
|
2976
|
+
laneRepoRoot,
|
|
2977
|
+
);
|
|
2581
2978
|
if (ancestorCheck.ok) {
|
|
2582
2979
|
deleteBranchBestEffort(lr.sourceBranch, laneRepoRoot);
|
|
2583
2980
|
}
|
|
@@ -2601,30 +2998,46 @@ export async function resumeOrchBatch(
|
|
|
2601
2998
|
let targetBranch = batchState.orchBranch;
|
|
2602
2999
|
if (repoId && perRepoRoot !== repoRoot) {
|
|
2603
3000
|
try {
|
|
2604
|
-
targetBranch = resolveBaseBranch(
|
|
2605
|
-
|
|
3001
|
+
targetBranch = resolveBaseBranch(
|
|
3002
|
+
repoId,
|
|
3003
|
+
perRepoRoot,
|
|
3004
|
+
batchState.orchBranch,
|
|
3005
|
+
workspaceConfig,
|
|
3006
|
+
);
|
|
3007
|
+
} catch {
|
|
3008
|
+
/* fall back to orchBranch */
|
|
3009
|
+
}
|
|
2606
3010
|
}
|
|
2607
3011
|
return { repoRoot: perRepoRoot, targetBranch };
|
|
2608
3012
|
},
|
|
2609
3013
|
);
|
|
2610
3014
|
ppUnsafeBranches = ppResult.unsafeBranches;
|
|
2611
|
-
if (ppResult.results.some(r => r.saved)) {
|
|
2612
|
-
execLog(
|
|
2613
|
-
|
|
3015
|
+
if (ppResult.results.some((r) => r.saved)) {
|
|
3016
|
+
execLog(
|
|
3017
|
+
"batch",
|
|
3018
|
+
batchState.batchId,
|
|
3019
|
+
`preserved partial progress for ${ppResult.results.filter((r) => r.saved).length} failed task(s) before inter-wave reset`,
|
|
3020
|
+
);
|
|
2614
3021
|
}
|
|
2615
3022
|
// Log per-task warnings for failed preservation attempts
|
|
2616
3023
|
for (const r of ppResult.results) {
|
|
2617
3024
|
if (!r.saved && (r.commitCount > 0 || r.error)) {
|
|
2618
|
-
execLog(
|
|
3025
|
+
execLog(
|
|
3026
|
+
"batch",
|
|
3027
|
+
batchState.batchId,
|
|
2619
3028
|
`WARNING: Failed to preserve partial progress for task ${r.taskId} ` +
|
|
2620
|
-
|
|
2621
|
-
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" }
|
|
3029
|
+
`(${r.commitCount} commit(s) at risk on lane branch)`,
|
|
3030
|
+
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" },
|
|
3031
|
+
);
|
|
2622
3032
|
}
|
|
2623
3033
|
}
|
|
2624
3034
|
if (ppUnsafeBranches.size > 0) {
|
|
2625
|
-
execLog(
|
|
3035
|
+
execLog(
|
|
3036
|
+
"batch",
|
|
3037
|
+
batchState.batchId,
|
|
2626
3038
|
`WARNING: ${ppUnsafeBranches.size} lane branch(es) could not be preserved — skipping reset for those lanes to prevent commit loss`,
|
|
2627
|
-
{ unsafeBranches: [...ppUnsafeBranches] }
|
|
3039
|
+
{ unsafeBranches: [...ppUnsafeBranches] },
|
|
3040
|
+
);
|
|
2628
3041
|
}
|
|
2629
3042
|
// TP-028: Stamp task outcomes with partial progress data for persistence
|
|
2630
3043
|
applyPartialProgressToOutcomes(ppResult, allTaskOutcomes);
|
|
@@ -2636,7 +3049,10 @@ export async function resumeOrchBatch(
|
|
|
2636
3049
|
// TP-029 R006: Track worktrees that failed reset AND removal
|
|
2637
3050
|
// so the cleanup gate only fires on true stale state, not
|
|
2638
3051
|
// successfully-reset reusable worktrees. (Parity with engine.ts)
|
|
2639
|
-
const failedRemovalWorktrees = new Map<
|
|
3052
|
+
const failedRemovalWorktrees = new Map<
|
|
3053
|
+
string,
|
|
3054
|
+
{ repoId: string | undefined; paths: string[] }
|
|
3055
|
+
>();
|
|
2640
3056
|
|
|
2641
3057
|
// Use encounteredRepoRoots which includes both persisted lanes
|
|
2642
3058
|
// AND newly allocated lanes from resumed waves, ensuring repos
|
|
@@ -2652,7 +3068,12 @@ export async function resumeOrchBatch(
|
|
|
2652
3068
|
} else {
|
|
2653
3069
|
const repoId = resolveRepoIdFromRoot(perRepoRoot, workspaceConfig);
|
|
2654
3070
|
try {
|
|
2655
|
-
targetBranch = resolveBaseBranch(
|
|
3071
|
+
targetBranch = resolveBaseBranch(
|
|
3072
|
+
repoId,
|
|
3073
|
+
perRepoRoot,
|
|
3074
|
+
batchState.orchBranch,
|
|
3075
|
+
workspaceConfig,
|
|
3076
|
+
);
|
|
2656
3077
|
} catch {
|
|
2657
3078
|
// If resolution fails, fall back to orchBranch (reset will
|
|
2658
3079
|
// fail gracefully and trigger worktree removal)
|
|
@@ -2663,9 +3084,12 @@ export async function resumeOrchBatch(
|
|
|
2663
3084
|
// TP-028: Skip reset for worktrees whose lane branch has
|
|
2664
3085
|
// unsaved partial progress (preservation failed with commits)
|
|
2665
3086
|
if (ppUnsafeBranches.has(wt.branch)) {
|
|
2666
|
-
execLog(
|
|
3087
|
+
execLog(
|
|
3088
|
+
"batch",
|
|
3089
|
+
batchState.batchId,
|
|
2667
3090
|
`skipping worktree reset for lane ${wt.laneNumber} — branch "${wt.branch}" has unsaved partial progress`,
|
|
2668
|
-
{ path: wt.path, branch: wt.branch }
|
|
3091
|
+
{ path: wt.path, branch: wt.branch },
|
|
3092
|
+
);
|
|
2669
3093
|
continue;
|
|
2670
3094
|
}
|
|
2671
3095
|
|
|
@@ -2676,9 +3100,8 @@ export async function resumeOrchBatch(
|
|
|
2676
3100
|
} catch {
|
|
2677
3101
|
forceCleanupWorktree(wt, perRepoRoot, batchState.batchId);
|
|
2678
3102
|
// Track this worktree for the cleanup gate — it may still be registered
|
|
2679
|
-
const perRepoId =
|
|
2680
|
-
? undefined
|
|
2681
|
-
: resolveRepoIdFromRoot(perRepoRoot, workspaceConfig);
|
|
3103
|
+
const perRepoId =
|
|
3104
|
+
perRepoRoot === repoRoot ? undefined : resolveRepoIdFromRoot(perRepoRoot, workspaceConfig);
|
|
2682
3105
|
if (!failedRemovalWorktrees.has(perRepoRoot)) {
|
|
2683
3106
|
failedRemovalWorktrees.set(perRepoRoot, { repoId: perRepoId, paths: [] });
|
|
2684
3107
|
}
|
|
@@ -2699,9 +3122,9 @@ export async function resumeOrchBatch(
|
|
|
2699
3122
|
if (failedRemovalWorktrees.size > 0) {
|
|
2700
3123
|
for (const [perRepoRoot, { repoId: perRepoId, paths: failedPaths }] of failedRemovalWorktrees) {
|
|
2701
3124
|
const remaining = listWorktrees(wtPrefix, perRepoRoot, resetOpId, batchState.batchId);
|
|
2702
|
-
const remainingPaths = new Set(remaining.map(wt => wt.path));
|
|
3125
|
+
const remainingPaths = new Set(remaining.map((wt) => wt.path));
|
|
2703
3126
|
// Only report worktrees that were targeted for removal but are still registered
|
|
2704
|
-
const stale = failedPaths.filter(p => remainingPaths.has(p));
|
|
3127
|
+
const stale = failedPaths.filter((p) => remainingPaths.has(p));
|
|
2705
3128
|
if (stale.length > 0) {
|
|
2706
3129
|
cleanupGateFailures.push({
|
|
2707
3130
|
repoRoot: perRepoRoot,
|
|
@@ -2715,11 +3138,24 @@ export async function resumeOrchBatch(
|
|
|
2715
3138
|
if (cleanupGateFailures.length > 0) {
|
|
2716
3139
|
const gatePolicyResult = computeCleanupGatePolicy(waveIdx, cleanupGateFailures);
|
|
2717
3140
|
|
|
2718
|
-
execLog(
|
|
3141
|
+
execLog(
|
|
3142
|
+
"batch",
|
|
3143
|
+
batchState.batchId,
|
|
3144
|
+
`cleanup gate failed — pausing batch`,
|
|
3145
|
+
gatePolicyResult.logDetails,
|
|
3146
|
+
);
|
|
2719
3147
|
|
|
2720
3148
|
batchState.phase = gatePolicyResult.targetPhase;
|
|
2721
3149
|
batchState.errors.push(gatePolicyResult.errorMessage);
|
|
2722
|
-
persistRuntimeState(
|
|
3150
|
+
persistRuntimeState(
|
|
3151
|
+
gatePolicyResult.persistTrigger,
|
|
3152
|
+
batchState,
|
|
3153
|
+
wavePlan,
|
|
3154
|
+
latestAllocatedLanes,
|
|
3155
|
+
allTaskOutcomes,
|
|
3156
|
+
discovery,
|
|
3157
|
+
stateRoot,
|
|
3158
|
+
);
|
|
2723
3159
|
onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
|
|
2724
3160
|
preserveWorktreesForResume = true;
|
|
2725
3161
|
break;
|
|
@@ -2731,11 +3167,18 @@ export async function resumeOrchBatch(
|
|
|
2731
3167
|
// TP-031 (R006): Parity with engine.ts — this check MUST run before cleanup
|
|
2732
3168
|
// so that worktrees survive when failedTasks > 0. Without this, cleanup
|
|
2733
3169
|
// deletes worktrees before the batch is marked "paused", breaking resumability.
|
|
2734
|
-
if (
|
|
2735
|
-
|
|
2736
|
-
batchState.
|
|
3170
|
+
if (
|
|
3171
|
+
!preserveWorktreesForResume &&
|
|
3172
|
+
((batchState.phase as OrchBatchPhase) === "executing" ||
|
|
3173
|
+
(batchState.phase as OrchBatchPhase) === "merging") &&
|
|
3174
|
+
batchState.failedTasks > 0
|
|
3175
|
+
) {
|
|
2737
3176
|
preserveWorktreesForResume = true;
|
|
2738
|
-
execLog(
|
|
3177
|
+
execLog(
|
|
3178
|
+
"resume",
|
|
3179
|
+
batchState.batchId,
|
|
3180
|
+
"pre-cleanup: failedTasks > 0 detected, preserving worktrees for resume",
|
|
3181
|
+
);
|
|
2739
3182
|
}
|
|
2740
3183
|
|
|
2741
3184
|
// ── 11. Cleanup and terminal state ───────────────────────────
|
|
@@ -2754,24 +3197,32 @@ export async function resumeOrchBatch(
|
|
|
2754
3197
|
if (repoId && perRepoRoot !== repoRoot) {
|
|
2755
3198
|
try {
|
|
2756
3199
|
targetBranch = resolveBaseBranch(repoId, perRepoRoot, batchState.orchBranch, workspaceConfig);
|
|
2757
|
-
} catch {
|
|
3200
|
+
} catch {
|
|
3201
|
+
/* fall back to orchBranch */
|
|
3202
|
+
}
|
|
2758
3203
|
}
|
|
2759
3204
|
return { repoRoot: perRepoRoot, targetBranch };
|
|
2760
3205
|
},
|
|
2761
3206
|
);
|
|
2762
|
-
if (ppResult.results.some(r => r.saved)) {
|
|
2763
|
-
execLog(
|
|
2764
|
-
|
|
3207
|
+
if (ppResult.results.some((r) => r.saved)) {
|
|
3208
|
+
execLog(
|
|
3209
|
+
"batch",
|
|
3210
|
+
batchState.batchId,
|
|
3211
|
+
`preserved partial progress for ${ppResult.results.filter((r) => r.saved).length} failed task(s) before terminal cleanup`,
|
|
3212
|
+
);
|
|
2765
3213
|
}
|
|
2766
3214
|
// Log warnings for failed preservation attempts — at terminal cleanup
|
|
2767
3215
|
// we cannot skip deletion (batch is ending), but operators need to know
|
|
2768
3216
|
// that commits may become unreachable via reflog only.
|
|
2769
3217
|
for (const r of ppResult.results) {
|
|
2770
3218
|
if (!r.saved && (r.commitCount > 0 || r.error)) {
|
|
2771
|
-
execLog(
|
|
3219
|
+
execLog(
|
|
3220
|
+
"batch",
|
|
3221
|
+
batchState.batchId,
|
|
2772
3222
|
`WARNING: Failed to preserve partial progress for task ${r.taskId} ` +
|
|
2773
|
-
|
|
2774
|
-
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" }
|
|
3223
|
+
`(${r.commitCount} commit(s) may become unreachable after cleanup)`,
|
|
3224
|
+
{ taskId: r.taskId, commitCount: r.commitCount, error: r.error ?? "unknown" },
|
|
3225
|
+
);
|
|
2775
3226
|
}
|
|
2776
3227
|
}
|
|
2777
3228
|
// TP-028: Stamp task outcomes with partial progress data for persistence
|
|
@@ -2813,14 +3264,24 @@ export async function resumeOrchBatch(
|
|
|
2813
3264
|
targetBranch = undefined;
|
|
2814
3265
|
}
|
|
2815
3266
|
}
|
|
2816
|
-
removeAllWorktrees(
|
|
3267
|
+
removeAllWorktrees(
|
|
3268
|
+
wtPrefix,
|
|
3269
|
+
perRepoRoot,
|
|
3270
|
+
cleanupOpId,
|
|
3271
|
+
targetBranch,
|
|
3272
|
+
batchState.batchId,
|
|
3273
|
+
orchConfig,
|
|
3274
|
+
);
|
|
2817
3275
|
}
|
|
2818
3276
|
}
|
|
2819
3277
|
|
|
2820
3278
|
batchState.endedAt = Date.now();
|
|
2821
3279
|
const totalElapsedSec = Math.round((batchState.endedAt - batchState.startedAt) / 1000);
|
|
2822
3280
|
|
|
2823
|
-
if (
|
|
3281
|
+
if (
|
|
3282
|
+
(batchState.phase as OrchBatchPhase) === "executing" ||
|
|
3283
|
+
(batchState.phase as OrchBatchPhase) === "merging"
|
|
3284
|
+
) {
|
|
2824
3285
|
if (batchState.failedTasks > 0) {
|
|
2825
3286
|
// TP-031: Parity with engine.ts — default to "paused" so the batch is
|
|
2826
3287
|
// resumable without --force. "failed" is reserved for unrecoverable
|
|
@@ -2842,28 +3303,65 @@ export async function resumeOrchBatch(
|
|
|
2842
3303
|
// supervisor agent. Legacy engine fast-forward is removed — supervisor
|
|
2843
3304
|
// handles all non-manual integration after batch_complete event.
|
|
2844
3305
|
const mergedTaskCount = batchState.succeededTasks;
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
3306
|
+
// TP-195: hoist `batchState.phase` to a fresh local with the wide
|
|
3307
|
+
// `OrchBatchPhase` type. TypeScript's narrowing-on-property semantics
|
|
3308
|
+
// under `strict: false` carries assignments forward through the
|
|
3309
|
+
// function (visible in the `(batchState.phase as OrchBatchPhase) === ...`
|
|
3310
|
+
// pattern already used at lines ~3366/~3476 above), which here narrows
|
|
3311
|
+
// `batchState.phase` to a subtype that excludes `"completed"` and
|
|
3312
|
+
// `"failed"`. Hoisting to a typed local breaks the narrowing chain so
|
|
3313
|
+
// the comparisons typecheck without a per-call cast. Runtime
|
|
3314
|
+
// evaluation is identical.
|
|
3315
|
+
const phaseAtTerminal = batchState.phase as OrchBatchPhase;
|
|
3316
|
+
const isTerminalPhase = phaseAtTerminal === "completed" || phaseAtTerminal === "failed";
|
|
3317
|
+
if (
|
|
3318
|
+
isTerminalPhase &&
|
|
3319
|
+
!preserveWorktreesForResume &&
|
|
3320
|
+
batchState.orchBranch &&
|
|
3321
|
+
mergedTaskCount > 0
|
|
3322
|
+
) {
|
|
3323
|
+
if (
|
|
3324
|
+
orchConfig.orchestrator.integration === "supervised" ||
|
|
3325
|
+
orchConfig.orchestrator.integration === "auto"
|
|
3326
|
+
) {
|
|
2848
3327
|
// TP-043: Supervisor-managed integration modes. Defer to supervisor.
|
|
2849
|
-
execLog(
|
|
3328
|
+
execLog(
|
|
3329
|
+
"resume",
|
|
3330
|
+
batchState.batchId,
|
|
3331
|
+
`integration deferred to supervisor (mode: ${orchConfig.orchestrator.integration})`,
|
|
3332
|
+
);
|
|
2850
3333
|
} else {
|
|
2851
3334
|
// Manual mode (default): show integration guidance
|
|
2852
3335
|
onNotify(
|
|
2853
|
-
ORCH_MESSAGES.orchIntegrationManual(
|
|
3336
|
+
ORCH_MESSAGES.orchIntegrationManual(
|
|
3337
|
+
batchState.orchBranch,
|
|
3338
|
+
batchState.baseBranch,
|
|
3339
|
+
mergedTaskCount,
|
|
3340
|
+
),
|
|
2854
3341
|
"info",
|
|
2855
3342
|
);
|
|
2856
3343
|
}
|
|
2857
3344
|
}
|
|
2858
3345
|
|
|
2859
|
-
persistRuntimeState(
|
|
3346
|
+
persistRuntimeState(
|
|
3347
|
+
"batch-terminal",
|
|
3348
|
+
batchState,
|
|
3349
|
+
wavePlan,
|
|
3350
|
+
latestAllocatedLanes,
|
|
3351
|
+
allTaskOutcomes,
|
|
3352
|
+
discovery,
|
|
3353
|
+
stateRoot,
|
|
3354
|
+
);
|
|
2860
3355
|
|
|
2861
3356
|
// ── TP-076: Emit supervisor alert for batch completion ──────
|
|
2862
|
-
|
|
3357
|
+
// TP-195: reuse the hoisted-typed phase to avoid the same narrowing
|
|
3358
|
+
// artifact as the `isTerminalPhase` check above.
|
|
3359
|
+
if (phaseAtTerminal === "completed" || phaseAtTerminal === "failed") {
|
|
2863
3360
|
const batchDurationMs = batchState.endedAt ? batchState.endedAt - batchState.startedAt : 0;
|
|
2864
|
-
const durationStr =
|
|
2865
|
-
|
|
2866
|
-
|
|
3361
|
+
const durationStr =
|
|
3362
|
+
batchDurationMs > 0
|
|
3363
|
+
? `${Math.floor(batchDurationMs / 60000)}m ${Math.round((batchDurationMs % 60000) / 1000)}s`
|
|
3364
|
+
: "unknown";
|
|
2867
3365
|
if (batchState.phase === "completed" && batchState.failedTasks === 0) {
|
|
2868
3366
|
emitAlert({
|
|
2869
3367
|
category: "batch-complete",
|
|
@@ -2900,10 +3398,21 @@ export async function resumeOrchBatch(
|
|
|
2900
3398
|
|
|
2901
3399
|
// ── TP-031: Emit diagnostic reports (JSONL + markdown) ──
|
|
2902
3400
|
// Non-fatal: errors are logged but never crash batch finalization.
|
|
2903
|
-
emitDiagnosticReports(
|
|
3401
|
+
emitDiagnosticReports(
|
|
3402
|
+
assembleDiagnosticInput(
|
|
3403
|
+
orchConfig,
|
|
3404
|
+
batchState,
|
|
3405
|
+
wavePlan,
|
|
3406
|
+
latestAllocatedLanes,
|
|
3407
|
+
allTaskOutcomes,
|
|
3408
|
+
stateRoot,
|
|
3409
|
+
),
|
|
3410
|
+
);
|
|
2904
3411
|
|
|
2905
3412
|
if (batchState.phase === "paused" || batchState.phase === "stopped") {
|
|
2906
|
-
execLog("resume", batchState.batchId, "resumed batch ended in non-terminal state", {
|
|
3413
|
+
execLog("resume", batchState.batchId, "resumed batch ended in non-terminal state", {
|
|
3414
|
+
phase: batchState.phase,
|
|
3415
|
+
});
|
|
2907
3416
|
} else {
|
|
2908
3417
|
onNotify(
|
|
2909
3418
|
ORCH_MESSAGES.resumeComplete(
|
|
@@ -2928,9 +3437,7 @@ export async function resumeOrchBatch(
|
|
|
2928
3437
|
}
|
|
2929
3438
|
}
|
|
2930
3439
|
|
|
2931
|
-
|
|
2932
3440
|
// TP-043: attemptAutoIntegration is no longer called from engine.ts or resume.ts.
|
|
2933
3441
|
// Supervisor-managed integration ("supervised" and "auto" modes) is handled by
|
|
2934
3442
|
// the supervisor agent after batch_complete. The helper remains in merge.ts for
|
|
2935
3443
|
// use by the supervisor's integration flow.
|
|
2936
|
-
|