taskplane 0.30.4 → 0.30.6

package/extensions/taskplane/resume.ts

@@ -12,10 +12,16 @@ import {
 	resolveDisplayWaveNumber,
 	buildSpawnFailureAlertExtras,
 } from "./engine.ts";
+import {
+	advanceActiveSegment,
+	applyReExecutionOutcomeToSegments,
+	taskSegmentsAllSucceeded,
+} from "./segment-recovery.ts";
 import {
 	buildReviewerEnv,
 	buildWorkerEnv,
 	buildWorkerExcludeEnv,
+	batchTaskScope,
 	computeTransitiveDependents,
 	execLog,
 	executeLaneV2,
@@ -31,23 +37,99 @@ import { readRegistrySnapshot, isTerminalStatus, isProcessAlive } from "./proces
  * Per Runtime V2 spec §7.3: detect + terminate + rehydrate.
  * Prevents duplicate concurrent agents for the same lane/task on resume.
  */
-function terminateAliveV2Agents(stateRoot: string, batchId: string, sessionName: string): void {
+/** #631: how long to wait for SIGTERM, then SIGKILL, before declaring termination unconfirmed. */
+const TERMINATE_GRACE_MS = 5_000;
+const TERMINATE_KILL_MS = 3_000;
+const TERMINATE_POLL_MS = 200;
+
+function sleep(ms: number): Promise<void> {
+	return new Promise((r) => setTimeout(r, ms));
+}
+
+async function waitForExit(pids: number[], timeoutMs: number): Promise<number[]> {
+	const deadline = Date.now() + timeoutMs;
+	let survivors = pids.filter((pid) => isProcessAlive(pid));
+	while (survivors.length > 0 && Date.now() < deadline) {
+		await sleep(TERMINATE_POLL_MS);
+		survivors = survivors.filter((pid) => isProcessAlive(pid));
+	}
+	return survivors;
+}
+
+/**
+ * Terminate the lane's still-alive V2 agents (worker/reviewer) before this
+ * resume re-executes the lane in the SAME worktree.
+ *
+ * #631: termination is VERIFIED, not fire-and-forget. SIGTERM → bounded wait →
+ * SIGKILL → bounded wait. If any agent is still alive after that, THROW: a
+ * signal-resistant or permission-protected worker would otherwise keep writing
+ * the worktree alongside its replacement. The callers' existing catch blocks
+ * mark the task failed with this reason instead of re-executing.
+ */
+async function terminateAliveV2Agents(
+	stateRoot: string,
+	batchId: string,
+	sessionName: string,
+): Promise<void> {
 	const registry = readRegistrySnapshot(stateRoot, batchId);
 	if (!registry) return;
+	const targets: Array<{ key: string; pid: number }> = [];
 	for (const suffix of ["-worker", "-reviewer", ""]) {
 		const key = `${sessionName}${suffix}`;
 		const manifest = registry.agents[key];
 		if (manifest && !isTerminalStatus(manifest.status) && isProcessAlive(manifest.pid)) {
+			targets.push({ key, pid: manifest.pid });
+		}
+	}
+	if (targets.length === 0) return;
+
+	for (const t of targets) {
+		try {
+			process.kill(t.pid, "SIGTERM");
+			execLog("resume", t.key, `SIGTERM sent to alive V2 agent (PID ${t.pid}) before re-execute`);
+		} catch {
+			/* already dead or not signalable — verified below */
+		}
+	}
+	let survivors = await waitForExit(
+		targets.map((t) => t.pid),
+		TERMINATE_GRACE_MS,
+	);
+	if (survivors.length > 0) {
+		for (const pid of survivors) {
 			try {
-				process.kill(manifest.pid, "SIGTERM");
-				execLog("resume", key, `terminated alive V2 agent (PID ${manifest.pid}) before re-execute`);
+				process.kill(pid, "SIGKILL");
+				execLog(
+					"resume",
+					sessionName,
+					`SIGKILL sent to V2 agent PID ${pid} (did not exit within ${TERMINATE_GRACE_MS}ms)`,
+				);
 			} catch {
-				/* already dead */
+				/* verified below */
 			}
 		}
+		survivors = await waitForExit(survivors, TERMINATE_KILL_MS);
 	}
+	if (survivors.length > 0) {
+		const list = targets
+			.filter((t) => survivors.includes(t.pid))
+			.map((t) => `${t.key} (PID ${t.pid})`)
+			.join(", ");
+		throw new Error(
+			`cannot confirm termination of ${list} after SIGTERM+SIGKILL — refusing to re-execute the lane alongside a live agent (#631). ` +
+				`Terminate the process manually and resume again.`,
+		);
+	}
+	execLog(
+		"resume",
+		sessionName,
+		`verified termination of ${targets.length} V2 agent(s) before re-execute`,
+		{
+			pids: targets.map((t) => t.pid).join(","),
+		},
+	);
 }
-import { getCurrentBranch, runGit } from "./git.ts";
+import { getCurrentBranch, runGit, describeOrchBranchStateAcrossRepos } from "./git.ts";
 import { mergeWaveByRepo } from "./merge.ts";
 import {
 	applyMergeRetryLoop,
@@ -128,6 +210,41 @@ import {
  * @param workspaceConfig  - Workspace configuration (null in repo mode)
  * @returns Array of unique absolute repo root paths
  */
+/**
+ * Catch-up merge eligibility (resume step 8d), pure: which persisted lanes hold
+ * succeeded-but-unmerged work? A lane qualifies when ALL its tasks are
+ * `succeeded`, none were re-executed in this resume pass, and the (LAST) wave
+ * of every task has no `succeeded` merge record. Branch existence is checked by
+ * the caller (needs git).
+ */
+export function selectCatchUpLanes(
+	persistedState: Pick<PersistedBatchState, "lanes" | "tasks" | "mergeResults">,
+	wavePlan: string[][],
+	reExecutedTaskIds: ReadonlySet<string>,
+): PersistedBatchState["lanes"] {
+	// LATEST merge status per wave (the same rule computeResumePoint uses) — an
+	// older succeeded record must not mask a later failure (success → failure
+	// while a third task was still pending left succeeded work permanently
+	// unmerged).
+	const waveMerged = (w: number) =>
+		getMergeStatusForWave(persistedState.mergeResults ?? [], w) === "succeeded";
+	const waveOfTask = new Map<string, number>();
+	wavePlan.forEach((wave, i) => {
+		for (const id of wave) waveOfTask.set(id, i);
+	});
+	const succeededById = new Map(
+		persistedState.tasks.map((t) => [t.taskId, t.status === "succeeded"]),
+	);
+	return persistedState.lanes.filter((laneRecord) => {
+		if (laneRecord.taskIds.length === 0) return false;
+		if (!laneRecord.taskIds.every((id) => succeededById.get(id))) return false;
+		if (laneRecord.taskIds.some((id) => reExecutedTaskIds.has(id))) return false;
+		const waves = laneRecord.taskIds.map((id) => waveOfTask.get(id));
+		if (waves.some((w) => w === undefined || waveMerged(w))) return false;
+		return true;
+	});
+}
+
 export function collectRepoRoots(
 	persistedState: PersistedBatchState,
 	defaultRepoRoot: string,
@@ -1267,6 +1384,22 @@ export async function resumeOrchBatch(
 			batchState.phase = "idle";
 			return;
 		}
+		// #631: the parent gated engine ownership for a SPECIFIC batch. If the
+		// deterministic reconstruction picked a different one, refuse BEFORE any
+		// write — persisting reconstructed state for an ungated batch is itself an
+		// unauthorized recovery mutation.
+		if (batchState.batchId && reconstruction.batchId !== batchState.batchId) {
+			const msg =
+				`resume target mismatch: this engine was authorized for batch ${batchState.batchId} but reconstruction ` +
+				`selected ${reconstruction.batchId}. Refusing without writing (ownership of ${reconstruction.batchId} was never verified).`;
+			execLog("resume", batchState.batchId, msg);
+			onNotify(`❌ ${msg}`, "error");
+			batchState.phase = "failed";
+			batchState.endedAt = Date.now();
+			batchState.errors.push(msg);
+			return;
+		}
+
 		// Successful reconstruction: persist so the rest of resumeOrchBatch
 		// proceeds with a normal on-disk batch-state.json picture.
 		onNotify(
@@ -1291,6 +1424,22 @@ export async function resumeOrchBatch(
 	}
 
 	// ── 2. Check eligibility ─────────────────────────────────────
+	// #631: the parent gated engine ownership against a specific target and
+	// published this engine's identity for it. Never resume a DIFFERENT batch
+	// than the one authorized (e.g. reconstruction selecting another runtime
+	// dir) — that batch's engine was never checked.
+	if (batchState.batchId && persistedState.batchId !== batchState.batchId) {
+		const msg =
+			`resume target mismatch: this engine was authorized for batch ${batchState.batchId} but the ` +
+			`persisted/reconstructed state is ${persistedState.batchId}. Refusing (ownership of ${persistedState.batchId} was never verified).`;
+		execLog("resume", batchState.batchId, msg);
+		onNotify(`❌ ${msg}`, "error");
+		batchState.phase = "failed";
+		batchState.endedAt = Date.now();
+		batchState.errors.push(msg);
+		return;
+	}
+
 	const eligibility = checkResumeEligibility(persistedState, force);
 	if (!eligibility.eligible) {
 		onNotify(
@@ -1557,6 +1706,23 @@ export async function resumeOrchBatch(
 	// v3: Carry forward resilience and diagnostics from persisted state
 	batchState.resilience = persistedState.resilience;
 	batchState.diagnostics = persistedState.diagnostics;
+	// Carry forward merge HISTORY. The runtime starts fresh on every resume, so
+	// without this the next checkpoint serialized only the merges appended by
+	// this pass and earlier waves' records were dropped — a later resume then
+	// re-flagged already-merged waves for retry (and catch-up re-selected their
+	// lanes). Persisted records are 0-based; runtime is 1-based (converted once
+	// here, back once in serializeBatchState). Order preserved (latest-wins reads).
+	batchState.mergeResults = (persistedState.mergeResults ?? []).map(
+		(mr) =>
+			({
+				waveIndex: mr.waveIndex + 1,
+				status: mr.status,
+				laneResults: [],
+				failedLane: mr.failedLane,
+				failureReason: mr.failureReason,
+				totalDurationMs: 0,
+			}) as MergeWaveResult,
+	);
 	// v4: Carry forward segment records (including dynamically expanded segments)
 	batchState.segments = [...(persistedState.segments ?? [])];
 	// Carry forward unknown fields for roundtrip preservation
@@ -1682,8 +1848,8 @@ export async function resumeOrchBatch(
 			execLog("resume", task.taskId, "V2 reconnect: terminate + rehydrate via lane-runner", {
 				repoId: laneRecord.repoId ?? "(default)",
 			});
-			terminateAliveV2Agents(stateRoot, persistedState.batchId, laneRecord.laneSessionId);
 			try {
+				await terminateAliveV2Agents(stateRoot, persistedState.batchId, laneRecord.laneSessionId);
 				const laneResult = await executeLaneV2(
 					lane,
 					orchConfig,
@@ -1693,6 +1859,7 @@ export async function resumeOrchBatch(
 					!!workspaceConfig,
 					{
 						ORCH_BATCH_ID: batchState.batchId,
+						...buildWorkerEnv(runnerConfig.worker),
 						...buildReviewerEnv(runnerConfig.reviewer),
 						...buildWorkerExcludeEnv(runnerConfig.workerExcludeExtensions),
 					},
@@ -1729,7 +1896,14 @@ export async function resumeOrchBatch(
 
 	// ── 8b. Handle re-execute tasks (dead session + existing worktree) ──
 	const reExecuteTasks = reconciledTasks.filter((t) => t.action === "re-execute");
+	// Worktree preservation flag (declared before 8c/8d so a merge failure on
+	// resume can set it; consumed by terminal cleanup).
+	let preserveWorktreesForResume = false;
 	const reExecuteFinalStatus = new Map<string, LaneTaskStatus>();
+	// #629: the REAL lane outcome of a re-executed task (telemetry, exit
+	// diagnostic, timestamps). Previously discarded — the synthesized outcome
+	// below carried the persisted (already-cleared) diagnostic instead.
+	const reExecuteOutcome = new Map<string, LaneTaskOutcome>();
 	const reExecAllocatedLanes: AllocatedLane[] = [];
 
 	if (reExecuteTasks.length > 0) {
@@ -1775,7 +1949,7 @@ export async function resumeOrchBatch(
 
 			try {
 				// TP-112: Runtime V2 re-execution.
-				terminateAliveV2Agents(stateRoot, batchState.batchId, laneRecord.laneSessionId);
+				await terminateAliveV2Agents(stateRoot, batchState.batchId, laneRecord.laneSessionId);
 				const laneResult = await executeLaneV2(
 					lane,
 					orchConfig,
@@ -1785,6 +1959,7 @@ export async function resumeOrchBatch(
 					!!workspaceConfig,
 					{
 						ORCH_BATCH_ID: batchState.batchId,
+						...buildWorkerEnv(runnerConfig.worker),
 						...buildReviewerEnv(runnerConfig.reviewer),
 						...buildWorkerExcludeEnv(runnerConfig.workerExcludeExtensions),
 					},
@@ -1796,8 +1971,80 @@ export async function resumeOrchBatch(
 					exitReason: taskResult?.exitReason ?? "V2 re-execution completed",
 					doneFileFound: taskResult?.doneFileFound ?? false,
 				};
+				// #629: a PAUSE during re-execution surfaces as `skipped` from
+				// executeLaneV2. That is not a terminal outcome: leave the task and
+				// its segments pending/re-executable (the previous code marked the
+				// task failed; treating the real outcome verbatim would make it an
+				// unretryable `skipped`). Nothing else changes; the next resume
+				// reconciles it again.
+				if (
+					pollResult.status === "pending" ||
+					(pollResult.status === "skipped" && /paused/i.test(pollResult.exitReason))
+				) {
+					reExecuteFinalStatus.set(task.taskId, "pending");
+					execLog(
+						"resume",
+						task.taskId,
+						"re-execution paused — task remains pending for the next resume",
+					);
+					continue;
+				}
+				if (taskResult) reExecuteOutcome.set(task.taskId, taskResult);
+
+				// #629: keep SEGMENT authority — transition the EXECUTED segment record
+				// to the re-execution result. Re-execution runs the unit built from the
+				// task's activeSegmentId, i.e. ONE segment (segmentId null = whole
+				// single-segment/legacy task). Without this a successful retry persisted
+				// task=succeeded / segment=pending and the next resume normalized the
+				// task straight back to pending.
+				const segFinal: "succeeded" | "failed" =
+					pollResult.status === "succeeded" ? "succeeded" : "failed";
+				const executedSegmentId =
+					taskResult?.segmentId ??
+					persistedState.tasks.find((t) => t.taskId === task.taskId)?.activeSegmentId ??
+					null;
+				const touchedSegments = applyReExecutionOutcomeToSegments(
+					batchState.segments,
+					task.taskId,
+					segFinal,
+					{
+						startTime: taskResult?.startTime,
+						endTime: taskResult?.endTime,
+						exitReason: pollResult.exitReason,
+						exitDiagnostic: taskResult?.exitDiagnostic,
+					},
+					executedSegmentId,
+				);
+				if (touchedSegments.length > 0) {
+					execLog("resume", task.taskId, `re-execution: segment records → ${segFinal}`, {
+						segments: touchedSegments.join(","),
+						executedSegmentId: executedSegmentId ?? "(whole task)",
+					});
+				}
 
-				if (pollResult.status === "succeeded") {
+				// Task completion is derived from the segment FRONTIER, not from one
+				// segment's success: a non-final segment succeeding leaves the task
+				// pending with downstream segments still to run.
+				const frontierComplete = taskSegmentsAllSucceeded(batchState.segments, task.taskId);
+				if (pollResult.status === "succeeded" && frontierComplete === false) {
+					reExecuteFinalStatus.set(task.taskId, "pending");
+					reExecuteOutcome.delete(task.taskId); // not a task-level outcome yet
+					reExecuteTaskSet.delete(task.taskId);
+					reExecAllocatedLanes.push(lane);
+					// Advance the frontier so the next execution runs the NEXT segment (on
+					// both the persisted record and the parsed task the wave loop will use).
+					const nextSeg = advanceActiveSegment(
+						{ tasks: persistedState.tasks, segments: batchState.segments } as never,
+						task.taskId,
+					);
+					const parsedForFrontier = discovery.pending.get(task.taskId);
+					if (parsedForFrontier) parsedForFrontier.activeSegmentId = nextSeg;
+					execLog(
+						"resume",
+						task.taskId,
+						`re-executed segment ${executedSegmentId} succeeded — task has further segments pending`,
+					);
+				} else if (pollResult.status === "succeeded") {
 					reExecuteFinalStatus.set(task.taskId, "succeeded");
 					completedTaskSet.add(task.taskId);
 					failedTaskSet.delete(task.taskId);
@@ -1825,6 +2072,9 @@ export async function resumeOrchBatch(
 				batchState.failedTasks++;
 				const msg = err instanceof Error ? err.message : String(err);
 				execLog("resume", task.taskId, `re-execution error: ${msg}`);
+				applyReExecutionOutcomeToSegments(batchState.segments, task.taskId, "failed", {
+					exitReason: `re-execution error: ${msg}`,
+				});
 			}
 		}
 	}
@@ -1919,13 +2169,206 @@ export async function resumeOrchBatch(
 					}
 				}
 			} else {
+				// Fail closed (mirrors 8d): a re-executed task that succeeded but whose
+				// branch did not merge must not let the batch proceed to a state that
+				// reads as complete with its work unmerged. Pause (resumable), preserve
+				// worktrees; the next resume re-executes nothing for it (it is succeeded)
+				// but 8d's catch-up merge picks its lane up.
 				onNotify(
-					`⚠️ Re-executed branch merge ${reExecMergeResult.status}: ${reExecMergeResult.failureReason || "unknown"}`,
+					`⚠️ Re-executed branch merge ${reExecMergeResult.status}: ${reExecMergeResult.failureReason || "unknown"} — batch paused with worktrees preserved; fix the conflict and orch_resume() to retry the merge.`,
 					"warning",
 				);
+				batchState.pauseSignal.paused = true;
+				batchState.pauseSignal.cause = "merge-failure";
+				preserveWorktreesForResume = true;
+				emitAlert({
+					category: "merge-failure",
+					summary:
+						`🔴 Merge of re-executed task(s) ${succeededReExecTaskIds.join(", ")} failed on resume: ${reExecMergeResult.failureReason || "unknown"}
+` +
+						`  The work remains on the lane branch(es); the batch is paused with worktrees preserved.
+` +
+						`  Resolve the conflict, then orch_resume() — the catch-up merge re-runs automatically.`,
+					context: {
+						batchProgress: buildBatchProgressSnapshot(batchState),
+						mergeError: reExecMergeResult.failureReason ?? undefined,
+					},
+				});
 			}
 
-			batchState.mergeResults.push(reExecMergeResult);
+			// Attribute the outcome to the ACTUAL original wave(s) of the re-executed
+			// tasks (runtime 1-indexed), not the -1 sentinel: persistence clamps the
+			// sentinel to wave 0, which mis-attributed a second-wave merge failure to
+			// wave 0 and let the real wave keep an older success record.
+			// (LAST wave per task — the same mapping selectCatchUpLanes uses for a
+			// multi-segment task's lane.)
+			const lastWaveOf = new Map<string, number>();
+			runtimeWavePlan.forEach((wave, i) => {
+				for (const id of wave) lastWaveOf.set(id, i);
+			});
+			const reExecWaves = new Set<number>();
+			for (const id of succeededReExecTaskIds) {
+				const w = lastWaveOf.get(id);
+				if (w !== undefined) reExecWaves.add(w);
+			}
+			if (reExecWaves.size === 0) {
+				batchState.mergeResults.push(reExecMergeResult);
+			} else {
+				for (const w of reExecWaves) {
+					batchState.mergeResults.push({ ...reExecMergeResult, waveIndex: w + 1 });
+				}
+			}
+		}
+	}
+
+	// ── 8d. Catch-up merge: succeeded-but-unmerged lane work ─────
+	// A pause (or crash) that lands DURING a wave can leave tasks that succeeded
+	// with their lane branches never merged — the wave's merge step did not run.
+	// On resume the wave loop only merges what it re-executes, so that work was
+	// silently dropped (Sage review of the owned-batch pause fix). Merge every
+	// persisted lane whose tasks are all succeeded and whose wave has no
+	// successful merge record. Idempotent: an already-merged branch is a no-op.
+	// Skipped entirely when 8c already raised a merge-failure pause: a successful
+	// subset catch-up must never certify a wave whose other lane work (the failed
+	// 8c merge) remains unmerged — the next resume retries BOTH via this step.
+	if (batchState.pauseSignal.cause !== "merge-failure") {
+		const waveOfTask = new Map<string, number>();
+		runtimeWavePlan.forEach((wave, i) => {
+			for (const id of wave) waveOfTask.set(id, i);
+		});
+		const catchUpLanes: AllocatedLane[] = [];
+		for (const laneRecord of selectCatchUpLanes(
+			persistedState,
+			runtimeWavePlan,
+			new Set(reExecuteFinalStatus.keys()),
+		)) {
+			const laneRepoRoot = resolveRepoRoot(laneRecord.repoId, repoRoot, workspaceConfig);
+			if (
+				!runGit(["rev-parse", "--verify", "--quiet", `refs/heads/${laneRecord.branch}`], laneRepoRoot)
+					.ok
+			) {
+				continue; // branch gone — nothing to merge
+			}
+			catchUpLanes.push({
+				laneNumber: laneRecord.laneNumber,
+				laneId: laneRecord.laneId,
+				laneSessionId: laneRecord.laneSessionId,
+				worktreePath: laneRecord.worktreePath,
+				branch: laneRecord.branch,
+				tasks: laneRecord.taskIds
+					// A task that succeeded may already be in the completed set (no ParsedTask).
+					// Enrich a stub from the persisted record so merge staging can still find
+					// the task folder (.DONE / STATUS / review artifacts).
+					.map((id) => {
+						const parsed = discovery.pending.get(id);
+						if (parsed) return parsed;
+						const rec = persistedState.tasks.find((t) => t.taskId === id);
+						return {
+							taskId: id,
+							taskName: id,
+							taskFolder: rec?.taskFolder ?? "",
+							promptPath: rec?.taskFolder ? join(rec.taskFolder, "PROMPT.md") : "",
+							fileScope: [],
+							dependencies: [],
+						} as unknown as ParsedTask;
+					})
+					.map((t) => ({ taskId: t.taskId, order: 0, task: t, estimatedMinutes: 0 })),
+				strategy: "round-robin",
+				estimatedLoad: 0,
+				estimatedMinutes: 0,
+				...(laneRecord.repoId !== undefined ? { repoId: laneRecord.repoId } : {}),
+			});
+		}
+		if (catchUpLanes.length > 0) {
+			const ids = catchUpLanes.flatMap((l) => l.tasks.map((t) => t.taskId));
+			onNotify(
+				`🔀 Merging ${catchUpLanes.length} succeeded-but-unmerged lane branch(es) from the interrupted wave (${ids.join(", ")})...`,
+				"info",
+			);
+			const CATCH_UP_WAVE_INDEX = -1;
+			const synthetic: WaveExecutionResult = {
+				waveIndex: CATCH_UP_WAVE_INDEX,
+				startedAt: Date.now(),
+				endedAt: Date.now(),
+				laneResults: catchUpLanes.map((lane) => ({
+					laneNumber: lane.laneNumber,
+					laneId: lane.laneId,
+					tasks: lane.tasks.map((t) => ({
+						taskId: t.taskId,
+						status: "succeeded" as LaneTaskStatus,
+						startTime: Date.now(),
+						endTime: Date.now(),
+						exitReason: "Succeeded before pause; merged on resume",
+						sessionName: lane.laneSessionId,
+						doneFileFound: true,
+						laneNumber: lane.laneNumber,
+					})),
+					overallStatus: "succeeded" as const,
+					startTime: Date.now(),
+					endTime: Date.now(),
+				})),
+				policyApplied: orchConfig.failure.on_task_failure,
+				stoppedEarly: false,
+				failedTaskIds: [],
+				skippedTaskIds: [],
+				succeededTaskIds: ids,
+				blockedTaskIds: [],
+				laneCount: catchUpLanes.length,
+				overallStatus: "succeeded",
+				finalMonitorState: null,
+				allocatedLanes: catchUpLanes,
+			};
+			const catchUp = await mergeWaveByRepo(
+				catchUpLanes,
+				synthetic,
+				CATCH_UP_WAVE_INDEX,
+				orchConfig,
+				repoRoot,
+				batchState.batchId,
+				batchState.orchBranch,
+				workspaceConfig,
+				stateRoot,
+				agentRoot,
+				runnerConfig.testing_commands,
+				undefined,
+				undefined,
+				resumeBackend,
+			);
+			if (catchUp.status === "succeeded") {
+				onNotify(`✅ Catch-up merge complete: ${catchUp.laneResults.length} lane(s) merged`, "info");
+			} else {
+				// Normal merge-failure handling: the batch must NOT proceed to a state
+				// that can read as complete while succeeded work sits unmerged. Pause
+				// (resumable), preserve worktrees, and stop before the wave loop; the
+				// next resume re-runs this catch-up (it is the retry mechanism).
+				onNotify(
+					`⚠️ Catch-up merge ${catchUp.status}: ${catchUp.failureReason || "unknown"} — the succeeded work remains on its lane branch(es). Batch paused; fix the conflict and orch_resume() to retry.`,
+					"warning",
+				);
+				batchState.pauseSignal.paused = true;
+				batchState.pauseSignal.cause = "merge-failure";
+				preserveWorktreesForResume = true;
+				emitAlert({
+					category: "merge-failure",
+					summary:
+						`🔴 Catch-up merge failed on resume for ${ids.join(", ")}: ${catchUp.failureReason || "unknown"}
+` +
+						`  The succeeded work remains on its lane branch(es); the batch is paused with worktrees preserved.
+` +
+						`  Resolve the conflict, then orch_resume() — the catch-up merge re-runs automatically.`,
+					context: {
+						batchProgress: buildBatchProgressSnapshot(batchState),
+						mergeError: catchUp.failureReason ?? undefined,
+					},
+				});
+			}
+			// Record it against the ORIGINAL wave(s) so the wave loop's merge-retry
+			// logic sees those waves as merged (or as needing retry on failure).
+			for (const w of new Set(
+				catchUpLanes.flatMap((l) => l.tasks.map((t) => waveOfTask.get(t.taskId) ?? 0)),
+			)) {
+				batchState.mergeResults.push({ ...catchUp, waveIndex: w + 1 });
+			}
 		}
 	}
 
@@ -1959,6 +2402,22 @@ export async function resumeOrchBatch(
 		const persistedTask = persistedState.tasks.find((t) => t.taskId === task.taskId);
 		const reconnectStatus = reconnectFinalStatus.get(task.taskId);
 		const reExecuteStatus = reExecuteFinalStatus.get(task.taskId);
+		// #629: a re-executed task has a REAL outcome — use it verbatim (telemetry,
+		// diagnostic, timestamps) rather than synthesizing one from stale state.
+		const realOutcome = task.action === "re-execute" ? reExecuteOutcome.get(task.taskId) : undefined;
+		if (realOutcome && (realOutcome.status === "succeeded" || realOutcome.status === "failed")) {
+			allTaskOutcomes.push({
+				...realOutcome,
+				laneNumber: realOutcome.laneNumber ?? persistedTask?.laneNumber,
+				// Preserve persisted partial-progress metadata when the fresh outcome
+				// did not set it (recovery metadata; the stale diagnostic is NOT restored).
+				partialProgressCommits:
+					realOutcome.partialProgressCommits ?? persistedTask?.partialProgressCommits,
+				partialProgressBranch:
+					realOutcome.partialProgressBranch ?? persistedTask?.partialProgressBranch,
+			});
+			continue;
+		}
 		const status =
 			task.action === "reconnect"
 				? reconnectStatus || "running"
@@ -2006,7 +2465,11 @@ export async function resumeOrchBatch(
 	// (mark-failed) or resolved during reconnect/re-execute must propagate
 	// to their transitive dependents BEFORE the wave loop begins.
 	if (orchConfig.failure.on_task_failure === "skip-dependents" && failedTaskSet.size > 0) {
-		const reconciledBlocked = computeTransitiveDependents(failedTaskSet, depGraph);
+		const reconciledBlocked = computeTransitiveDependents(
+			failedTaskSet,
+			depGraph,
+			batchTaskScope(wavePlan),
+		);
 		for (const taskId of reconciledBlocked) {
 			batchState.blockedTaskIds.add(taskId);
 		}
@@ -2037,7 +2500,6 @@ export async function resumeOrchBatch(
 	// We need to execute remaining waves starting from resumeWaveIndex.
 	// For waves where some tasks are already done, we filter them out.
 
-	let preserveWorktreesForResume = false;
 	const persistedStatusByTaskId = new Map(
 		persistedState.tasks.map((task) => [task.taskId, task.status] as const),
 	);
@@ -2050,6 +2512,7 @@ export async function resumeOrchBatch(
 		// Check pause signal
 		if (batchState.pauseSignal.paused) {
 			batchState.phase = "paused";
+			preserveWorktreesForResume = true; // every pause exit preserves recovery worktrees
 			persistRuntimeState(
 				"pause-before-wave",
 				batchState,
@@ -2403,8 +2866,12 @@ export async function resumeOrchBatch(
 			reconnectTaskSet.delete(taskId);
 		}
 
-		for (const blocked of waveResult.blockedTaskIds) {
-			batchState.blockedTaskIds.add(blocked);
+		{
+			// #629: scope to batch tasks (dependency graph is repo-wide)
+			const scope = batchTaskScope(wavePlan);
+			for (const blocked of waveResult.blockedTaskIds) {
+				if (scope.has(blocked)) batchState.blockedTaskIds.add(blocked);
+			}
 		}
 
 		// ── TP-076: Emit supervisor alerts for task failures ────
@@ -2480,6 +2947,38 @@ export async function resumeOrchBatch(
 			});
 		}
 
+		// ── Pause finalizer (mirrors engine.ts; penster 20260906T194514) ──
+		// Paused tasks are pending, never skipped; a wave with any is not complete.
+		// Finalize as paused (worktrees preserved, no merge) and stop.
+		{
+			const pausedIds = waveResult.pausedTaskIds ?? [];
+			const notAborting =
+				batchState.pauseSignal.cause !== "abort" && waveResult.overallStatus !== "aborted";
+			const operatorPaused = batchState.pauseSignal.paused && notAborting;
+			if (notAborting && (pausedIds.length > 0 || operatorPaused)) {
+				batchState.phase = "paused";
+				preserveWorktreesForResume = true;
+				execLog("resume", batchState.batchId, `batch paused during wave ${waveIdx + 1}`, {
+					cause: batchState.pauseSignal.cause ?? "operator",
+					pendingTasks: pausedIds.join(",") || "(none)",
+				});
+				persistRuntimeState(
+					"pause-during-wave",
+					batchState,
+					wavePlan,
+					latestAllocatedLanes,
+					allTaskOutcomes,
+					discovery,
+					stateRoot,
+				);
+				onNotify(
+					`⏸️  Batch paused during wave ${waveIdx + 1}: ${pausedIds.length} task(s) remain pending. Worktrees preserved. Use orch_resume() to continue.`,
+					"warning",
+				);
+				break;
+			}
+		}
+
 		persistRuntimeState(
 			"wave-execution-complete",
 			batchState,
@@ -3140,17 +3639,32 @@ export async function resumeOrchBatch(
 
 						const resetResult = safeResetWorktree(wt, targetBranch, perRepoRoot);
 						if (!resetResult.success) {
-							try {
-								removeWorktree(wt, perRepoRoot);
-							} catch {
-								forceCleanupWorktree(wt, perRepoRoot, batchState.batchId);
-								// Track this worktree for the cleanup gate — it may still be registered
+							// Track for the cleanup gate on ANY non-removal outcome: throw,
+							// OR a #628 dirty refusal (refusal is NOT success — the worktree
+							// still exists, preserving uncommitted work; never force-clean it).
+							const trackFailedRemoval = () => {
 								const perRepoId =
 									perRepoRoot === repoRoot ? undefined : resolveRepoIdFromRoot(perRepoRoot, workspaceConfig);
 								if (!failedRemovalWorktrees.has(perRepoRoot)) {
 									failedRemovalWorktrees.set(perRepoRoot, { repoId: perRepoId, paths: [] });
 								}
 								failedRemovalWorktrees.get(perRepoRoot)!.paths.push(wt.path);
+							};
+							try {
+								const rm = removeWorktree(wt, perRepoRoot);
+								if (rm.refusedDirty) {
+									execLog(
+										"batch",
+										batchState.batchId,
+										`worktree removal REFUSED for lane ${wt.laneNumber}: ${rm.dirtyFileCount} uncommitted change(s) — preserve progress before cleanup (#628)`,
+										{ path: wt.path },
+									);
+									trackFailedRemoval();
+								}
+							} catch {
+								forceCleanupWorktree(wt, perRepoRoot, batchState.batchId);
+								// Track this worktree for the cleanup gate — it may still be registered
+								trackFailedRemoval();
 							}
 						}
 					}
@@ -3408,14 +3922,36 @@ export async function resumeOrchBatch(
 				? `${Math.floor(batchDurationMs / 60000)}m ${Math.round((batchDurationMs % 60000) / 1000)}s`
 				: "unknown";
 		if (batchState.phase === "completed" && batchState.failedTasks === 0) {
+			// Report outcomes and branch state SEPARATELY and truthfully (penster
+			// 20260906T194514 saw "Merged … Ready for integration" on 0/1 succeeded
+			// with an empty orch branch). Never say "merged" unless the orch branch is
+			// verifiably ahead of base; a failed comparison is "unknown", not "nothing".
+			const branchState = describeOrchBranchStateAcrossRepos(
+				batchState.orchBranch,
+				batchState.baseBranch,
+				encounteredRepoRoots.keys(),
+			);
+			const hasSuccess = batchState.succeededTasks > 0;
+			const outcomeLine =
+				`  ${batchState.succeededTasks}/${batchState.totalTasks} tasks succeeded` +
+				(batchState.skippedTasks > 0 ? `, ${batchState.skippedTasks} skipped` : "") +
+				"\n";
+			const nextStep =
+				hasSuccess && branchState.kind === "ahead"
+					? `Ready for integration. Run orch_integrate() or review first.`
+					: branchState.kind === "ahead"
+						? `⚠️ No task succeeded, yet ${branchState.detail} — partial work was merged; inspect before integrating.`
+						: branchState.kind === "unknown"
+							? `⚠️ Could not verify the orch branch (${branchState.detail}). Inspect before integrating.`
+							: `Nothing to integrate: ${branchState.detail}.`;
 			emitAlert({
 				category: "batch-complete",
 				summary:
-					`✅ Batch ${batchState.batchId} completed\n` +
-					`  ${batchState.succeededTasks}/${batchState.totalTasks} tasks succeeded\n` +
+					`${hasSuccess ? "✅" : "⚠️"} Batch ${batchState.batchId} completed\n` +
+					outcomeLine +
 					`  ${batchState.taskLevelWaveCount ?? batchState.totalWaves} wave(s), duration: ${durationStr}\n` +
-					`  Merged to orch branch: ${batchState.orchBranch}\n\n` +
-					`Ready for integration. Run orch_integrate() or review first.`,
+					`  Orch branch ${batchState.orchBranch}: ${branchState.detail}\n\n` +
+					nextStep,
 				context: {
 					batchProgress: buildBatchProgressSnapshot(batchState),
 					batchDurationMs,