taskplane 0.6.1 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,21 +6,459 @@ import { existsSync, readdirSync, readFileSync, unlinkSync } from "fs";
6
6
  import { join, resolve } from "path";
7
7
 
8
8
  import { formatDiscoveryResults, runDiscovery } from "./discovery.ts";
9
- import { execLog, executeWave, tmuxKillSession } from "./execution.ts";
9
+ import { computeTransitiveDependents, execLog, executeLane, executeWave, tmuxKillSession } from "./execution.ts";
10
10
  import type { MonitorUpdateCallback } from "./execution.ts";
11
+ // classifyExit no longer called directly — Tier 0 uses exitDiagnostic.classification
12
+ // from the diagnostic-reports pipeline (populated by assembleDiagnosticInput).
11
13
  import { getCurrentBranch, runGit } from "./git.ts";
12
- import { attemptAutoIntegration, mergeWaveByRepo } from "./merge.ts";
13
- import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolicy, formatRepoMergeSummary, ORCH_MESSAGES } from "./messages.ts";
14
+ import { mergeWaveByRepo } from "./merge.ts";
15
+ import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolicy, extractFailedRepoId, formatRepoMergeSummary, ORCH_MESSAGES } from "./messages.ts";
14
16
  import type { CleanupGateRepoFailure } from "./messages.ts";
15
17
  import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-reports.ts";
16
18
  import { resolveOperatorId } from "./naming.ts";
17
- import { applyPartialProgressToOutcomes, deleteBatchState, loadBatchHistory, persistRuntimeState, saveBatchHistory, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
19
+ import { applyPartialProgressToOutcomes, buildTier0EventBase, deleteBatchState, emitEngineEvent, emitTier0Event, loadBatchHistory, persistRuntimeState, saveBatchHistory, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
18
20
  import { listOrchSessions } from "./sessions.ts";
19
- import { defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId } from "./types.ts";
20
- import type { AllocatedLane, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, TaskRunnerConfig, TokenCounts, WorkspaceConfig } from "./types.ts";
21
+ import { buildEngineEventBase, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
22
+ import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, TaskRunnerConfig, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
21
23
  import { buildDependencyGraph, computeWaves, resolveBaseBranch, resolveRepoRoot, validateGraph } from "./waves.ts";
22
24
  import { deleteBranchBestEffort, forceCleanupWorktree, formatPreflightResults, listWorktrees, preserveFailedLaneProgress, removeAllWorktrees, removeWorktree, runPreflight, safeResetWorktree, sleepSync } from "./worktree.ts";
23
25
 
26
+ // ── Tier 0: Automatic Recovery Helpers (TP-039) ─────────────────────
27
+
28
+ /**
29
+ * Emit a `tier0_escalation` event with a typed `EscalationContext` payload.
30
+ *
31
+ * Called at every exhaustion path alongside the existing `tier0_recovery_exhausted`
32
+ * event. The escalation event carries a structured payload for the future
33
+ * supervisor agent (TP-041). In Tier 0, no automated action is taken on the
34
+ * escalation — the engine falls through to its existing pause behaviour.
35
+ *
36
+ * @since TP-039
37
+ */
38
+ function emitTier0Escalation(
39
+ stateRoot: string,
40
+ batchId: string,
41
+ waveIndex: number,
42
+ pattern: Tier0EscalationPattern,
43
+ attempts: number,
44
+ maxAttempts: number,
45
+ lastError: string,
46
+ affectedTasks: string[],
47
+ suggestion: string,
48
+ extra?: Partial<Pick<import("./persistence.ts").Tier0Event, "taskId" | "laneNumber" | "repoId" | "classification" | "scopeKey">>,
49
+ ): void {
50
+ const escalation: EscalationContext = {
51
+ pattern,
52
+ attempts,
53
+ maxAttempts,
54
+ lastError,
55
+ affectedTasks,
56
+ suggestion,
57
+ };
58
+ emitTier0Event(stateRoot, {
59
+ ...buildTier0EventBase("tier0_escalation", batchId, waveIndex, pattern, attempts, maxAttempts),
60
+ ...extra,
61
+ escalation,
62
+ });
63
+ }
64
+
65
+ /**
66
+ * Attempt automatic retry for failed tasks with retryable exit classifications.
67
+ *
68
+ * After a wave completes, this function inspects each failed task's canonical
69
+ * `exitDiagnostic.classification` and re-executes the task if:
70
+ * - The classification is in TIER0_RETRYABLE_CLASSIFICATIONS (api_error, process_crash, session_vanished)
71
+ * - The retry budget for this scope has not been exhausted
72
+ *
73
+ * Partial progress is preserved before retry. On success, the task is moved from
74
+ * failedTaskIds to succeededTaskIds and the waveResult counts are updated in-place.
75
+ *
76
+ * @returns Object with retried count and updated task outcomes
77
+ */
78
+ async function attemptWorkerCrashRetry(
79
+ waveResult: WaveExecutionResult,
80
+ waveIdx: number,
81
+ batchState: OrchBatchRuntimeState,
82
+ orchConfig: OrchestratorConfig,
83
+ repoRoot: string,
84
+ workspaceConfig: WorkspaceConfig | null | undefined,
85
+ allTaskOutcomes: LaneTaskOutcome[],
86
+ onNotify: (message: string, level: "info" | "warning" | "error") => void,
87
+ stateRoot: string,
88
+ ): Promise<{ retriedCount: number; succeededRetries: string[]; failedRetries: string[] }> {
89
+ if (!batchState.resilience) {
90
+ batchState.resilience = defaultResilienceState();
91
+ }
92
+
93
+ const budget = TIER0_RETRY_BUDGETS.worker_crash;
94
+ const succeededRetries: string[] = [];
95
+ const failedRetries: string[] = [];
96
+ let retriedCount = 0;
97
+
98
+ // Build a map from taskId → lane for re-execution
99
+ const taskToLane = new Map<string, AllocatedLane>();
100
+ for (const lane of waveResult.allocatedLanes) {
101
+ for (const task of lane.tasks) {
102
+ taskToLane.set(task.taskId, lane);
103
+ }
104
+ }
105
+
106
+ // Check each failed task for retryability
107
+ for (const taskId of [...waveResult.failedTaskIds]) {
108
+ const lane = taskToLane.get(taskId);
109
+ if (!lane) continue;
110
+
111
+ // Find the task outcome to get exit info
112
+ const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
113
+ if (!outcome) continue;
114
+
115
+ // Use the canonical exit diagnostic classification when available.
116
+ // If exitDiagnostic is not populated (executeLane doesn't set it),
117
+ // we conservatively skip auto-retry rather than synthesizing a
118
+ // classification from incomplete data — which could incorrectly
119
+ // retry non-retryable failures (e.g., deterministic task errors).
120
+ const classification = outcome.exitDiagnostic?.classification;
121
+
122
+ if (!classification) {
123
+ execLog("batch", batchState.batchId,
124
+ `tier0: task ${taskId} has no exit diagnostic classification — skipping auto-retry (conservative)`,
125
+ );
126
+ continue;
127
+ }
128
+
129
+ // Check if retryable
130
+ if (!TIER0_RETRYABLE_CLASSIFICATIONS.has(classification)) {
131
+ execLog("batch", batchState.batchId,
132
+ `tier0: task ${taskId} exit classification "${classification}" is not retryable — skipping`,
133
+ );
134
+ continue;
135
+ }
136
+
137
+ // Check retry budget
138
+ const scopeKey = tier0ScopeKey("worker_crash", taskId, waveIdx);
139
+ const currentCount = batchState.resilience.retryCountByScope[scopeKey] ?? 0;
140
+ if (currentCount >= budget.maxRetries) {
141
+ execLog("batch", batchState.batchId,
142
+ `tier0: task ${taskId} retry budget exhausted (${currentCount}/${budget.maxRetries}) — skipping`,
143
+ { scopeKey },
144
+ );
145
+ // Emit exhausted event
146
+ emitTier0Event(stateRoot, {
147
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "worker_crash", currentCount, budget.maxRetries),
148
+ taskId,
149
+ laneNumber: lane.laneNumber,
150
+ repoId: lane.repoId ?? null,
151
+ classification,
152
+ error: `Retry budget exhausted for task ${taskId} (${classification})`,
153
+ scopeKey,
154
+ affectedTaskIds: [taskId],
155
+ suggestion: `Task ${taskId} failed with ${classification} and exhausted ${budget.maxRetries} retry attempt(s). Consider investigating the root cause or manually re-running the task.`,
156
+ });
157
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "worker_crash", currentCount, budget.maxRetries,
158
+ `Retry budget exhausted for task ${taskId} (${classification})`, [taskId],
159
+ `Task ${taskId} failed with ${classification} and exhausted ${budget.maxRetries} retry attempt(s). Consider investigating the root cause or manually re-running the task.`,
160
+ { taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
161
+ );
162
+ continue;
163
+ }
164
+
165
+ // Increment retry counter
166
+ batchState.resilience.retryCountByScope[scopeKey] = currentCount + 1;
167
+ retriedCount++;
168
+
169
+ execLog("batch", batchState.batchId,
170
+ `tier0: retrying task ${taskId} (worker_crash, attempt ${currentCount + 1}/${budget.maxRetries}, classification=${classification})`,
171
+ { scopeKey, classification },
172
+ );
173
+ onNotify(
174
+ `🔄 Tier 0: Retrying task ${taskId} (${classification}, attempt ${currentCount + 1}/${budget.maxRetries})`,
175
+ "info",
176
+ );
177
+
178
+ // Emit attempt event
179
+ emitTier0Event(stateRoot, {
180
+ ...buildTier0EventBase("tier0_recovery_attempt", batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries),
181
+ taskId,
182
+ laneNumber: lane.laneNumber,
183
+ repoId: lane.repoId ?? null,
184
+ classification,
185
+ cooldownMs: budget.cooldownMs,
186
+ scopeKey,
187
+ });
188
+
189
+ // Cooldown before retry
190
+ if (budget.cooldownMs > 0) {
191
+ sleepSync(budget.cooldownMs);
192
+ }
193
+
194
+ // Find the specific AllocatedTask
195
+ const allocatedTask = lane.tasks.find(t => t.taskId === taskId);
196
+ if (!allocatedTask) continue;
197
+
198
+ // Re-execute: create a single-task lane config for executeLane
199
+ const retryLane: AllocatedLane = {
200
+ ...lane,
201
+ tasks: [allocatedTask],
202
+ };
203
+
204
+ const isWsMode = !!workspaceConfig;
205
+ const wsRoot = workspaceConfig
206
+ ? resolve(workspaceConfig.configPath, "..", "..")
207
+ : undefined;
208
+
209
+ try {
210
+ // Use a fresh pause signal for the retry — the batch pauseSignal
211
+ // may be paused due to stop-wave policy, but Tier 0 retry should
212
+ // attempt recovery before the stop decision takes effect (R002-4).
213
+ const retryPauseSignal = { paused: false };
214
+ const retryResult = await executeLane(
215
+ retryLane,
216
+ orchConfig,
217
+ repoRoot,
218
+ retryPauseSignal,
219
+ wsRoot,
220
+ isWsMode,
221
+ );
222
+
223
+ const retryOutcome = retryResult.tasks[0];
224
+ if (retryOutcome && retryOutcome.status === "succeeded") {
225
+ succeededRetries.push(taskId);
226
+
227
+ // Update waveResult: move from failed to succeeded
228
+ const failIdx = waveResult.failedTaskIds.indexOf(taskId);
229
+ if (failIdx !== -1) waveResult.failedTaskIds.splice(failIdx, 1);
230
+ waveResult.succeededTaskIds.push(taskId);
231
+
232
+ // Update lane results — replace the failed task outcome
233
+ for (const lr of waveResult.laneResults) {
234
+ const taskIdx = lr.tasks.findIndex(t => t.taskId === taskId);
235
+ if (taskIdx !== -1) {
236
+ lr.tasks[taskIdx] = retryOutcome;
237
+ break;
238
+ }
239
+ }
240
+
241
+ // Update allTaskOutcomes
242
+ upsertTaskOutcome(allTaskOutcomes, retryOutcome);
243
+
244
+ execLog("batch", batchState.batchId,
245
+ `tier0: task ${taskId} retry succeeded`,
246
+ { scopeKey },
247
+ );
248
+ onNotify(
249
+ `✅ Tier 0: Task ${taskId} retry succeeded`,
250
+ "info",
251
+ );
252
+
253
+ // Emit success event
254
+ emitTier0Event(stateRoot, {
255
+ ...buildTier0EventBase("tier0_recovery_success", batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries),
256
+ taskId,
257
+ laneNumber: lane.laneNumber,
258
+ repoId: lane.repoId ?? null,
259
+ classification,
260
+ resolution: `Task ${taskId} succeeded on retry attempt ${currentCount + 1}`,
261
+ scopeKey,
262
+ });
263
+ } else {
264
+ failedRetries.push(taskId);
265
+ if (retryOutcome) {
266
+ upsertTaskOutcome(allTaskOutcomes, retryOutcome);
267
+ }
268
+ execLog("batch", batchState.batchId,
269
+ `tier0: task ${taskId} retry failed again`,
270
+ { scopeKey, exitReason: retryOutcome?.exitReason },
271
+ );
272
+
273
+ // Emit exhausted event (retry failed and budget now consumed)
274
+ const retryFailError = retryOutcome?.exitReason ?? `Task ${taskId} retry failed again`;
275
+ const retryFailSuggestion = `Task ${taskId} failed again after retry (${classification}). The failure may be persistent — investigate task logs.`;
276
+ emitTier0Event(stateRoot, {
277
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries),
278
+ taskId,
279
+ laneNumber: lane.laneNumber,
280
+ repoId: lane.repoId ?? null,
281
+ classification,
282
+ error: retryFailError,
283
+ scopeKey,
284
+ affectedTaskIds: [taskId],
285
+ suggestion: retryFailSuggestion,
286
+ });
287
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries,
288
+ retryFailError, [taskId], retryFailSuggestion,
289
+ { taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
290
+ );
291
+ }
292
+ } catch (err: unknown) {
293
+ failedRetries.push(taskId);
294
+ const errMsg = err instanceof Error ? err.message : String(err);
295
+ execLog("batch", batchState.batchId,
296
+ `tier0: task ${taskId} retry threw error: ${errMsg}`,
297
+ { scopeKey },
298
+ );
299
+
300
+ // Emit exhausted event for exception during retry
301
+ const exceptionSuggestion = `Task ${taskId} retry threw an exception: ${errMsg}. Investigate the execution environment.`;
302
+ emitTier0Event(stateRoot, {
303
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries),
304
+ taskId,
305
+ laneNumber: lane.laneNumber,
306
+ repoId: lane.repoId ?? null,
307
+ classification,
308
+ error: errMsg,
309
+ scopeKey,
310
+ affectedTaskIds: [taskId],
311
+ suggestion: exceptionSuggestion,
312
+ });
313
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries,
314
+ errMsg, [taskId], exceptionSuggestion,
315
+ { taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
316
+ );
317
+ }
318
+ }
319
+
320
+ // Recalculate wave-level status if retries changed outcomes.
321
+ // NOTE: Batch-level counters (succeededTasks, failedTasks) are NOT updated
322
+ // here — the caller accumulates them from waveResult AFTER retry so that
323
+ // counts are only applied once (R002-2 fix).
324
+ if (succeededRetries.length > 0) {
325
+ if (waveResult.failedTaskIds.length === 0) {
326
+ waveResult.overallStatus = "succeeded";
327
+ waveResult.stoppedEarly = false;
328
+ } else if (waveResult.succeededTaskIds.length > 0) {
329
+ waveResult.overallStatus = "partial";
330
+ }
331
+ }
332
+
333
+ return { retriedCount, succeededRetries, failedRetries };
334
+ }
335
+
336
+ /**
337
+ * Attempt stale worktree recovery when lane allocation fails with ALLOC_WORKTREE_FAILED.
338
+ *
339
+ * Forces cleanup of all matching worktrees, prunes git state, then retries
340
+ * the wave execution.
341
+ *
342
+ * @returns The retry waveResult, or null if recovery was not attempted
343
+ */
344
+ async function attemptStaleWorktreeRecovery(
345
+ waveResult: WaveExecutionResult,
346
+ waveTasks: string[],
347
+ waveIdx: number,
348
+ discovery: DiscoveryResult,
349
+ orchConfig: OrchestratorConfig,
350
+ repoRoot: string,
351
+ batchState: OrchBatchRuntimeState,
352
+ depGraph: ReturnType<typeof buildDependencyGraph>,
353
+ workspaceConfig: WorkspaceConfig | null | undefined,
354
+ onMonitorUpdate: MonitorUpdateCallback | undefined,
355
+ onLanesAllocated: (lanes: AllocatedLane[]) => void,
356
+ stateRoot: string,
357
+ ): Promise<WaveExecutionResult | null> {
358
+ // Only attempt recovery for ALLOC_WORKTREE_FAILED
359
+ if (!waveResult.allocationError || waveResult.allocationError.code !== "ALLOC_WORKTREE_FAILED") {
360
+ return null;
361
+ }
362
+
363
+ if (!batchState.resilience) {
364
+ batchState.resilience = defaultResilienceState();
365
+ }
366
+
367
+ const budget = TIER0_RETRY_BUDGETS.stale_worktree;
368
+ const scopeKey = tier0WaveScopeKey("stale_worktree", waveIdx);
369
+ const currentCount = batchState.resilience.retryCountByScope[scopeKey] ?? 0;
370
+
371
+ if (currentCount >= budget.maxRetries) {
372
+ execLog("batch", batchState.batchId,
373
+ `tier0: stale worktree retry budget exhausted (${currentCount}/${budget.maxRetries})`,
374
+ { scopeKey },
375
+ );
376
+ const staleExhaustedError = waveResult.allocationError.message;
377
+ const staleExhaustedSuggestion = `Stale worktree cleanup exhausted ${budget.maxRetries} retry(s). Manually remove worktrees and prune git state.`;
378
+ emitTier0Event(stateRoot, {
379
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "stale_worktree", currentCount, budget.maxRetries),
380
+ repoId: null, // wave-scoped
381
+ error: staleExhaustedError,
382
+ scopeKey,
383
+ affectedTaskIds: waveTasks,
384
+ suggestion: staleExhaustedSuggestion,
385
+ });
386
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "stale_worktree", currentCount, budget.maxRetries,
387
+ staleExhaustedError, waveTasks, staleExhaustedSuggestion,
388
+ { repoId: null, scopeKey },
389
+ );
390
+ return null;
391
+ }
392
+
393
+ batchState.resilience.retryCountByScope[scopeKey] = currentCount + 1;
394
+
395
+ execLog("batch", batchState.batchId,
396
+ `tier0: attempting stale worktree recovery (attempt ${currentCount + 1}/${budget.maxRetries})`,
397
+ { scopeKey, allocationError: waveResult.allocationError.message },
398
+ );
399
+
400
+ // Emit attempt event
401
+ emitTier0Event(stateRoot, {
402
+ ...buildTier0EventBase("tier0_recovery_attempt", batchState.batchId, waveIdx, "stale_worktree", currentCount + 1, budget.maxRetries),
403
+ repoId: null, // wave-scoped: allocation failure may span multiple repos
404
+ classification: waveResult.allocationError.code,
405
+ cooldownMs: budget.cooldownMs,
406
+ scopeKey,
407
+ });
408
+
409
+ // Force cleanup: remove all worktrees for this batch and prune.
410
+ // In workspace mode, iterate ALL workspace repos — allocation failures
411
+ // can come from non-default repos (R002-3 fix).
412
+ const prefix = orchConfig.orchestrator.worktree_prefix;
413
+ const opId = resolveOperatorId(orchConfig);
414
+
415
+ const repoRootsToClean: string[] = [repoRoot];
416
+ if (workspaceConfig) {
417
+ for (const [, repoConf] of workspaceConfig.repos) {
418
+ if (repoConf.path !== repoRoot && !repoRootsToClean.includes(repoConf.path)) {
419
+ repoRootsToClean.push(repoConf.path);
420
+ }
421
+ }
422
+ }
423
+
424
+ for (const cleanRoot of repoRootsToClean) {
425
+ const existingWorktrees = listWorktrees(prefix, cleanRoot, opId, batchState.batchId);
426
+ for (const wt of existingWorktrees) {
427
+ forceCleanupWorktree(wt, cleanRoot, batchState.batchId);
428
+ }
429
+ // Also prune git worktree state in case of orphaned references
430
+ runGit(["worktree", "prune"], cleanRoot);
431
+ }
432
+
433
+ // Cooldown before retry
434
+ if (budget.cooldownMs > 0) {
435
+ sleepSync(budget.cooldownMs);
436
+ }
437
+
438
+ // Retry the wave execution
439
+ execLog("batch", batchState.batchId,
440
+ `tier0: retrying wave ${waveIdx + 1} after stale worktree cleanup`,
441
+ );
442
+
443
+ const retryResult = await executeWave(
444
+ waveTasks,
445
+ waveIdx + 1,
446
+ discovery.pending,
447
+ orchConfig,
448
+ repoRoot,
449
+ batchState.batchId,
450
+ batchState.pauseSignal,
451
+ depGraph,
452
+ batchState.orchBranch,
453
+ onMonitorUpdate,
454
+ onLanesAllocated,
455
+ workspaceConfig,
456
+ );
457
+
458
+ return retryResult;
459
+ }
460
+
461
+
24
462
  // ── /orch Execution Engine ───────────────────────────────────────────
25
463
 
26
464
  /**
@@ -37,6 +475,8 @@ import { deleteBranchBestEffort, forceCleanupWorktree, formatPreflightResults, l
37
475
  * @param onMonitorUpdate - Optional callback for dashboard updates
38
476
  * @param workspaceConfig - Workspace configuration for repo routing (null = repo mode)
39
477
  * @param workspaceRoot - Workspace root for resolving task area paths (defaults to cwd)
478
+ * @param agentRoot - Agent root for config resolution
479
+ * @param onEngineEvent - Optional callback for engine lifecycle events (TP-040)
40
480
  */
41
481
  export async function executeOrchBatch(
42
482
  args: string,
@@ -49,17 +489,54 @@ export async function executeOrchBatch(
49
489
  workspaceConfig?: WorkspaceConfig | null,
50
490
  workspaceRoot?: string,
51
491
  agentRoot?: string,
492
+ onEngineEvent?: EngineEventCallback | null,
52
493
  ): Promise<void> {
53
494
  const repoRoot = cwd;
54
495
  // State files (.pi/batch-state.json, lane-state, etc.) belong in the workspace root,
55
496
  // which is where .pi/ config lives. In repo mode, workspaceRoot === repoRoot.
56
497
  const stateRoot = workspaceRoot ?? cwd;
57
498
 
499
+ // ── TP-040: Engine event emission helper ─────────────────────
500
+ // Closure over stateRoot and onEngineEvent to keep emit calls terse.
501
+ // batchState.batchId is read at call time (it's set in Phase 1).
502
+ const emitEvent: typeof emitEngineEvent = (sr, event, cb) => emitEngineEvent(sr, event, cb);
503
+
504
+ // ── TP-040 R002: Terminal event emission helper ──────────────
505
+ // Routes all early-return and terminal paths through consistent event
506
+ // emission so external consumers always receive a deterministic terminal
507
+ // signal (batch_complete for completed/failed, batch_paused for paused/stopped).
508
+ // Uses a guard flag to enforce one-transition/one-event semantics — once a
509
+ // terminal event has been emitted, subsequent calls are no-ops.
510
+ let terminalEventEmitted = false;
511
+ const emitTerminalEvent = (reason?: string): void => {
512
+ if (terminalEventEmitted) return;
513
+ terminalEventEmitted = true;
514
+ if (batchState.phase === "completed" || batchState.phase === "failed") {
515
+ emitEvent(stateRoot, {
516
+ ...buildEngineEventBase("batch_complete", batchState.batchId, batchState.currentWaveIndex, batchState.phase),
517
+ succeededTasks: batchState.succeededTasks,
518
+ failedTasks: batchState.failedTasks,
519
+ skippedTasks: batchState.skippedTasks,
520
+ blockedTasks: batchState.blockedTasks,
521
+ batchDurationMs: batchState.endedAt ? batchState.endedAt - batchState.startedAt : undefined,
522
+ }, onEngineEvent);
523
+ } else if (batchState.phase === "paused" || batchState.phase === "stopped") {
524
+ emitEvent(stateRoot, {
525
+ ...buildEngineEventBase("batch_paused", batchState.batchId, batchState.currentWaveIndex, batchState.phase),
526
+ reason: reason || (batchState.errors.length > 0 ? batchState.errors[batchState.errors.length - 1] : "paused"),
527
+ failedTasks: batchState.failedTasks,
528
+ }, onEngineEvent);
529
+ }
530
+ };
531
+
58
532
  // ── Phase 1: Planning ────────────────────────────────────────
59
533
  batchState.phase = "planning";
60
534
  batchState.batchId = generateBatchId();
61
- batchState.startedAt = Date.now();
62
- batchState.pauseSignal = { paused: false };
535
+ // Preserve startedAt if set during "launching" phase (TP-040)
536
+ if (!batchState.startedAt) batchState.startedAt = Date.now();
537
+ // Preserve pauseSignal if already set during "launching" phase (TP-040)
538
+ // — e.g., /orch-pause issued between /orch return and engine start
539
+ if (!batchState.pauseSignal?.paused) batchState.pauseSignal = { paused: false };
63
540
  batchState.mergeResults = [];
64
541
  batchState.mode = workspaceConfig ? "workspace" : "repo";
65
542
 
@@ -70,6 +547,7 @@ export async function executeOrchBatch(
70
547
  batchState.endedAt = Date.now();
71
548
  batchState.errors.push("Cannot determine current branch (detached HEAD or not a git repo)");
72
549
  onNotify("❌ Cannot determine current branch. Ensure HEAD is on a branch (not detached).", "error");
550
+ emitTerminalEvent();
73
551
  return;
74
552
  }
75
553
  batchState.baseBranch = detectedBranch;
@@ -105,6 +583,7 @@ export async function executeOrchBatch(
105
583
  batchState.phase = "failed";
106
584
  batchState.endedAt = Date.now();
107
585
  batchState.errors.push("Preflight check failed");
586
+ emitTerminalEvent();
108
587
  return;
109
588
  }
110
589
 
@@ -147,6 +626,7 @@ export async function executeOrchBatch(
147
626
  "info",
148
627
  );
149
628
  }
629
+ emitTerminalEvent();
150
630
  return;
151
631
  }
152
632
 
@@ -154,6 +634,7 @@ export async function executeOrchBatch(
154
634
  batchState.phase = "completed";
155
635
  batchState.endedAt = Date.now();
156
636
  onNotify("No pending tasks found. Nothing to execute.", "info");
637
+ emitTerminalEvent();
157
638
  return;
158
639
  }
159
640
 
@@ -169,6 +650,7 @@ export async function executeOrchBatch(
169
650
  const errMsgs = validation.errors.map(e => `[${e.code}] ${e.message}`).join("\n");
170
651
  batchState.errors.push(`Graph validation failed:\n${errMsgs}`);
171
652
  onNotify(`❌ Dependency graph errors:\n${errMsgs}`, "error");
653
+ emitTerminalEvent();
172
654
  return;
173
655
  }
174
656
 
@@ -180,6 +662,7 @@ export async function executeOrchBatch(
180
662
  const errMsgs = waveErrors.map(e => `[${e.code}] ${e.message}`).join("\n");
181
663
  batchState.errors.push(`Wave computation failed:\n${errMsgs}`);
182
664
  onNotify(`❌ Wave computation errors:\n${errMsgs}`, "error");
665
+ emitTerminalEvent();
183
666
  return;
184
667
  }
185
668
 
@@ -220,7 +703,7 @@ export async function executeOrchBatch(
220
703
  break;
221
704
  }
222
705
  }
223
- if (orchBranchFailed) return;
706
+ if (orchBranchFailed) { emitTerminalEvent(); return; }
224
707
  } else {
225
708
  const branchResult = runGit(["branch", orchBranch, batchState.baseBranch], repoRoot);
226
709
  if (!branchResult.ok) {
@@ -229,6 +712,7 @@ export async function executeOrchBatch(
229
712
  const errDetail = branchResult.stderr || branchResult.stdout || "unknown error";
230
713
  batchState.errors.push(`Failed to create orch branch '${orchBranch}': ${errDetail}`);
231
714
  onNotify(`❌ Failed to create orch branch '${orchBranch}': ${errDetail}`, "error");
715
+ emitTerminalEvent();
232
716
  return;
233
717
  }
234
718
  execLog("batch", batchState.batchId, "created orch branch", { orchBranch, baseBranch: batchState.baseBranch });
@@ -254,6 +738,8 @@ export async function executeOrchBatch(
254
738
  onNotify(`⏸️ Batch paused before wave ${waveIdx + 1}. Resume not yet implemented (TS-009).`, "warning");
255
739
  // ── TS-009: Persist state on pause ──
256
740
  persistRuntimeState("pause-before-wave", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
741
+ // TP-040: Emit batch_paused event (via terminal helper for dedup)
742
+ emitTerminalEvent(`Paused before wave ${waveIdx + 1}`);
257
743
  break;
258
744
  }
259
745
 
@@ -288,6 +774,13 @@ export async function executeOrchBatch(
288
774
  "info",
289
775
  );
290
776
 
777
+ // TP-040: Emit wave_start event
778
+ emitEvent(stateRoot, {
779
+ ...buildEngineEventBase("wave_start", batchState.batchId, waveIdx, batchState.phase),
780
+ taskIds: waveTasks,
781
+ laneCount: Math.min(waveTasks.length, orchConfig.orchestrator.max_lanes),
782
+ }, onEngineEvent);
783
+
291
784
  const handleWaveMonitorUpdate: MonitorUpdateCallback = (monitorState) => {
292
785
  const changed = syncTaskOutcomesFromMonitor(monitorState, allTaskOutcomes);
293
786
  if (changed) {
@@ -297,7 +790,20 @@ export async function executeOrchBatch(
297
790
  };
298
791
 
299
792
  // Execute the wave
300
- const waveResult = await executeWave(
793
+ const onLanesAllocatedCb = (lanes: AllocatedLane[]) => {
794
+ latestAllocatedLanes = lanes;
795
+ batchState.currentLanes = lanes;
796
+ // TP-029: Track repos from newly allocated lanes for cleanup coverage
797
+ for (const lane of lanes) {
798
+ const laneRepoRoot = resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig);
799
+ encounteredRepoRoots.set(laneRepoRoot, lane.repoId);
800
+ }
801
+ if (seedPendingOutcomesForAllocatedLanes(lanes, allTaskOutcomes)) {
802
+ persistRuntimeState("wave-lanes-allocated", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
803
+ }
804
+ };
805
+
806
+ let waveResult = await executeWave(
301
807
  waveTasks,
302
808
  waveIdx + 1,
303
809
  discovery.pending,
@@ -308,21 +814,66 @@ export async function executeOrchBatch(
308
814
  depGraph,
309
815
  batchState.orchBranch,
310
816
  handleWaveMonitorUpdate,
311
- (lanes) => {
312
- latestAllocatedLanes = lanes;
313
- batchState.currentLanes = lanes;
314
- // TP-029: Track repos from newly allocated lanes for cleanup coverage
315
- for (const lane of lanes) {
316
- const laneRepoRoot = resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig);
317
- encounteredRepoRoots.set(laneRepoRoot, lane.repoId);
318
- }
319
- if (seedPendingOutcomesForAllocatedLanes(lanes, allTaskOutcomes)) {
320
- persistRuntimeState("wave-lanes-allocated", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
321
- }
322
- },
817
+ onLanesAllocatedCb,
323
818
  workspaceConfig,
324
819
  );
325
820
 
821
+ // ── TP-039: Tier 0 — Stale worktree recovery ────────────
822
+ // If allocation failed with ALLOC_WORKTREE_FAILED, force cleanup
823
+ // and retry the entire wave execution once.
824
+ if (waveResult.allocationError?.code === "ALLOC_WORKTREE_FAILED") {
825
+ const retryResult = await attemptStaleWorktreeRecovery(
826
+ waveResult,
827
+ waveTasks,
828
+ waveIdx,
829
+ discovery,
830
+ orchConfig,
831
+ repoRoot,
832
+ batchState,
833
+ depGraph,
834
+ workspaceConfig,
835
+ handleWaveMonitorUpdate,
836
+ onLanesAllocatedCb,
837
+ stateRoot,
838
+ );
839
+ if (retryResult) {
840
+ const staleRecovered = !retryResult.allocationError;
841
+ onNotify(
842
+ `🔄 Tier 0: Stale worktree recovery ${staleRecovered ? "succeeded" : "failed"} for wave ${waveIdx + 1}`,
843
+ staleRecovered ? "info" : "warning",
844
+ );
845
+
846
+ // Emit success or exhausted event based on retry result
847
+ const staleScopeKey = tier0WaveScopeKey("stale_worktree", waveIdx);
848
+ const staleCount = batchState.resilience?.retryCountByScope[staleScopeKey] ?? 1;
849
+ if (staleRecovered) {
850
+ emitTier0Event(stateRoot, {
851
+ ...buildTier0EventBase("tier0_recovery_success", batchState.batchId, waveIdx, "stale_worktree", staleCount, TIER0_RETRY_BUDGETS.stale_worktree.maxRetries),
852
+ repoId: null, // wave-scoped
853
+ resolution: `Stale worktree cleanup succeeded — wave ${waveIdx + 1} re-executed successfully`,
854
+ scopeKey: staleScopeKey,
855
+ });
856
+ } else {
857
+ const staleRetryError = retryResult.allocationError?.message ?? "Allocation failed again after cleanup";
858
+ const staleRetrySuggestion = "Stale worktree cleanup did not resolve the allocation failure. Manually inspect and remove worktrees.";
859
+ emitTier0Event(stateRoot, {
860
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "stale_worktree", staleCount, TIER0_RETRY_BUDGETS.stale_worktree.maxRetries),
861
+ repoId: null, // wave-scoped
862
+ error: staleRetryError,
863
+ scopeKey: staleScopeKey,
864
+ affectedTaskIds: waveTasks,
865
+ suggestion: staleRetrySuggestion,
866
+ });
867
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "stale_worktree", staleCount, TIER0_RETRY_BUDGETS.stale_worktree.maxRetries,
868
+ staleRetryError, waveTasks, staleRetrySuggestion,
869
+ { repoId: null, scopeKey: staleScopeKey },
870
+ );
871
+ }
872
+
873
+ waveResult = retryResult;
874
+ }
875
+ }
876
+
326
877
  batchState.waveResults.push(waveResult);
327
878
  batchState.currentLanes = []; // Clear current lanes after wave completes
328
879
 
@@ -334,16 +885,98 @@ export async function executeOrchBatch(
334
885
  }
335
886
  }
336
887
 
337
- // Accumulate results
888
+ // ── TP-039: Tier 0 — Worker crash retry ─────────────────
889
+ // Run retry BEFORE accumulating counts and blocked tasks so that
890
+ // successfully retried tasks don't inflate failedTasks count and
891
+ // their dependents aren't incorrectly blocked (R002-2 fix).
892
+ if (waveResult.failedTaskIds.length > 0) {
893
+ const retryOutcome = await attemptWorkerCrashRetry(
894
+ waveResult,
895
+ waveIdx,
896
+ batchState,
897
+ orchConfig,
898
+ repoRoot,
899
+ workspaceConfig,
900
+ allTaskOutcomes,
901
+ onNotify,
902
+ stateRoot,
903
+ );
904
+ if (retryOutcome.succeededRetries.length > 0) {
905
+ // Recompute blockedTaskIds from remaining failures (R002-2).
906
+ // attemptWorkerCrashRetry already updated waveResult.failedTaskIds
907
+ // and waveResult.succeededTaskIds in-place.
908
+ if (waveResult.policyApplied === "skip-dependents" && waveResult.failedTaskIds.length > 0) {
909
+ const recomputed = computeTransitiveDependents(
910
+ new Set(waveResult.failedTaskIds),
911
+ depGraph,
912
+ );
913
+ waveResult.blockedTaskIds = [...recomputed].sort();
914
+ } else if (waveResult.failedTaskIds.length === 0) {
915
+ // All failures recovered — no blocked tasks
916
+ waveResult.blockedTaskIds = [];
917
+ }
918
+ }
919
+ if (retryOutcome.retriedCount > 0) {
920
+ // Persist updated state after retries
921
+ persistRuntimeState("tier0-worker-retry", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
922
+ }
923
+
924
+ // If stop-wave had paused the batch but Tier 0 retry recovered all
925
+ // failures, clear the policy-induced pause so subsequent waves can
926
+ // proceed. attemptWorkerCrashRetry already set stoppedEarly=false
927
+ // and overallStatus="succeeded" on the waveResult (R002-4 fix).
928
+ if (
929
+ waveResult.failedTaskIds.length === 0
930
+ && batchState.pauseSignal.paused
931
+ && waveResult.policyApplied === "stop-wave"
932
+ ) {
933
+ batchState.pauseSignal.paused = false;
934
+ execLog("batch", batchState.batchId,
935
+ `tier0: all failed tasks recovered — clearing stop-wave pause`,
936
+ );
937
+ onNotify(
938
+ `✅ Tier 0: All failed tasks recovered — batch continuing past stop-wave`,
939
+ "info",
940
+ );
941
+ }
942
+ }
943
+
944
+ // Accumulate results (after retry so counts reflect recovered tasks)
338
945
  batchState.succeededTasks += waveResult.succeededTaskIds.length;
339
946
  batchState.failedTasks += waveResult.failedTaskIds.length;
340
947
  batchState.skippedTasks += waveResult.skippedTaskIds.length;
341
948
 
342
- // Add newly blocked tasks
949
+ // Add newly blocked tasks (after retry so recovered tasks don't block dependents)
343
950
  for (const blocked of waveResult.blockedTaskIds) {
344
951
  batchState.blockedTaskIds.add(blocked);
345
952
  }
346
953
 
954
+ // ── TP-040: Emit task_complete / task_failed events ──────
955
+ // Emitted after Tier 0 retry so events reflect final status.
956
+ for (const taskId of waveResult.succeededTaskIds) {
957
+ const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
958
+ emitEvent(stateRoot, {
959
+ ...buildEngineEventBase("task_complete", batchState.batchId, waveIdx, batchState.phase),
960
+ taskId,
961
+ durationMs: outcome?.startTime && outcome?.endTime
962
+ ? outcome.endTime - outcome.startTime
963
+ : undefined,
964
+ outcome: "succeeded",
965
+ }, onEngineEvent);
966
+ }
967
+ for (const taskId of waveResult.failedTaskIds) {
968
+ const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
969
+ emitEvent(stateRoot, {
970
+ ...buildEngineEventBase("task_failed", batchState.batchId, waveIdx, batchState.phase),
971
+ taskId,
972
+ durationMs: outcome?.startTime && outcome?.endTime
973
+ ? outcome.endTime - outcome.startTime
974
+ : undefined,
975
+ reason: outcome?.exitReason || "unknown",
976
+ partialProgress: (outcome?.partialProgressCommits ?? 0) > 0,
977
+ }, onEngineEvent);
978
+ }
979
+
347
980
  // ── TS-009: Persist state after wave execution ──
348
981
  persistRuntimeState("wave-execution-complete", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
349
982
 
@@ -359,6 +992,10 @@ export async function executeOrchBatch(
359
992
  waveResult.failedTaskIds.length > 0 ? "warning" : "info",
360
993
  );
361
994
 
995
+ // NOTE: No explicit wave_complete event in the spec event set. The supervisor
996
+ // infers wave completion from the sequence of task_complete/task_failed events
997
+ // followed by merge_start or the next wave_start.
998
+
362
999
  // Check if we should stop based on task failure policy
363
1000
  if (waveResult.stoppedEarly) {
364
1001
  if (waveResult.policyApplied === "stop-all") {
@@ -366,6 +1003,8 @@ export async function executeOrchBatch(
366
1003
  // ── TS-009: Persist state on stop-all ──
367
1004
  persistRuntimeState("stop-all", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
368
1005
  onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-all"), "error");
1006
+ // TP-040: Emit batch_paused event (via terminal helper for dedup)
1007
+ emitTerminalEvent(`Stopped by stop-all policy at wave ${waveIdx + 1}`);
369
1008
  break;
370
1009
  }
371
1010
  if (waveResult.policyApplied === "stop-wave") {
@@ -373,6 +1012,8 @@ export async function executeOrchBatch(
373
1012
  // ── TS-009: Persist state on stop-wave ──
374
1013
  persistRuntimeState("stop-wave", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
375
1014
  onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-wave"), "error");
1015
+ // TP-040: Emit batch_paused event (via terminal helper for dedup)
1016
+ emitTerminalEvent(`Stopped by stop-wave policy at wave ${waveIdx + 1}`);
376
1017
  break;
377
1018
  }
378
1019
  }
@@ -411,6 +1052,11 @@ export async function executeOrchBatch(
411
1052
  // ── TS-009: Persist state on executing→merging transition ──
412
1053
  persistRuntimeState("merge-start", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
413
1054
  onNotify(ORCH_MESSAGES.orchMergeStart(waveIdx + 1, mergeableLaneCount), "info");
1055
+ // TP-040: Emit merge_start event
1056
+ emitEvent(stateRoot, {
1057
+ ...buildEngineEventBase("merge_start", batchState.batchId, waveIdx, batchState.phase),
1058
+ laneCount: mergeableLaneCount,
1059
+ }, onEngineEvent);
414
1060
 
415
1061
  mergeResult = mergeWaveByRepo(
416
1062
  waveResult.allocatedLanes,
@@ -474,12 +1120,27 @@ export async function executeOrchBatch(
474
1120
 
475
1121
  if (mergeResult.status === "succeeded") {
476
1122
  onNotify(ORCH_MESSAGES.orchMergeComplete(waveIdx + 1, mergedCount, mergeTotalSec), "info");
1123
+
1124
+ // TP-040: Emit merge_success event
1125
+ emitEvent(stateRoot, {
1126
+ ...buildEngineEventBase("merge_success", batchState.batchId, waveIdx, batchState.phase),
1127
+ laneCount: mergedCount,
1128
+ durationMs: mergeResult.totalDurationMs,
1129
+ totalWaves: rawWaves.length,
1130
+ }, onEngineEvent);
477
1131
  } else {
478
1132
  onNotify(
479
1133
  ORCH_MESSAGES.orchMergeFailed(waveIdx + 1, mergeResult.failedLane ?? 0, mergeResult.failureReason || "unknown"),
480
1134
  "error",
481
1135
  );
482
1136
 
1137
+ // TP-040: Emit merge_failed event
1138
+ emitEvent(stateRoot, {
1139
+ ...buildEngineEventBase("merge_failed", batchState.batchId, waveIdx, batchState.phase),
1140
+ laneNumber: mergeResult.failedLane ?? undefined,
1141
+ error: mergeResult.failureReason || "unknown",
1142
+ }, onEngineEvent);
1143
+
483
1144
  // Emit repo-divergence summary when partial is caused by cross-repo outcome differences
484
1145
  if (mergeResult.status === "partial") {
485
1146
  const repoSummary = formatRepoMergeSummary(mergeResult);
@@ -509,6 +1170,13 @@ export async function executeOrchBatch(
509
1170
  ORCH_MESSAGES.orchMergeFailed(waveIdx + 1, mergeResult.failedLane, mergeResult.failureReason || "unknown"),
510
1171
  "error",
511
1172
  );
1173
+
1174
+ // TP-040 R002: Emit merge_failed for mixed-outcome/no-mergeable-lane path
1175
+ emitEvent(stateRoot, {
1176
+ ...buildEngineEventBase("merge_failed", batchState.batchId, waveIdx, batchState.phase),
1177
+ laneNumber: mergeResult.failedLane,
1178
+ error: mergeResult.failureReason,
1179
+ }, onEngineEvent);
512
1180
  } else {
513
1181
  // No mergeable lanes and no mixed outcomes (e.g., only skipped tasks)
514
1182
  onNotify(ORCH_MESSAGES.orchMergeSkipped(waveIdx + 1), "info");
@@ -565,6 +1233,10 @@ export async function executeOrchBatch(
565
1233
  batchState.resilience = defaultResilienceState();
566
1234
  }
567
1235
 
1236
+ // Extract repoId and lane for event attribution before entering retry loop
1237
+ const mergeRepoId = extractFailedRepoId(mergeResult) ?? null;
1238
+ const mergeFailedLane = mergeResult.failedLane ?? undefined;
1239
+
568
1240
  const retryOutcome = applyMergeRetryLoop(
569
1241
  mergeResult,
570
1242
  waveIdx,
@@ -595,6 +1267,17 @@ export async function executeOrchBatch(
595
1267
  batchState.mergeResults[batchState.mergeResults.length - 1] = result;
596
1268
  },
597
1269
  sleep: sleepSync,
1270
+ // TP-039 R004: Emit attempt event only when retry is actually scheduled,
1271
+ // with accurate classification/attempt data from the retry decision.
1272
+ onRetryAttempt: (decision) => {
1273
+ emitTier0Event(stateRoot, {
1274
+ ...buildTier0EventBase("tier0_recovery_attempt", batchState.batchId, waveIdx, "merge_timeout", decision.currentAttempt, decision.maxAttempts),
1275
+ laneNumber: mergeFailedLane,
1276
+ repoId: mergeRepoId,
1277
+ classification: decision.classification,
1278
+ cooldownMs: decision.cooldownMs,
1279
+ });
1280
+ },
598
1281
  },
599
1282
  );
600
1283
 
@@ -602,6 +1285,17 @@ export async function executeOrchBatch(
602
1285
  mergeResult = retryOutcome.mergeResult;
603
1286
  batchState.phase = "executing";
604
1287
  persistRuntimeState("merge-retry-succeeded", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
1288
+
1289
+ // Emit merge retry success event
1290
+ emitTier0Event(stateRoot, {
1291
+ ...buildTier0EventBase("tier0_recovery_success", batchState.batchId, waveIdx, "merge_timeout", retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts),
1292
+ laneNumber: mergeFailedLane,
1293
+ repoId: mergeRepoId,
1294
+ classification: retryOutcome.classification ?? undefined,
1295
+ resolution: `Merge retry succeeded at wave ${waveIdx + 1}`,
1296
+ scopeKey: retryOutcome.scopeKey,
1297
+ });
1298
+
605
1299
  // Fall through to normal post-merge flow (worktree cleanup, etc.)
606
1300
  } else if (retryOutcome.kind === "safe_stop") {
607
1301
  mergeResult = retryOutcome.mergeResult;
@@ -609,6 +1303,24 @@ export async function executeOrchBatch(
609
1303
  batchState.errors.push(retryOutcome.errorMessage);
610
1304
  persistRuntimeState("merge-rollback-safe-stop", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
611
1305
  onNotify(retryOutcome.notifyMessage, "error");
1306
+
1307
+ // Emit merge safe-stop event (treated as exhausted — no further automatic recovery possible)
1308
+ const mergeSafeStopSuggestion = "Merge rollback failed — batch force-paused for manual recovery. Check .pi/verification/ for recovery commands.";
1309
+ emitTier0Event(stateRoot, {
1310
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "merge_timeout", retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts),
1311
+ laneNumber: mergeFailedLane,
1312
+ repoId: mergeRepoId,
1313
+ classification: retryOutcome.classification ?? undefined,
1314
+ error: retryOutcome.errorMessage,
1315
+ scopeKey: retryOutcome.scopeKey,
1316
+ suggestion: mergeSafeStopSuggestion,
1317
+ });
1318
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "merge_timeout",
1319
+ retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts,
1320
+ retryOutcome.errorMessage, [], mergeSafeStopSuggestion,
1321
+ { laneNumber: mergeFailedLane, repoId: mergeRepoId, classification: retryOutcome.classification ?? undefined, scopeKey: retryOutcome.scopeKey },
1322
+ );
1323
+
612
1324
  preserveWorktreesForResume = true;
613
1325
  break;
614
1326
  } else if (retryOutcome.kind === "exhausted") {
@@ -625,6 +1337,23 @@ export async function executeOrchBatch(
625
1337
  maxAttempts: retryOutcome.lastDecision.maxAttempts,
626
1338
  });
627
1339
 
1340
+ // Emit merge retry exhausted event
1341
+ const mergeExhaustedSuggestion = `Merge retry exhausted (${retryOutcome.classification ?? "unknown"}) after ${retryOutcome.lastDecision.currentAttempt} attempt(s). Investigate merge failure and retry manually.`;
1342
+ emitTier0Event(stateRoot, {
1343
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "merge_timeout", retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts),
1344
+ laneNumber: mergeFailedLane,
1345
+ repoId: mergeRepoId,
1346
+ classification: retryOutcome.classification ?? undefined,
1347
+ error: exhaustionMsg,
1348
+ scopeKey: retryOutcome.scopeKey,
1349
+ suggestion: mergeExhaustedSuggestion,
1350
+ });
1351
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "merge_timeout",
1352
+ retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts,
1353
+ exhaustionMsg, [], mergeExhaustedSuggestion,
1354
+ { laneNumber: mergeFailedLane, repoId: mergeRepoId, classification: retryOutcome.classification ?? undefined, scopeKey: retryOutcome.scopeKey },
1355
+ );
1356
+
628
1357
  batchState.phase = "paused";
629
1358
  batchState.errors.push(exhaustionMsg);
630
1359
  persistRuntimeState("merge-retry-exhausted", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
@@ -807,17 +1536,152 @@ export async function executeOrchBatch(
807
1536
  }
808
1537
 
809
1538
  if (cleanupGateFailures.length > 0) {
810
- const gatePolicyResult = computeCleanupGatePolicy(waveIdx, cleanupGateFailures);
1539
+ // ── TP-039: Tier 0 — Cleanup gate retry ──────────────
1540
+ // Before pausing, attempt one more force cleanup + prune
1541
+ // on the stale worktrees. This handles cases where the
1542
+ // first force cleanup partially succeeded (e.g., directory
1543
+ // removed but git state not yet pruned).
1544
+ if (!batchState.resilience) {
1545
+ batchState.resilience = defaultResilienceState();
1546
+ }
811
1547
 
812
- execLog("batch", batchState.batchId, `cleanup gate failed — pausing batch`, gatePolicyResult.logDetails);
1548
+ const cleanupBudget = TIER0_RETRY_BUDGETS.cleanup_gate;
1549
+ const cleanupScopeKey = tier0WaveScopeKey("cleanup_gate", waveIdx);
1550
+ const cleanupRetryCount = batchState.resilience.retryCountByScope[cleanupScopeKey] ?? 0;
813
1551
 
814
- batchState.phase = gatePolicyResult.targetPhase;
815
- batchState.errors.push(gatePolicyResult.errorMessage);
816
- persistRuntimeState(gatePolicyResult.persistTrigger, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
817
- onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
818
- // Preserve remaining worktrees for manual cleanup do NOT remove them
819
- preserveWorktreesForResume = true;
820
- break;
1552
+ if (cleanupRetryCount < cleanupBudget.maxRetries) {
1553
+ batchState.resilience.retryCountByScope[cleanupScopeKey] = cleanupRetryCount + 1;
1554
+
1555
+ execLog("batch", batchState.batchId,
1556
+ `tier0: retrying cleanup gate (attempt ${cleanupRetryCount + 1}/${cleanupBudget.maxRetries})`,
1557
+ { cleanupScopeKey, staleCount: cleanupGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0) },
1558
+ );
1559
+
1560
+ // Emit attempt event
1561
+ const staleWorktreeCount = cleanupGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0);
1562
+ emitTier0Event(stateRoot, {
1563
+ ...buildTier0EventBase("tier0_recovery_attempt", batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount + 1, cleanupBudget.maxRetries),
1564
+ repoId: null, // wave-scoped: cleanup gate spans all repos
1565
+ classification: `stale_worktrees:${staleWorktreeCount}`,
1566
+ cooldownMs: cleanupBudget.cooldownMs,
1567
+ scopeKey: cleanupScopeKey,
1568
+ });
1569
+
1570
+ if (cleanupBudget.cooldownMs > 0) {
1571
+ sleepSync(cleanupBudget.cooldownMs);
1572
+ }
1573
+
1574
+ // Force-cleanup each stale worktree again
1575
+ for (const failure of cleanupGateFailures) {
1576
+ const remaining = listWorktrees(resetPrefix, failure.repoRoot, resetOpId, batchState.batchId);
1577
+ for (const wt of remaining) {
1578
+ if (failure.staleWorktrees.includes(wt.path)) {
1579
+ forceCleanupWorktree(wt, failure.repoRoot, batchState.batchId);
1580
+ }
1581
+ }
1582
+ // Prune after force cleanup
1583
+ runGit(["worktree", "prune"], failure.repoRoot);
1584
+ }
1585
+
1586
+ // Re-check: are any worktrees still stale?
1587
+ const retriedGateFailures: CleanupGateRepoFailure[] = [];
1588
+ for (const failure of cleanupGateFailures) {
1589
+ const remaining = listWorktrees(resetPrefix, failure.repoRoot, resetOpId, batchState.batchId);
1590
+ const remainingPaths = new Set(remaining.map(wt => wt.path));
1591
+ const stillStale = failure.staleWorktrees.filter(p => remainingPaths.has(p));
1592
+ if (stillStale.length > 0) {
1593
+ retriedGateFailures.push({
1594
+ repoRoot: failure.repoRoot,
1595
+ repoId: failure.repoId,
1596
+ staleWorktrees: stillStale,
1597
+ });
1598
+ }
1599
+ }
1600
+
1601
+ if (retriedGateFailures.length === 0) {
1602
+ execLog("batch", batchState.batchId,
1603
+ `tier0: cleanup gate retry succeeded — all stale worktrees removed`,
1604
+ { cleanupScopeKey },
1605
+ );
1606
+ onNotify(
1607
+ `✅ Tier 0: Cleanup gate retry succeeded at wave ${waveIdx + 1} — continuing`,
1608
+ "info",
1609
+ );
1610
+
1611
+ // Emit success event
1612
+ emitTier0Event(stateRoot, {
1613
+ ...buildTier0EventBase("tier0_recovery_success", batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount + 1, cleanupBudget.maxRetries),
1614
+ repoId: null, // wave-scoped
1615
+ resolution: `Cleanup gate retry succeeded — all stale worktrees removed at wave ${waveIdx + 1}`,
1616
+ scopeKey: cleanupScopeKey,
1617
+ });
1618
+
1619
+ persistRuntimeState("tier0-cleanup-retry-success", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
1620
+ // Fall through to continue the wave loop (don't break)
1621
+ } else {
1622
+ // Retry failed — fall through to pausing
1623
+ const gatePolicyResult = computeCleanupGatePolicy(waveIdx, retriedGateFailures);
1624
+
1625
+ execLog("batch", batchState.batchId,
1626
+ `tier0: cleanup gate retry failed — still ${retriedGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0)} stale worktree(s), pausing batch`,
1627
+ gatePolicyResult.logDetails,
1628
+ );
1629
+
1630
+ const stillStaleCount = retriedGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0);
1631
+ const cleanupRetryError = `Cleanup gate retry failed — ${stillStaleCount} stale worktree(s) remain`;
1632
+ const cleanupRetrySuggestion = `Post-merge cleanup retry did not remove all stale worktrees. Manually remove the remaining ${stillStaleCount} worktree(s) and prune git state.`;
1633
+ const cleanupRetryAffected = retriedGateFailures.flatMap(f => f.staleWorktrees);
1634
+ // Emit exhausted event (retry attempted but failed)
1635
+ emitTier0Event(stateRoot, {
1636
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount + 1, cleanupBudget.maxRetries),
1637
+ repoId: null, // wave-scoped
1638
+ error: cleanupRetryError,
1639
+ scopeKey: cleanupScopeKey,
1640
+ affectedTaskIds: cleanupRetryAffected,
1641
+ suggestion: cleanupRetrySuggestion,
1642
+ });
1643
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount + 1, cleanupBudget.maxRetries,
1644
+ cleanupRetryError, cleanupRetryAffected, cleanupRetrySuggestion,
1645
+ { repoId: null, scopeKey: cleanupScopeKey },
1646
+ );
1647
+
1648
+ batchState.phase = gatePolicyResult.targetPhase;
1649
+ batchState.errors.push(gatePolicyResult.errorMessage);
1650
+ persistRuntimeState(gatePolicyResult.persistTrigger, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
1651
+ onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
1652
+ preserveWorktreesForResume = true;
1653
+ break;
1654
+ }
1655
+ } else {
1656
+ // Cleanup retry budget exhausted — pause immediately
1657
+ const gatePolicyResult = computeCleanupGatePolicy(waveIdx, cleanupGateFailures);
1658
+
1659
+ execLog("batch", batchState.batchId, `cleanup gate failed — pausing batch (retry budget exhausted)`, gatePolicyResult.logDetails);
1660
+
1661
+ // Emit exhausted event (budget already consumed from prior waves)
1662
+ const cleanupBudgetError = `Cleanup gate retry budget exhausted (${cleanupRetryCount}/${cleanupBudget.maxRetries})`;
1663
+ const cleanupBudgetSuggestion = `Cleanup gate retry budget was already consumed. Manually remove stale worktrees and prune git state.`;
1664
+ const cleanupBudgetAffected = cleanupGateFailures.flatMap(f => f.staleWorktrees);
1665
+ emitTier0Event(stateRoot, {
1666
+ ...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount, cleanupBudget.maxRetries),
1667
+ repoId: null, // wave-scoped
1668
+ error: cleanupBudgetError,
1669
+ scopeKey: cleanupScopeKey,
1670
+ affectedTaskIds: cleanupBudgetAffected,
1671
+ suggestion: cleanupBudgetSuggestion,
1672
+ });
1673
+ emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount, cleanupBudget.maxRetries,
1674
+ cleanupBudgetError, cleanupBudgetAffected, cleanupBudgetSuggestion,
1675
+ { repoId: null, scopeKey: cleanupScopeKey },
1676
+ );
1677
+
1678
+ batchState.phase = gatePolicyResult.targetPhase;
1679
+ batchState.errors.push(gatePolicyResult.errorMessage);
1680
+ persistRuntimeState(gatePolicyResult.persistTrigger, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
1681
+ onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
1682
+ preserveWorktreesForResume = true;
1683
+ break;
1684
+ }
821
1685
  }
822
1686
  }
823
1687
  }
@@ -1144,22 +2008,22 @@ export async function executeOrchBatch(
1144
2008
  // are warnings that preserve the orch branch for manual integration.
1145
2009
  // Gate: only run for terminal phases (completed/failed). Paused/stopped batches
1146
2010
  // are not yet done — integration would mutate refs prematurely.
1147
- let autoIntegrated = false;
2011
+ //
2012
+ // TP-043: "supervised" and "auto" integration modes are now owned by the
2013
+ // supervisor agent (which stays alive through post-batch integration).
2014
+ // The legacy engine fast-forward only runs for "auto" mode when no
2015
+ // supervisor is active (fallback). For "supervised" mode, the supervisor
2016
+ // always handles integration.
1148
2017
  const mergedTaskCount = batchState.succeededTasks;
1149
2018
  const isTerminalPhase = batchState.phase === "completed" || batchState.phase === "failed";
1150
2019
  if (isTerminalPhase && !preserveWorktreesForResume && batchState.orchBranch && mergedTaskCount > 0) {
1151
- if (orchConfig.orchestrator.integration === "auto") {
1152
- autoIntegrated = attemptAutoIntegration(
1153
- batchState.orchBranch,
1154
- batchState.baseBranch,
1155
- repoRoot,
1156
- batchState.batchId,
1157
- "batch",
1158
- onNotify,
1159
- );
1160
- }
1161
- // Manual mode (default) or auto-integration skipped: show integration guidance
1162
- if (!autoIntegrated) {
2020
+ if (orchConfig.orchestrator.integration === "supervised" || orchConfig.orchestrator.integration === "auto") {
2021
+ // TP-043: Supervisor-managed integration modes. The supervisor
2022
+ // agent handles integration after batch_complete event. The engine
2023
+ // does NOT perform legacy fast-forward here — defer to supervisor.
2024
+ execLog("batch", batchState.batchId, `integration deferred to supervisor (mode: ${orchConfig.orchestrator.integration})`);
2025
+ } else {
2026
+ // Manual mode (default): show integration guidance
1163
2027
  onNotify(
1164
2028
  ORCH_MESSAGES.orchIntegrationManual(batchState.orchBranch, batchState.baseBranch, mergedTaskCount),
1165
2029
  "info",
@@ -1170,6 +2034,9 @@ export async function executeOrchBatch(
1170
2034
  // ── TS-009: Persist terminal state ──
1171
2035
  persistRuntimeState("batch-terminal", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
1172
2036
 
2037
+ // ── TP-040: Emit batch terminal event (R002: unified via helper) ─
2038
+ emitTerminalEvent();
2039
+
1173
2040
  // ── TP-031: Emit diagnostic reports (JSONL + markdown) ──
1174
2041
  // Non-fatal: errors are logged but never crash batch finalization.
1175
2042
  emitDiagnosticReports(assembleDiagnosticInput(orchConfig, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, stateRoot));