taskplane 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/dashboard/public/app.js +227 -0
- package/dashboard/public/index.html +19 -0
- package/dashboard/public/style.css +319 -0
- package/dashboard/server.cjs +219 -1
- package/extensions/taskplane/config-loader.ts +7 -1
- package/extensions/taskplane/config-schema.ts +19 -2
- package/extensions/taskplane/config.ts +23 -1
- package/extensions/taskplane/engine.ts +913 -46
- package/extensions/taskplane/execution.ts +1 -0
- package/extensions/taskplane/extension.ts +975 -54
- package/extensions/taskplane/formatting.ts +713 -712
- package/extensions/taskplane/index.ts +1 -0
- package/extensions/taskplane/merge.ts +236 -48
- package/extensions/taskplane/messages.ts +10 -0
- package/extensions/taskplane/persistence.ts +183 -3
- package/extensions/taskplane/resume.ts +243 -24
- package/extensions/taskplane/settings-tui.ts +10 -3
- package/extensions/taskplane/supervisor-primer.md +626 -0
- package/extensions/taskplane/supervisor.ts +3659 -0
- package/extensions/taskplane/types.ts +330 -3
- package/package.json +1 -1
|
@@ -6,21 +6,459 @@ import { existsSync, readdirSync, readFileSync, unlinkSync } from "fs";
|
|
|
6
6
|
import { join, resolve } from "path";
|
|
7
7
|
|
|
8
8
|
import { formatDiscoveryResults, runDiscovery } from "./discovery.ts";
|
|
9
|
-
import { execLog, executeWave, tmuxKillSession } from "./execution.ts";
|
|
9
|
+
import { computeTransitiveDependents, execLog, executeLane, executeWave, tmuxKillSession } from "./execution.ts";
|
|
10
10
|
import type { MonitorUpdateCallback } from "./execution.ts";
|
|
11
|
+
// classifyExit no longer called directly — Tier 0 uses exitDiagnostic.classification
|
|
12
|
+
// from the diagnostic-reports pipeline (populated by assembleDiagnosticInput).
|
|
11
13
|
import { getCurrentBranch, runGit } from "./git.ts";
|
|
12
|
-
import {
|
|
13
|
-
import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolicy, formatRepoMergeSummary, ORCH_MESSAGES } from "./messages.ts";
|
|
14
|
+
import { mergeWaveByRepo } from "./merge.ts";
|
|
15
|
+
import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolicy, extractFailedRepoId, formatRepoMergeSummary, ORCH_MESSAGES } from "./messages.ts";
|
|
14
16
|
import type { CleanupGateRepoFailure } from "./messages.ts";
|
|
15
17
|
import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-reports.ts";
|
|
16
18
|
import { resolveOperatorId } from "./naming.ts";
|
|
17
|
-
import { applyPartialProgressToOutcomes, deleteBatchState, loadBatchHistory, persistRuntimeState, saveBatchHistory, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
19
|
+
import { applyPartialProgressToOutcomes, buildTier0EventBase, deleteBatchState, emitEngineEvent, emitTier0Event, loadBatchHistory, persistRuntimeState, saveBatchHistory, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
18
20
|
import { listOrchSessions } from "./sessions.ts";
|
|
19
|
-
import { defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId } from "./types.ts";
|
|
20
|
-
import type { AllocatedLane, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, TaskRunnerConfig, TokenCounts, WorkspaceConfig } from "./types.ts";
|
|
21
|
+
import { buildEngineEventBase, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
|
|
22
|
+
import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, TaskRunnerConfig, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
21
23
|
import { buildDependencyGraph, computeWaves, resolveBaseBranch, resolveRepoRoot, validateGraph } from "./waves.ts";
|
|
22
24
|
import { deleteBranchBestEffort, forceCleanupWorktree, formatPreflightResults, listWorktrees, preserveFailedLaneProgress, removeAllWorktrees, removeWorktree, runPreflight, safeResetWorktree, sleepSync } from "./worktree.ts";
|
|
23
25
|
|
|
26
|
+
// ── Tier 0: Automatic Recovery Helpers (TP-039) ─────────────────────
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Emit a `tier0_escalation` event with a typed `EscalationContext` payload.
|
|
30
|
+
*
|
|
31
|
+
* Called at every exhaustion path alongside the existing `tier0_recovery_exhausted`
|
|
32
|
+
* event. The escalation event carries a structured payload for the future
|
|
33
|
+
* supervisor agent (TP-041). In Tier 0, no automated action is taken on the
|
|
34
|
+
* escalation — the engine falls through to its existing pause behaviour.
|
|
35
|
+
*
|
|
36
|
+
* @since TP-039
|
|
37
|
+
*/
|
|
38
|
+
function emitTier0Escalation(
|
|
39
|
+
stateRoot: string,
|
|
40
|
+
batchId: string,
|
|
41
|
+
waveIndex: number,
|
|
42
|
+
pattern: Tier0EscalationPattern,
|
|
43
|
+
attempts: number,
|
|
44
|
+
maxAttempts: number,
|
|
45
|
+
lastError: string,
|
|
46
|
+
affectedTasks: string[],
|
|
47
|
+
suggestion: string,
|
|
48
|
+
extra?: Partial<Pick<import("./persistence.ts").Tier0Event, "taskId" | "laneNumber" | "repoId" | "classification" | "scopeKey">>,
|
|
49
|
+
): void {
|
|
50
|
+
const escalation: EscalationContext = {
|
|
51
|
+
pattern,
|
|
52
|
+
attempts,
|
|
53
|
+
maxAttempts,
|
|
54
|
+
lastError,
|
|
55
|
+
affectedTasks,
|
|
56
|
+
suggestion,
|
|
57
|
+
};
|
|
58
|
+
emitTier0Event(stateRoot, {
|
|
59
|
+
...buildTier0EventBase("tier0_escalation", batchId, waveIndex, pattern, attempts, maxAttempts),
|
|
60
|
+
...extra,
|
|
61
|
+
escalation,
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Attempt automatic retry for failed tasks with retryable exit classifications.
|
|
67
|
+
*
|
|
68
|
+
* After a wave completes, this function inspects each failed task's canonical
|
|
69
|
+
* `exitDiagnostic.classification` and re-executes the task if:
|
|
70
|
+
* - The classification is in TIER0_RETRYABLE_CLASSIFICATIONS (api_error, process_crash, session_vanished)
|
|
71
|
+
* - The retry budget for this scope has not been exhausted
|
|
72
|
+
*
|
|
73
|
+
* Partial progress is preserved before retry. On success, the task is moved from
|
|
74
|
+
* failedTaskIds to succeededTaskIds and the waveResult counts are updated in-place.
|
|
75
|
+
*
|
|
76
|
+
* @returns Object with retried count and updated task outcomes
|
|
77
|
+
*/
|
|
78
|
+
async function attemptWorkerCrashRetry(
|
|
79
|
+
waveResult: WaveExecutionResult,
|
|
80
|
+
waveIdx: number,
|
|
81
|
+
batchState: OrchBatchRuntimeState,
|
|
82
|
+
orchConfig: OrchestratorConfig,
|
|
83
|
+
repoRoot: string,
|
|
84
|
+
workspaceConfig: WorkspaceConfig | null | undefined,
|
|
85
|
+
allTaskOutcomes: LaneTaskOutcome[],
|
|
86
|
+
onNotify: (message: string, level: "info" | "warning" | "error") => void,
|
|
87
|
+
stateRoot: string,
|
|
88
|
+
): Promise<{ retriedCount: number; succeededRetries: string[]; failedRetries: string[] }> {
|
|
89
|
+
if (!batchState.resilience) {
|
|
90
|
+
batchState.resilience = defaultResilienceState();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const budget = TIER0_RETRY_BUDGETS.worker_crash;
|
|
94
|
+
const succeededRetries: string[] = [];
|
|
95
|
+
const failedRetries: string[] = [];
|
|
96
|
+
let retriedCount = 0;
|
|
97
|
+
|
|
98
|
+
// Build a map from taskId → lane for re-execution
|
|
99
|
+
const taskToLane = new Map<string, AllocatedLane>();
|
|
100
|
+
for (const lane of waveResult.allocatedLanes) {
|
|
101
|
+
for (const task of lane.tasks) {
|
|
102
|
+
taskToLane.set(task.taskId, lane);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Check each failed task for retryability
|
|
107
|
+
for (const taskId of [...waveResult.failedTaskIds]) {
|
|
108
|
+
const lane = taskToLane.get(taskId);
|
|
109
|
+
if (!lane) continue;
|
|
110
|
+
|
|
111
|
+
// Find the task outcome to get exit info
|
|
112
|
+
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
113
|
+
if (!outcome) continue;
|
|
114
|
+
|
|
115
|
+
// Use the canonical exit diagnostic classification when available.
|
|
116
|
+
// If exitDiagnostic is not populated (executeLane doesn't set it),
|
|
117
|
+
// we conservatively skip auto-retry rather than synthesizing a
|
|
118
|
+
// classification from incomplete data — which could incorrectly
|
|
119
|
+
// retry non-retryable failures (e.g., deterministic task errors).
|
|
120
|
+
const classification = outcome.exitDiagnostic?.classification;
|
|
121
|
+
|
|
122
|
+
if (!classification) {
|
|
123
|
+
execLog("batch", batchState.batchId,
|
|
124
|
+
`tier0: task ${taskId} has no exit diagnostic classification — skipping auto-retry (conservative)`,
|
|
125
|
+
);
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Check if retryable
|
|
130
|
+
if (!TIER0_RETRYABLE_CLASSIFICATIONS.has(classification)) {
|
|
131
|
+
execLog("batch", batchState.batchId,
|
|
132
|
+
`tier0: task ${taskId} exit classification "${classification}" is not retryable — skipping`,
|
|
133
|
+
);
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Check retry budget
|
|
138
|
+
const scopeKey = tier0ScopeKey("worker_crash", taskId, waveIdx);
|
|
139
|
+
const currentCount = batchState.resilience.retryCountByScope[scopeKey] ?? 0;
|
|
140
|
+
if (currentCount >= budget.maxRetries) {
|
|
141
|
+
execLog("batch", batchState.batchId,
|
|
142
|
+
`tier0: task ${taskId} retry budget exhausted (${currentCount}/${budget.maxRetries}) — skipping`,
|
|
143
|
+
{ scopeKey },
|
|
144
|
+
);
|
|
145
|
+
// Emit exhausted event
|
|
146
|
+
emitTier0Event(stateRoot, {
|
|
147
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "worker_crash", currentCount, budget.maxRetries),
|
|
148
|
+
taskId,
|
|
149
|
+
laneNumber: lane.laneNumber,
|
|
150
|
+
repoId: lane.repoId ?? null,
|
|
151
|
+
classification,
|
|
152
|
+
error: `Retry budget exhausted for task ${taskId} (${classification})`,
|
|
153
|
+
scopeKey,
|
|
154
|
+
affectedTaskIds: [taskId],
|
|
155
|
+
suggestion: `Task ${taskId} failed with ${classification} and exhausted ${budget.maxRetries} retry attempt(s). Consider investigating the root cause or manually re-running the task.`,
|
|
156
|
+
});
|
|
157
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "worker_crash", currentCount, budget.maxRetries,
|
|
158
|
+
`Retry budget exhausted for task ${taskId} (${classification})`, [taskId],
|
|
159
|
+
`Task ${taskId} failed with ${classification} and exhausted ${budget.maxRetries} retry attempt(s). Consider investigating the root cause or manually re-running the task.`,
|
|
160
|
+
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
161
|
+
);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Increment retry counter
|
|
166
|
+
batchState.resilience.retryCountByScope[scopeKey] = currentCount + 1;
|
|
167
|
+
retriedCount++;
|
|
168
|
+
|
|
169
|
+
execLog("batch", batchState.batchId,
|
|
170
|
+
`tier0: retrying task ${taskId} (worker_crash, attempt ${currentCount + 1}/${budget.maxRetries}, classification=${classification})`,
|
|
171
|
+
{ scopeKey, classification },
|
|
172
|
+
);
|
|
173
|
+
onNotify(
|
|
174
|
+
`🔄 Tier 0: Retrying task ${taskId} (${classification}, attempt ${currentCount + 1}/${budget.maxRetries})`,
|
|
175
|
+
"info",
|
|
176
|
+
);
|
|
177
|
+
|
|
178
|
+
// Emit attempt event
|
|
179
|
+
emitTier0Event(stateRoot, {
|
|
180
|
+
...buildTier0EventBase("tier0_recovery_attempt", batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries),
|
|
181
|
+
taskId,
|
|
182
|
+
laneNumber: lane.laneNumber,
|
|
183
|
+
repoId: lane.repoId ?? null,
|
|
184
|
+
classification,
|
|
185
|
+
cooldownMs: budget.cooldownMs,
|
|
186
|
+
scopeKey,
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
// Cooldown before retry
|
|
190
|
+
if (budget.cooldownMs > 0) {
|
|
191
|
+
sleepSync(budget.cooldownMs);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Find the specific AllocatedTask
|
|
195
|
+
const allocatedTask = lane.tasks.find(t => t.taskId === taskId);
|
|
196
|
+
if (!allocatedTask) continue;
|
|
197
|
+
|
|
198
|
+
// Re-execute: create a single-task lane config for executeLane
|
|
199
|
+
const retryLane: AllocatedLane = {
|
|
200
|
+
...lane,
|
|
201
|
+
tasks: [allocatedTask],
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
const isWsMode = !!workspaceConfig;
|
|
205
|
+
const wsRoot = workspaceConfig
|
|
206
|
+
? resolve(workspaceConfig.configPath, "..", "..")
|
|
207
|
+
: undefined;
|
|
208
|
+
|
|
209
|
+
try {
|
|
210
|
+
// Use a fresh pause signal for the retry — the batch pauseSignal
|
|
211
|
+
// may be paused due to stop-wave policy, but Tier 0 retry should
|
|
212
|
+
// attempt recovery before the stop decision takes effect (R002-4).
|
|
213
|
+
const retryPauseSignal = { paused: false };
|
|
214
|
+
const retryResult = await executeLane(
|
|
215
|
+
retryLane,
|
|
216
|
+
orchConfig,
|
|
217
|
+
repoRoot,
|
|
218
|
+
retryPauseSignal,
|
|
219
|
+
wsRoot,
|
|
220
|
+
isWsMode,
|
|
221
|
+
);
|
|
222
|
+
|
|
223
|
+
const retryOutcome = retryResult.tasks[0];
|
|
224
|
+
if (retryOutcome && retryOutcome.status === "succeeded") {
|
|
225
|
+
succeededRetries.push(taskId);
|
|
226
|
+
|
|
227
|
+
// Update waveResult: move from failed to succeeded
|
|
228
|
+
const failIdx = waveResult.failedTaskIds.indexOf(taskId);
|
|
229
|
+
if (failIdx !== -1) waveResult.failedTaskIds.splice(failIdx, 1);
|
|
230
|
+
waveResult.succeededTaskIds.push(taskId);
|
|
231
|
+
|
|
232
|
+
// Update lane results — replace the failed task outcome
|
|
233
|
+
for (const lr of waveResult.laneResults) {
|
|
234
|
+
const taskIdx = lr.tasks.findIndex(t => t.taskId === taskId);
|
|
235
|
+
if (taskIdx !== -1) {
|
|
236
|
+
lr.tasks[taskIdx] = retryOutcome;
|
|
237
|
+
break;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Update allTaskOutcomes
|
|
242
|
+
upsertTaskOutcome(allTaskOutcomes, retryOutcome);
|
|
243
|
+
|
|
244
|
+
execLog("batch", batchState.batchId,
|
|
245
|
+
`tier0: task ${taskId} retry succeeded`,
|
|
246
|
+
{ scopeKey },
|
|
247
|
+
);
|
|
248
|
+
onNotify(
|
|
249
|
+
`✅ Tier 0: Task ${taskId} retry succeeded`,
|
|
250
|
+
"info",
|
|
251
|
+
);
|
|
252
|
+
|
|
253
|
+
// Emit success event
|
|
254
|
+
emitTier0Event(stateRoot, {
|
|
255
|
+
...buildTier0EventBase("tier0_recovery_success", batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries),
|
|
256
|
+
taskId,
|
|
257
|
+
laneNumber: lane.laneNumber,
|
|
258
|
+
repoId: lane.repoId ?? null,
|
|
259
|
+
classification,
|
|
260
|
+
resolution: `Task ${taskId} succeeded on retry attempt ${currentCount + 1}`,
|
|
261
|
+
scopeKey,
|
|
262
|
+
});
|
|
263
|
+
} else {
|
|
264
|
+
failedRetries.push(taskId);
|
|
265
|
+
if (retryOutcome) {
|
|
266
|
+
upsertTaskOutcome(allTaskOutcomes, retryOutcome);
|
|
267
|
+
}
|
|
268
|
+
execLog("batch", batchState.batchId,
|
|
269
|
+
`tier0: task ${taskId} retry failed again`,
|
|
270
|
+
{ scopeKey, exitReason: retryOutcome?.exitReason },
|
|
271
|
+
);
|
|
272
|
+
|
|
273
|
+
// Emit exhausted event (retry failed and budget now consumed)
|
|
274
|
+
const retryFailError = retryOutcome?.exitReason ?? `Task ${taskId} retry failed again`;
|
|
275
|
+
const retryFailSuggestion = `Task ${taskId} failed again after retry (${classification}). The failure may be persistent — investigate task logs.`;
|
|
276
|
+
emitTier0Event(stateRoot, {
|
|
277
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries),
|
|
278
|
+
taskId,
|
|
279
|
+
laneNumber: lane.laneNumber,
|
|
280
|
+
repoId: lane.repoId ?? null,
|
|
281
|
+
classification,
|
|
282
|
+
error: retryFailError,
|
|
283
|
+
scopeKey,
|
|
284
|
+
affectedTaskIds: [taskId],
|
|
285
|
+
suggestion: retryFailSuggestion,
|
|
286
|
+
});
|
|
287
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries,
|
|
288
|
+
retryFailError, [taskId], retryFailSuggestion,
|
|
289
|
+
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
} catch (err: unknown) {
|
|
293
|
+
failedRetries.push(taskId);
|
|
294
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
295
|
+
execLog("batch", batchState.batchId,
|
|
296
|
+
`tier0: task ${taskId} retry threw error: ${errMsg}`,
|
|
297
|
+
{ scopeKey },
|
|
298
|
+
);
|
|
299
|
+
|
|
300
|
+
// Emit exhausted event for exception during retry
|
|
301
|
+
const exceptionSuggestion = `Task ${taskId} retry threw an exception: ${errMsg}. Investigate the execution environment.`;
|
|
302
|
+
emitTier0Event(stateRoot, {
|
|
303
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries),
|
|
304
|
+
taskId,
|
|
305
|
+
laneNumber: lane.laneNumber,
|
|
306
|
+
repoId: lane.repoId ?? null,
|
|
307
|
+
classification,
|
|
308
|
+
error: errMsg,
|
|
309
|
+
scopeKey,
|
|
310
|
+
affectedTaskIds: [taskId],
|
|
311
|
+
suggestion: exceptionSuggestion,
|
|
312
|
+
});
|
|
313
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "worker_crash", currentCount + 1, budget.maxRetries,
|
|
314
|
+
errMsg, [taskId], exceptionSuggestion,
|
|
315
|
+
{ taskId, laneNumber: lane.laneNumber, repoId: lane.repoId ?? null, classification, scopeKey },
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// Recalculate wave-level status if retries changed outcomes.
|
|
321
|
+
// NOTE: Batch-level counters (succeededTasks, failedTasks) are NOT updated
|
|
322
|
+
// here — the caller accumulates them from waveResult AFTER retry so that
|
|
323
|
+
// counts are only applied once (R002-2 fix).
|
|
324
|
+
if (succeededRetries.length > 0) {
|
|
325
|
+
if (waveResult.failedTaskIds.length === 0) {
|
|
326
|
+
waveResult.overallStatus = "succeeded";
|
|
327
|
+
waveResult.stoppedEarly = false;
|
|
328
|
+
} else if (waveResult.succeededTaskIds.length > 0) {
|
|
329
|
+
waveResult.overallStatus = "partial";
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
return { retriedCount, succeededRetries, failedRetries };
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* Attempt stale worktree recovery when lane allocation fails with ALLOC_WORKTREE_FAILED.
|
|
338
|
+
*
|
|
339
|
+
* Forces cleanup of all matching worktrees, prunes git state, then retries
|
|
340
|
+
* the wave execution.
|
|
341
|
+
*
|
|
342
|
+
* @returns The retry waveResult, or null if recovery was not attempted
|
|
343
|
+
*/
|
|
344
|
+
async function attemptStaleWorktreeRecovery(
|
|
345
|
+
waveResult: WaveExecutionResult,
|
|
346
|
+
waveTasks: string[],
|
|
347
|
+
waveIdx: number,
|
|
348
|
+
discovery: DiscoveryResult,
|
|
349
|
+
orchConfig: OrchestratorConfig,
|
|
350
|
+
repoRoot: string,
|
|
351
|
+
batchState: OrchBatchRuntimeState,
|
|
352
|
+
depGraph: ReturnType<typeof buildDependencyGraph>,
|
|
353
|
+
workspaceConfig: WorkspaceConfig | null | undefined,
|
|
354
|
+
onMonitorUpdate: MonitorUpdateCallback | undefined,
|
|
355
|
+
onLanesAllocated: (lanes: AllocatedLane[]) => void,
|
|
356
|
+
stateRoot: string,
|
|
357
|
+
): Promise<WaveExecutionResult | null> {
|
|
358
|
+
// Only attempt recovery for ALLOC_WORKTREE_FAILED
|
|
359
|
+
if (!waveResult.allocationError || waveResult.allocationError.code !== "ALLOC_WORKTREE_FAILED") {
|
|
360
|
+
return null;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
if (!batchState.resilience) {
|
|
364
|
+
batchState.resilience = defaultResilienceState();
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
const budget = TIER0_RETRY_BUDGETS.stale_worktree;
|
|
368
|
+
const scopeKey = tier0WaveScopeKey("stale_worktree", waveIdx);
|
|
369
|
+
const currentCount = batchState.resilience.retryCountByScope[scopeKey] ?? 0;
|
|
370
|
+
|
|
371
|
+
if (currentCount >= budget.maxRetries) {
|
|
372
|
+
execLog("batch", batchState.batchId,
|
|
373
|
+
`tier0: stale worktree retry budget exhausted (${currentCount}/${budget.maxRetries})`,
|
|
374
|
+
{ scopeKey },
|
|
375
|
+
);
|
|
376
|
+
const staleExhaustedError = waveResult.allocationError.message;
|
|
377
|
+
const staleExhaustedSuggestion = `Stale worktree cleanup exhausted ${budget.maxRetries} retry(s). Manually remove worktrees and prune git state.`;
|
|
378
|
+
emitTier0Event(stateRoot, {
|
|
379
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "stale_worktree", currentCount, budget.maxRetries),
|
|
380
|
+
repoId: null, // wave-scoped
|
|
381
|
+
error: staleExhaustedError,
|
|
382
|
+
scopeKey,
|
|
383
|
+
affectedTaskIds: waveTasks,
|
|
384
|
+
suggestion: staleExhaustedSuggestion,
|
|
385
|
+
});
|
|
386
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "stale_worktree", currentCount, budget.maxRetries,
|
|
387
|
+
staleExhaustedError, waveTasks, staleExhaustedSuggestion,
|
|
388
|
+
{ repoId: null, scopeKey },
|
|
389
|
+
);
|
|
390
|
+
return null;
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
batchState.resilience.retryCountByScope[scopeKey] = currentCount + 1;
|
|
394
|
+
|
|
395
|
+
execLog("batch", batchState.batchId,
|
|
396
|
+
`tier0: attempting stale worktree recovery (attempt ${currentCount + 1}/${budget.maxRetries})`,
|
|
397
|
+
{ scopeKey, allocationError: waveResult.allocationError.message },
|
|
398
|
+
);
|
|
399
|
+
|
|
400
|
+
// Emit attempt event
|
|
401
|
+
emitTier0Event(stateRoot, {
|
|
402
|
+
...buildTier0EventBase("tier0_recovery_attempt", batchState.batchId, waveIdx, "stale_worktree", currentCount + 1, budget.maxRetries),
|
|
403
|
+
repoId: null, // wave-scoped: allocation failure may span multiple repos
|
|
404
|
+
classification: waveResult.allocationError.code,
|
|
405
|
+
cooldownMs: budget.cooldownMs,
|
|
406
|
+
scopeKey,
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
// Force cleanup: remove all worktrees for this batch and prune.
|
|
410
|
+
// In workspace mode, iterate ALL workspace repos — allocation failures
|
|
411
|
+
// can come from non-default repos (R002-3 fix).
|
|
412
|
+
const prefix = orchConfig.orchestrator.worktree_prefix;
|
|
413
|
+
const opId = resolveOperatorId(orchConfig);
|
|
414
|
+
|
|
415
|
+
const repoRootsToClean: string[] = [repoRoot];
|
|
416
|
+
if (workspaceConfig) {
|
|
417
|
+
for (const [, repoConf] of workspaceConfig.repos) {
|
|
418
|
+
if (repoConf.path !== repoRoot && !repoRootsToClean.includes(repoConf.path)) {
|
|
419
|
+
repoRootsToClean.push(repoConf.path);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
for (const cleanRoot of repoRootsToClean) {
|
|
425
|
+
const existingWorktrees = listWorktrees(prefix, cleanRoot, opId, batchState.batchId);
|
|
426
|
+
for (const wt of existingWorktrees) {
|
|
427
|
+
forceCleanupWorktree(wt, cleanRoot, batchState.batchId);
|
|
428
|
+
}
|
|
429
|
+
// Also prune git worktree state in case of orphaned references
|
|
430
|
+
runGit(["worktree", "prune"], cleanRoot);
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
// Cooldown before retry
|
|
434
|
+
if (budget.cooldownMs > 0) {
|
|
435
|
+
sleepSync(budget.cooldownMs);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// Retry the wave execution
|
|
439
|
+
execLog("batch", batchState.batchId,
|
|
440
|
+
`tier0: retrying wave ${waveIdx + 1} after stale worktree cleanup`,
|
|
441
|
+
);
|
|
442
|
+
|
|
443
|
+
const retryResult = await executeWave(
|
|
444
|
+
waveTasks,
|
|
445
|
+
waveIdx + 1,
|
|
446
|
+
discovery.pending,
|
|
447
|
+
orchConfig,
|
|
448
|
+
repoRoot,
|
|
449
|
+
batchState.batchId,
|
|
450
|
+
batchState.pauseSignal,
|
|
451
|
+
depGraph,
|
|
452
|
+
batchState.orchBranch,
|
|
453
|
+
onMonitorUpdate,
|
|
454
|
+
onLanesAllocated,
|
|
455
|
+
workspaceConfig,
|
|
456
|
+
);
|
|
457
|
+
|
|
458
|
+
return retryResult;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
|
|
24
462
|
// ── /orch Execution Engine ───────────────────────────────────────────
|
|
25
463
|
|
|
26
464
|
/**
|
|
@@ -37,6 +475,8 @@ import { deleteBranchBestEffort, forceCleanupWorktree, formatPreflightResults, l
|
|
|
37
475
|
* @param onMonitorUpdate - Optional callback for dashboard updates
|
|
38
476
|
* @param workspaceConfig - Workspace configuration for repo routing (null = repo mode)
|
|
39
477
|
* @param workspaceRoot - Workspace root for resolving task area paths (defaults to cwd)
|
|
478
|
+
* @param agentRoot - Agent root for config resolution
|
|
479
|
+
* @param onEngineEvent - Optional callback for engine lifecycle events (TP-040)
|
|
40
480
|
*/
|
|
41
481
|
export async function executeOrchBatch(
|
|
42
482
|
args: string,
|
|
@@ -49,17 +489,54 @@ export async function executeOrchBatch(
|
|
|
49
489
|
workspaceConfig?: WorkspaceConfig | null,
|
|
50
490
|
workspaceRoot?: string,
|
|
51
491
|
agentRoot?: string,
|
|
492
|
+
onEngineEvent?: EngineEventCallback | null,
|
|
52
493
|
): Promise<void> {
|
|
53
494
|
const repoRoot = cwd;
|
|
54
495
|
// State files (.pi/batch-state.json, lane-state, etc.) belong in the workspace root,
|
|
55
496
|
// which is where .pi/ config lives. In repo mode, workspaceRoot === repoRoot.
|
|
56
497
|
const stateRoot = workspaceRoot ?? cwd;
|
|
57
498
|
|
|
499
|
+
// ── TP-040: Engine event emission helper ─────────────────────
|
|
500
|
+
// Closure over stateRoot and onEngineEvent to keep emit calls terse.
|
|
501
|
+
// batchState.batchId is read at call time (it's set in Phase 1).
|
|
502
|
+
const emitEvent: typeof emitEngineEvent = (sr, event, cb) => emitEngineEvent(sr, event, cb);
|
|
503
|
+
|
|
504
|
+
// ── TP-040 R002: Terminal event emission helper ──────────────
|
|
505
|
+
// Routes all early-return and terminal paths through consistent event
|
|
506
|
+
// emission so external consumers always receive a deterministic terminal
|
|
507
|
+
// signal (batch_complete for completed/failed, batch_paused for paused/stopped).
|
|
508
|
+
// Uses a guard flag to enforce one-transition/one-event semantics — once a
|
|
509
|
+
// terminal event has been emitted, subsequent calls are no-ops.
|
|
510
|
+
let terminalEventEmitted = false;
|
|
511
|
+
const emitTerminalEvent = (reason?: string): void => {
|
|
512
|
+
if (terminalEventEmitted) return;
|
|
513
|
+
terminalEventEmitted = true;
|
|
514
|
+
if (batchState.phase === "completed" || batchState.phase === "failed") {
|
|
515
|
+
emitEvent(stateRoot, {
|
|
516
|
+
...buildEngineEventBase("batch_complete", batchState.batchId, batchState.currentWaveIndex, batchState.phase),
|
|
517
|
+
succeededTasks: batchState.succeededTasks,
|
|
518
|
+
failedTasks: batchState.failedTasks,
|
|
519
|
+
skippedTasks: batchState.skippedTasks,
|
|
520
|
+
blockedTasks: batchState.blockedTasks,
|
|
521
|
+
batchDurationMs: batchState.endedAt ? batchState.endedAt - batchState.startedAt : undefined,
|
|
522
|
+
}, onEngineEvent);
|
|
523
|
+
} else if (batchState.phase === "paused" || batchState.phase === "stopped") {
|
|
524
|
+
emitEvent(stateRoot, {
|
|
525
|
+
...buildEngineEventBase("batch_paused", batchState.batchId, batchState.currentWaveIndex, batchState.phase),
|
|
526
|
+
reason: reason || (batchState.errors.length > 0 ? batchState.errors[batchState.errors.length - 1] : "paused"),
|
|
527
|
+
failedTasks: batchState.failedTasks,
|
|
528
|
+
}, onEngineEvent);
|
|
529
|
+
}
|
|
530
|
+
};
|
|
531
|
+
|
|
58
532
|
// ── Phase 1: Planning ────────────────────────────────────────
|
|
59
533
|
batchState.phase = "planning";
|
|
60
534
|
batchState.batchId = generateBatchId();
|
|
61
|
-
|
|
62
|
-
batchState.
|
|
535
|
+
// Preserve startedAt if set during "launching" phase (TP-040)
|
|
536
|
+
if (!batchState.startedAt) batchState.startedAt = Date.now();
|
|
537
|
+
// Preserve pauseSignal if already set during "launching" phase (TP-040)
|
|
538
|
+
// — e.g., /orch-pause issued between /orch return and engine start
|
|
539
|
+
if (!batchState.pauseSignal?.paused) batchState.pauseSignal = { paused: false };
|
|
63
540
|
batchState.mergeResults = [];
|
|
64
541
|
batchState.mode = workspaceConfig ? "workspace" : "repo";
|
|
65
542
|
|
|
@@ -70,6 +547,7 @@ export async function executeOrchBatch(
|
|
|
70
547
|
batchState.endedAt = Date.now();
|
|
71
548
|
batchState.errors.push("Cannot determine current branch (detached HEAD or not a git repo)");
|
|
72
549
|
onNotify("❌ Cannot determine current branch. Ensure HEAD is on a branch (not detached).", "error");
|
|
550
|
+
emitTerminalEvent();
|
|
73
551
|
return;
|
|
74
552
|
}
|
|
75
553
|
batchState.baseBranch = detectedBranch;
|
|
@@ -105,6 +583,7 @@ export async function executeOrchBatch(
|
|
|
105
583
|
batchState.phase = "failed";
|
|
106
584
|
batchState.endedAt = Date.now();
|
|
107
585
|
batchState.errors.push("Preflight check failed");
|
|
586
|
+
emitTerminalEvent();
|
|
108
587
|
return;
|
|
109
588
|
}
|
|
110
589
|
|
|
@@ -147,6 +626,7 @@ export async function executeOrchBatch(
|
|
|
147
626
|
"info",
|
|
148
627
|
);
|
|
149
628
|
}
|
|
629
|
+
emitTerminalEvent();
|
|
150
630
|
return;
|
|
151
631
|
}
|
|
152
632
|
|
|
@@ -154,6 +634,7 @@ export async function executeOrchBatch(
|
|
|
154
634
|
batchState.phase = "completed";
|
|
155
635
|
batchState.endedAt = Date.now();
|
|
156
636
|
onNotify("No pending tasks found. Nothing to execute.", "info");
|
|
637
|
+
emitTerminalEvent();
|
|
157
638
|
return;
|
|
158
639
|
}
|
|
159
640
|
|
|
@@ -169,6 +650,7 @@ export async function executeOrchBatch(
|
|
|
169
650
|
const errMsgs = validation.errors.map(e => `[${e.code}] ${e.message}`).join("\n");
|
|
170
651
|
batchState.errors.push(`Graph validation failed:\n${errMsgs}`);
|
|
171
652
|
onNotify(`❌ Dependency graph errors:\n${errMsgs}`, "error");
|
|
653
|
+
emitTerminalEvent();
|
|
172
654
|
return;
|
|
173
655
|
}
|
|
174
656
|
|
|
@@ -180,6 +662,7 @@ export async function executeOrchBatch(
|
|
|
180
662
|
const errMsgs = waveErrors.map(e => `[${e.code}] ${e.message}`).join("\n");
|
|
181
663
|
batchState.errors.push(`Wave computation failed:\n${errMsgs}`);
|
|
182
664
|
onNotify(`❌ Wave computation errors:\n${errMsgs}`, "error");
|
|
665
|
+
emitTerminalEvent();
|
|
183
666
|
return;
|
|
184
667
|
}
|
|
185
668
|
|
|
@@ -220,7 +703,7 @@ export async function executeOrchBatch(
|
|
|
220
703
|
break;
|
|
221
704
|
}
|
|
222
705
|
}
|
|
223
|
-
if (orchBranchFailed) return;
|
|
706
|
+
if (orchBranchFailed) { emitTerminalEvent(); return; }
|
|
224
707
|
} else {
|
|
225
708
|
const branchResult = runGit(["branch", orchBranch, batchState.baseBranch], repoRoot);
|
|
226
709
|
if (!branchResult.ok) {
|
|
@@ -229,6 +712,7 @@ export async function executeOrchBatch(
|
|
|
229
712
|
const errDetail = branchResult.stderr || branchResult.stdout || "unknown error";
|
|
230
713
|
batchState.errors.push(`Failed to create orch branch '${orchBranch}': ${errDetail}`);
|
|
231
714
|
onNotify(`❌ Failed to create orch branch '${orchBranch}': ${errDetail}`, "error");
|
|
715
|
+
emitTerminalEvent();
|
|
232
716
|
return;
|
|
233
717
|
}
|
|
234
718
|
execLog("batch", batchState.batchId, "created orch branch", { orchBranch, baseBranch: batchState.baseBranch });
|
|
@@ -254,6 +738,8 @@ export async function executeOrchBatch(
|
|
|
254
738
|
onNotify(`⏸️ Batch paused before wave ${waveIdx + 1}. Resume not yet implemented (TS-009).`, "warning");
|
|
255
739
|
// ── TS-009: Persist state on pause ──
|
|
256
740
|
persistRuntimeState("pause-before-wave", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
741
|
+
// TP-040: Emit batch_paused event (via terminal helper for dedup)
|
|
742
|
+
emitTerminalEvent(`Paused before wave ${waveIdx + 1}`);
|
|
257
743
|
break;
|
|
258
744
|
}
|
|
259
745
|
|
|
@@ -288,6 +774,13 @@ export async function executeOrchBatch(
|
|
|
288
774
|
"info",
|
|
289
775
|
);
|
|
290
776
|
|
|
777
|
+
// TP-040: Emit wave_start event
|
|
778
|
+
emitEvent(stateRoot, {
|
|
779
|
+
...buildEngineEventBase("wave_start", batchState.batchId, waveIdx, batchState.phase),
|
|
780
|
+
taskIds: waveTasks,
|
|
781
|
+
laneCount: Math.min(waveTasks.length, orchConfig.orchestrator.max_lanes),
|
|
782
|
+
}, onEngineEvent);
|
|
783
|
+
|
|
291
784
|
const handleWaveMonitorUpdate: MonitorUpdateCallback = (monitorState) => {
|
|
292
785
|
const changed = syncTaskOutcomesFromMonitor(monitorState, allTaskOutcomes);
|
|
293
786
|
if (changed) {
|
|
@@ -297,7 +790,20 @@ export async function executeOrchBatch(
|
|
|
297
790
|
};
|
|
298
791
|
|
|
299
792
|
// Execute the wave
|
|
300
|
-
const
|
|
793
|
+
const onLanesAllocatedCb = (lanes: AllocatedLane[]) => {
|
|
794
|
+
latestAllocatedLanes = lanes;
|
|
795
|
+
batchState.currentLanes = lanes;
|
|
796
|
+
// TP-029: Track repos from newly allocated lanes for cleanup coverage
|
|
797
|
+
for (const lane of lanes) {
|
|
798
|
+
const laneRepoRoot = resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig);
|
|
799
|
+
encounteredRepoRoots.set(laneRepoRoot, lane.repoId);
|
|
800
|
+
}
|
|
801
|
+
if (seedPendingOutcomesForAllocatedLanes(lanes, allTaskOutcomes)) {
|
|
802
|
+
persistRuntimeState("wave-lanes-allocated", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
803
|
+
}
|
|
804
|
+
};
|
|
805
|
+
|
|
806
|
+
let waveResult = await executeWave(
|
|
301
807
|
waveTasks,
|
|
302
808
|
waveIdx + 1,
|
|
303
809
|
discovery.pending,
|
|
@@ -308,21 +814,66 @@ export async function executeOrchBatch(
|
|
|
308
814
|
depGraph,
|
|
309
815
|
batchState.orchBranch,
|
|
310
816
|
handleWaveMonitorUpdate,
|
|
311
|
-
|
|
312
|
-
latestAllocatedLanes = lanes;
|
|
313
|
-
batchState.currentLanes = lanes;
|
|
314
|
-
// TP-029: Track repos from newly allocated lanes for cleanup coverage
|
|
315
|
-
for (const lane of lanes) {
|
|
316
|
-
const laneRepoRoot = resolveRepoRoot(lane.repoId, repoRoot, workspaceConfig);
|
|
317
|
-
encounteredRepoRoots.set(laneRepoRoot, lane.repoId);
|
|
318
|
-
}
|
|
319
|
-
if (seedPendingOutcomesForAllocatedLanes(lanes, allTaskOutcomes)) {
|
|
320
|
-
persistRuntimeState("wave-lanes-allocated", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
321
|
-
}
|
|
322
|
-
},
|
|
817
|
+
onLanesAllocatedCb,
|
|
323
818
|
workspaceConfig,
|
|
324
819
|
);
|
|
325
820
|
|
|
821
|
+
// ── TP-039: Tier 0 — Stale worktree recovery ────────────
|
|
822
|
+
// If allocation failed with ALLOC_WORKTREE_FAILED, force cleanup
|
|
823
|
+
// and retry the entire wave execution once.
|
|
824
|
+
if (waveResult.allocationError?.code === "ALLOC_WORKTREE_FAILED") {
|
|
825
|
+
const retryResult = await attemptStaleWorktreeRecovery(
|
|
826
|
+
waveResult,
|
|
827
|
+
waveTasks,
|
|
828
|
+
waveIdx,
|
|
829
|
+
discovery,
|
|
830
|
+
orchConfig,
|
|
831
|
+
repoRoot,
|
|
832
|
+
batchState,
|
|
833
|
+
depGraph,
|
|
834
|
+
workspaceConfig,
|
|
835
|
+
handleWaveMonitorUpdate,
|
|
836
|
+
onLanesAllocatedCb,
|
|
837
|
+
stateRoot,
|
|
838
|
+
);
|
|
839
|
+
if (retryResult) {
|
|
840
|
+
const staleRecovered = !retryResult.allocationError;
|
|
841
|
+
onNotify(
|
|
842
|
+
`🔄 Tier 0: Stale worktree recovery ${staleRecovered ? "succeeded" : "failed"} for wave ${waveIdx + 1}`,
|
|
843
|
+
staleRecovered ? "info" : "warning",
|
|
844
|
+
);
|
|
845
|
+
|
|
846
|
+
// Emit success or exhausted event based on retry result
|
|
847
|
+
const staleScopeKey = tier0WaveScopeKey("stale_worktree", waveIdx);
|
|
848
|
+
const staleCount = batchState.resilience?.retryCountByScope[staleScopeKey] ?? 1;
|
|
849
|
+
if (staleRecovered) {
|
|
850
|
+
emitTier0Event(stateRoot, {
|
|
851
|
+
...buildTier0EventBase("tier0_recovery_success", batchState.batchId, waveIdx, "stale_worktree", staleCount, TIER0_RETRY_BUDGETS.stale_worktree.maxRetries),
|
|
852
|
+
repoId: null, // wave-scoped
|
|
853
|
+
resolution: `Stale worktree cleanup succeeded — wave ${waveIdx + 1} re-executed successfully`,
|
|
854
|
+
scopeKey: staleScopeKey,
|
|
855
|
+
});
|
|
856
|
+
} else {
|
|
857
|
+
const staleRetryError = retryResult.allocationError?.message ?? "Allocation failed again after cleanup";
|
|
858
|
+
const staleRetrySuggestion = "Stale worktree cleanup did not resolve the allocation failure. Manually inspect and remove worktrees.";
|
|
859
|
+
emitTier0Event(stateRoot, {
|
|
860
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "stale_worktree", staleCount, TIER0_RETRY_BUDGETS.stale_worktree.maxRetries),
|
|
861
|
+
repoId: null, // wave-scoped
|
|
862
|
+
error: staleRetryError,
|
|
863
|
+
scopeKey: staleScopeKey,
|
|
864
|
+
affectedTaskIds: waveTasks,
|
|
865
|
+
suggestion: staleRetrySuggestion,
|
|
866
|
+
});
|
|
867
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "stale_worktree", staleCount, TIER0_RETRY_BUDGETS.stale_worktree.maxRetries,
|
|
868
|
+
staleRetryError, waveTasks, staleRetrySuggestion,
|
|
869
|
+
{ repoId: null, scopeKey: staleScopeKey },
|
|
870
|
+
);
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
waveResult = retryResult;
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
|
|
326
877
|
batchState.waveResults.push(waveResult);
|
|
327
878
|
batchState.currentLanes = []; // Clear current lanes after wave completes
|
|
328
879
|
|
|
@@ -334,16 +885,98 @@ export async function executeOrchBatch(
|
|
|
334
885
|
}
|
|
335
886
|
}
|
|
336
887
|
|
|
337
|
-
//
|
|
888
|
+
// ── TP-039: Tier 0 — Worker crash retry ─────────────────
|
|
889
|
+
// Run retry BEFORE accumulating counts and blocked tasks so that
|
|
890
|
+
// successfully retried tasks don't inflate failedTasks count and
|
|
891
|
+
// their dependents aren't incorrectly blocked (R002-2 fix).
|
|
892
|
+
if (waveResult.failedTaskIds.length > 0) {
|
|
893
|
+
const retryOutcome = await attemptWorkerCrashRetry(
|
|
894
|
+
waveResult,
|
|
895
|
+
waveIdx,
|
|
896
|
+
batchState,
|
|
897
|
+
orchConfig,
|
|
898
|
+
repoRoot,
|
|
899
|
+
workspaceConfig,
|
|
900
|
+
allTaskOutcomes,
|
|
901
|
+
onNotify,
|
|
902
|
+
stateRoot,
|
|
903
|
+
);
|
|
904
|
+
if (retryOutcome.succeededRetries.length > 0) {
|
|
905
|
+
// Recompute blockedTaskIds from remaining failures (R002-2).
|
|
906
|
+
// attemptWorkerCrashRetry already updated waveResult.failedTaskIds
|
|
907
|
+
// and waveResult.succeededTaskIds in-place.
|
|
908
|
+
if (waveResult.policyApplied === "skip-dependents" && waveResult.failedTaskIds.length > 0) {
|
|
909
|
+
const recomputed = computeTransitiveDependents(
|
|
910
|
+
new Set(waveResult.failedTaskIds),
|
|
911
|
+
depGraph,
|
|
912
|
+
);
|
|
913
|
+
waveResult.blockedTaskIds = [...recomputed].sort();
|
|
914
|
+
} else if (waveResult.failedTaskIds.length === 0) {
|
|
915
|
+
// All failures recovered — no blocked tasks
|
|
916
|
+
waveResult.blockedTaskIds = [];
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
if (retryOutcome.retriedCount > 0) {
|
|
920
|
+
// Persist updated state after retries
|
|
921
|
+
persistRuntimeState("tier0-worker-retry", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
922
|
+
}
|
|
923
|
+
|
|
924
|
+
// If stop-wave had paused the batch but Tier 0 retry recovered all
|
|
925
|
+
// failures, clear the policy-induced pause so subsequent waves can
|
|
926
|
+
// proceed. attemptWorkerCrashRetry already set stoppedEarly=false
|
|
927
|
+
// and overallStatus="succeeded" on the waveResult (R002-4 fix).
|
|
928
|
+
if (
|
|
929
|
+
waveResult.failedTaskIds.length === 0
|
|
930
|
+
&& batchState.pauseSignal.paused
|
|
931
|
+
&& waveResult.policyApplied === "stop-wave"
|
|
932
|
+
) {
|
|
933
|
+
batchState.pauseSignal.paused = false;
|
|
934
|
+
execLog("batch", batchState.batchId,
|
|
935
|
+
`tier0: all failed tasks recovered — clearing stop-wave pause`,
|
|
936
|
+
);
|
|
937
|
+
onNotify(
|
|
938
|
+
`✅ Tier 0: All failed tasks recovered — batch continuing past stop-wave`,
|
|
939
|
+
"info",
|
|
940
|
+
);
|
|
941
|
+
}
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
// Accumulate results (after retry so counts reflect recovered tasks)
|
|
338
945
|
batchState.succeededTasks += waveResult.succeededTaskIds.length;
|
|
339
946
|
batchState.failedTasks += waveResult.failedTaskIds.length;
|
|
340
947
|
batchState.skippedTasks += waveResult.skippedTaskIds.length;
|
|
341
948
|
|
|
342
|
-
// Add newly blocked tasks
|
|
949
|
+
// Add newly blocked tasks (after retry so recovered tasks don't block dependents)
|
|
343
950
|
for (const blocked of waveResult.blockedTaskIds) {
|
|
344
951
|
batchState.blockedTaskIds.add(blocked);
|
|
345
952
|
}
|
|
346
953
|
|
|
954
|
+
// ── TP-040: Emit task_complete / task_failed events ──────
|
|
955
|
+
// Emitted after Tier 0 retry so events reflect final status.
|
|
956
|
+
for (const taskId of waveResult.succeededTaskIds) {
|
|
957
|
+
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
958
|
+
emitEvent(stateRoot, {
|
|
959
|
+
...buildEngineEventBase("task_complete", batchState.batchId, waveIdx, batchState.phase),
|
|
960
|
+
taskId,
|
|
961
|
+
durationMs: outcome?.startTime && outcome?.endTime
|
|
962
|
+
? outcome.endTime - outcome.startTime
|
|
963
|
+
: undefined,
|
|
964
|
+
outcome: "succeeded",
|
|
965
|
+
}, onEngineEvent);
|
|
966
|
+
}
|
|
967
|
+
for (const taskId of waveResult.failedTaskIds) {
|
|
968
|
+
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
969
|
+
emitEvent(stateRoot, {
|
|
970
|
+
...buildEngineEventBase("task_failed", batchState.batchId, waveIdx, batchState.phase),
|
|
971
|
+
taskId,
|
|
972
|
+
durationMs: outcome?.startTime && outcome?.endTime
|
|
973
|
+
? outcome.endTime - outcome.startTime
|
|
974
|
+
: undefined,
|
|
975
|
+
reason: outcome?.exitReason || "unknown",
|
|
976
|
+
partialProgress: (outcome?.partialProgressCommits ?? 0) > 0,
|
|
977
|
+
}, onEngineEvent);
|
|
978
|
+
}
|
|
979
|
+
|
|
347
980
|
// ── TS-009: Persist state after wave execution ──
|
|
348
981
|
persistRuntimeState("wave-execution-complete", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
349
982
|
|
|
@@ -359,6 +992,10 @@ export async function executeOrchBatch(
|
|
|
359
992
|
waveResult.failedTaskIds.length > 0 ? "warning" : "info",
|
|
360
993
|
);
|
|
361
994
|
|
|
995
|
+
// NOTE: No explicit wave_complete event in the spec event set. The supervisor
|
|
996
|
+
// infers wave completion from the sequence of task_complete/task_failed events
|
|
997
|
+
// followed by merge_start or the next wave_start.
|
|
998
|
+
|
|
362
999
|
// Check if we should stop based on task failure policy
|
|
363
1000
|
if (waveResult.stoppedEarly) {
|
|
364
1001
|
if (waveResult.policyApplied === "stop-all") {
|
|
@@ -366,6 +1003,8 @@ export async function executeOrchBatch(
|
|
|
366
1003
|
// ── TS-009: Persist state on stop-all ──
|
|
367
1004
|
persistRuntimeState("stop-all", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
368
1005
|
onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-all"), "error");
|
|
1006
|
+
// TP-040: Emit batch_paused event (via terminal helper for dedup)
|
|
1007
|
+
emitTerminalEvent(`Stopped by stop-all policy at wave ${waveIdx + 1}`);
|
|
369
1008
|
break;
|
|
370
1009
|
}
|
|
371
1010
|
if (waveResult.policyApplied === "stop-wave") {
|
|
@@ -373,6 +1012,8 @@ export async function executeOrchBatch(
|
|
|
373
1012
|
// ── TS-009: Persist state on stop-wave ──
|
|
374
1013
|
persistRuntimeState("stop-wave", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
375
1014
|
onNotify(ORCH_MESSAGES.orchBatchStopped(batchState.batchId, "stop-wave"), "error");
|
|
1015
|
+
// TP-040: Emit batch_paused event (via terminal helper for dedup)
|
|
1016
|
+
emitTerminalEvent(`Stopped by stop-wave policy at wave ${waveIdx + 1}`);
|
|
376
1017
|
break;
|
|
377
1018
|
}
|
|
378
1019
|
}
|
|
@@ -411,6 +1052,11 @@ export async function executeOrchBatch(
|
|
|
411
1052
|
// ── TS-009: Persist state on executing→merging transition ──
|
|
412
1053
|
persistRuntimeState("merge-start", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
413
1054
|
onNotify(ORCH_MESSAGES.orchMergeStart(waveIdx + 1, mergeableLaneCount), "info");
|
|
1055
|
+
// TP-040: Emit merge_start event
|
|
1056
|
+
emitEvent(stateRoot, {
|
|
1057
|
+
...buildEngineEventBase("merge_start", batchState.batchId, waveIdx, batchState.phase),
|
|
1058
|
+
laneCount: mergeableLaneCount,
|
|
1059
|
+
}, onEngineEvent);
|
|
414
1060
|
|
|
415
1061
|
mergeResult = mergeWaveByRepo(
|
|
416
1062
|
waveResult.allocatedLanes,
|
|
@@ -474,12 +1120,27 @@ export async function executeOrchBatch(
|
|
|
474
1120
|
|
|
475
1121
|
if (mergeResult.status === "succeeded") {
|
|
476
1122
|
onNotify(ORCH_MESSAGES.orchMergeComplete(waveIdx + 1, mergedCount, mergeTotalSec), "info");
|
|
1123
|
+
|
|
1124
|
+
// TP-040: Emit merge_success event
|
|
1125
|
+
emitEvent(stateRoot, {
|
|
1126
|
+
...buildEngineEventBase("merge_success", batchState.batchId, waveIdx, batchState.phase),
|
|
1127
|
+
laneCount: mergedCount,
|
|
1128
|
+
durationMs: mergeResult.totalDurationMs,
|
|
1129
|
+
totalWaves: rawWaves.length,
|
|
1130
|
+
}, onEngineEvent);
|
|
477
1131
|
} else {
|
|
478
1132
|
onNotify(
|
|
479
1133
|
ORCH_MESSAGES.orchMergeFailed(waveIdx + 1, mergeResult.failedLane ?? 0, mergeResult.failureReason || "unknown"),
|
|
480
1134
|
"error",
|
|
481
1135
|
);
|
|
482
1136
|
|
|
1137
|
+
// TP-040: Emit merge_failed event
|
|
1138
|
+
emitEvent(stateRoot, {
|
|
1139
|
+
...buildEngineEventBase("merge_failed", batchState.batchId, waveIdx, batchState.phase),
|
|
1140
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1141
|
+
error: mergeResult.failureReason || "unknown",
|
|
1142
|
+
}, onEngineEvent);
|
|
1143
|
+
|
|
483
1144
|
// Emit repo-divergence summary when partial is caused by cross-repo outcome differences
|
|
484
1145
|
if (mergeResult.status === "partial") {
|
|
485
1146
|
const repoSummary = formatRepoMergeSummary(mergeResult);
|
|
@@ -509,6 +1170,13 @@ export async function executeOrchBatch(
|
|
|
509
1170
|
ORCH_MESSAGES.orchMergeFailed(waveIdx + 1, mergeResult.failedLane, mergeResult.failureReason || "unknown"),
|
|
510
1171
|
"error",
|
|
511
1172
|
);
|
|
1173
|
+
|
|
1174
|
+
// TP-040 R002: Emit merge_failed for mixed-outcome/no-mergeable-lane path
|
|
1175
|
+
emitEvent(stateRoot, {
|
|
1176
|
+
...buildEngineEventBase("merge_failed", batchState.batchId, waveIdx, batchState.phase),
|
|
1177
|
+
laneNumber: mergeResult.failedLane,
|
|
1178
|
+
error: mergeResult.failureReason,
|
|
1179
|
+
}, onEngineEvent);
|
|
512
1180
|
} else {
|
|
513
1181
|
// No mergeable lanes and no mixed outcomes (e.g., only skipped tasks)
|
|
514
1182
|
onNotify(ORCH_MESSAGES.orchMergeSkipped(waveIdx + 1), "info");
|
|
@@ -565,6 +1233,10 @@ export async function executeOrchBatch(
|
|
|
565
1233
|
batchState.resilience = defaultResilienceState();
|
|
566
1234
|
}
|
|
567
1235
|
|
|
1236
|
+
// Extract repoId and lane for event attribution before entering retry loop
|
|
1237
|
+
const mergeRepoId = extractFailedRepoId(mergeResult) ?? null;
|
|
1238
|
+
const mergeFailedLane = mergeResult.failedLane ?? undefined;
|
|
1239
|
+
|
|
568
1240
|
const retryOutcome = applyMergeRetryLoop(
|
|
569
1241
|
mergeResult,
|
|
570
1242
|
waveIdx,
|
|
@@ -595,6 +1267,17 @@ export async function executeOrchBatch(
|
|
|
595
1267
|
batchState.mergeResults[batchState.mergeResults.length - 1] = result;
|
|
596
1268
|
},
|
|
597
1269
|
sleep: sleepSync,
|
|
1270
|
+
// TP-039 R004: Emit attempt event only when retry is actually scheduled,
|
|
1271
|
+
// with accurate classification/attempt data from the retry decision.
|
|
1272
|
+
onRetryAttempt: (decision) => {
|
|
1273
|
+
emitTier0Event(stateRoot, {
|
|
1274
|
+
...buildTier0EventBase("tier0_recovery_attempt", batchState.batchId, waveIdx, "merge_timeout", decision.currentAttempt, decision.maxAttempts),
|
|
1275
|
+
laneNumber: mergeFailedLane,
|
|
1276
|
+
repoId: mergeRepoId,
|
|
1277
|
+
classification: decision.classification,
|
|
1278
|
+
cooldownMs: decision.cooldownMs,
|
|
1279
|
+
});
|
|
1280
|
+
},
|
|
598
1281
|
},
|
|
599
1282
|
);
|
|
600
1283
|
|
|
@@ -602,6 +1285,17 @@ export async function executeOrchBatch(
|
|
|
602
1285
|
mergeResult = retryOutcome.mergeResult;
|
|
603
1286
|
batchState.phase = "executing";
|
|
604
1287
|
persistRuntimeState("merge-retry-succeeded", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1288
|
+
|
|
1289
|
+
// Emit merge retry success event
|
|
1290
|
+
emitTier0Event(stateRoot, {
|
|
1291
|
+
...buildTier0EventBase("tier0_recovery_success", batchState.batchId, waveIdx, "merge_timeout", retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts),
|
|
1292
|
+
laneNumber: mergeFailedLane,
|
|
1293
|
+
repoId: mergeRepoId,
|
|
1294
|
+
classification: retryOutcome.classification ?? undefined,
|
|
1295
|
+
resolution: `Merge retry succeeded at wave ${waveIdx + 1}`,
|
|
1296
|
+
scopeKey: retryOutcome.scopeKey,
|
|
1297
|
+
});
|
|
1298
|
+
|
|
605
1299
|
// Fall through to normal post-merge flow (worktree cleanup, etc.)
|
|
606
1300
|
} else if (retryOutcome.kind === "safe_stop") {
|
|
607
1301
|
mergeResult = retryOutcome.mergeResult;
|
|
@@ -609,6 +1303,24 @@ export async function executeOrchBatch(
|
|
|
609
1303
|
batchState.errors.push(retryOutcome.errorMessage);
|
|
610
1304
|
persistRuntimeState("merge-rollback-safe-stop", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
611
1305
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
1306
|
+
|
|
1307
|
+
// Emit merge safe-stop event (treated as exhausted — no further automatic recovery possible)
|
|
1308
|
+
const mergeSafeStopSuggestion = "Merge rollback failed — batch force-paused for manual recovery. Check .pi/verification/ for recovery commands.";
|
|
1309
|
+
emitTier0Event(stateRoot, {
|
|
1310
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "merge_timeout", retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts),
|
|
1311
|
+
laneNumber: mergeFailedLane,
|
|
1312
|
+
repoId: mergeRepoId,
|
|
1313
|
+
classification: retryOutcome.classification ?? undefined,
|
|
1314
|
+
error: retryOutcome.errorMessage,
|
|
1315
|
+
scopeKey: retryOutcome.scopeKey,
|
|
1316
|
+
suggestion: mergeSafeStopSuggestion,
|
|
1317
|
+
});
|
|
1318
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "merge_timeout",
|
|
1319
|
+
retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts,
|
|
1320
|
+
retryOutcome.errorMessage, [], mergeSafeStopSuggestion,
|
|
1321
|
+
{ laneNumber: mergeFailedLane, repoId: mergeRepoId, classification: retryOutcome.classification ?? undefined, scopeKey: retryOutcome.scopeKey },
|
|
1322
|
+
);
|
|
1323
|
+
|
|
612
1324
|
preserveWorktreesForResume = true;
|
|
613
1325
|
break;
|
|
614
1326
|
} else if (retryOutcome.kind === "exhausted") {
|
|
@@ -625,6 +1337,23 @@ export async function executeOrchBatch(
|
|
|
625
1337
|
maxAttempts: retryOutcome.lastDecision.maxAttempts,
|
|
626
1338
|
});
|
|
627
1339
|
|
|
1340
|
+
// Emit merge retry exhausted event
|
|
1341
|
+
const mergeExhaustedSuggestion = `Merge retry exhausted (${retryOutcome.classification ?? "unknown"}) after ${retryOutcome.lastDecision.currentAttempt} attempt(s). Investigate merge failure and retry manually.`;
|
|
1342
|
+
emitTier0Event(stateRoot, {
|
|
1343
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "merge_timeout", retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts),
|
|
1344
|
+
laneNumber: mergeFailedLane,
|
|
1345
|
+
repoId: mergeRepoId,
|
|
1346
|
+
classification: retryOutcome.classification ?? undefined,
|
|
1347
|
+
error: exhaustionMsg,
|
|
1348
|
+
scopeKey: retryOutcome.scopeKey,
|
|
1349
|
+
suggestion: mergeExhaustedSuggestion,
|
|
1350
|
+
});
|
|
1351
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "merge_timeout",
|
|
1352
|
+
retryOutcome.lastDecision.currentAttempt, retryOutcome.lastDecision.maxAttempts,
|
|
1353
|
+
exhaustionMsg, [], mergeExhaustedSuggestion,
|
|
1354
|
+
{ laneNumber: mergeFailedLane, repoId: mergeRepoId, classification: retryOutcome.classification ?? undefined, scopeKey: retryOutcome.scopeKey },
|
|
1355
|
+
);
|
|
1356
|
+
|
|
628
1357
|
batchState.phase = "paused";
|
|
629
1358
|
batchState.errors.push(exhaustionMsg);
|
|
630
1359
|
persistRuntimeState("merge-retry-exhausted", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
@@ -807,17 +1536,152 @@ export async function executeOrchBatch(
|
|
|
807
1536
|
}
|
|
808
1537
|
|
|
809
1538
|
if (cleanupGateFailures.length > 0) {
|
|
810
|
-
|
|
1539
|
+
// ── TP-039: Tier 0 — Cleanup gate retry ──────────────
|
|
1540
|
+
// Before pausing, attempt one more force cleanup + prune
|
|
1541
|
+
// on the stale worktrees. This handles cases where the
|
|
1542
|
+
// first force cleanup partially succeeded (e.g., directory
|
|
1543
|
+
// removed but git state not yet pruned).
|
|
1544
|
+
if (!batchState.resilience) {
|
|
1545
|
+
batchState.resilience = defaultResilienceState();
|
|
1546
|
+
}
|
|
811
1547
|
|
|
812
|
-
|
|
1548
|
+
const cleanupBudget = TIER0_RETRY_BUDGETS.cleanup_gate;
|
|
1549
|
+
const cleanupScopeKey = tier0WaveScopeKey("cleanup_gate", waveIdx);
|
|
1550
|
+
const cleanupRetryCount = batchState.resilience.retryCountByScope[cleanupScopeKey] ?? 0;
|
|
813
1551
|
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
1552
|
+
if (cleanupRetryCount < cleanupBudget.maxRetries) {
|
|
1553
|
+
batchState.resilience.retryCountByScope[cleanupScopeKey] = cleanupRetryCount + 1;
|
|
1554
|
+
|
|
1555
|
+
execLog("batch", batchState.batchId,
|
|
1556
|
+
`tier0: retrying cleanup gate (attempt ${cleanupRetryCount + 1}/${cleanupBudget.maxRetries})`,
|
|
1557
|
+
{ cleanupScopeKey, staleCount: cleanupGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0) },
|
|
1558
|
+
);
|
|
1559
|
+
|
|
1560
|
+
// Emit attempt event
|
|
1561
|
+
const staleWorktreeCount = cleanupGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0);
|
|
1562
|
+
emitTier0Event(stateRoot, {
|
|
1563
|
+
...buildTier0EventBase("tier0_recovery_attempt", batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount + 1, cleanupBudget.maxRetries),
|
|
1564
|
+
repoId: null, // wave-scoped: cleanup gate spans all repos
|
|
1565
|
+
classification: `stale_worktrees:${staleWorktreeCount}`,
|
|
1566
|
+
cooldownMs: cleanupBudget.cooldownMs,
|
|
1567
|
+
scopeKey: cleanupScopeKey,
|
|
1568
|
+
});
|
|
1569
|
+
|
|
1570
|
+
if (cleanupBudget.cooldownMs > 0) {
|
|
1571
|
+
sleepSync(cleanupBudget.cooldownMs);
|
|
1572
|
+
}
|
|
1573
|
+
|
|
1574
|
+
// Force-cleanup each stale worktree again
|
|
1575
|
+
for (const failure of cleanupGateFailures) {
|
|
1576
|
+
const remaining = listWorktrees(resetPrefix, failure.repoRoot, resetOpId, batchState.batchId);
|
|
1577
|
+
for (const wt of remaining) {
|
|
1578
|
+
if (failure.staleWorktrees.includes(wt.path)) {
|
|
1579
|
+
forceCleanupWorktree(wt, failure.repoRoot, batchState.batchId);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
// Prune after force cleanup
|
|
1583
|
+
runGit(["worktree", "prune"], failure.repoRoot);
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
// Re-check: are any worktrees still stale?
|
|
1587
|
+
const retriedGateFailures: CleanupGateRepoFailure[] = [];
|
|
1588
|
+
for (const failure of cleanupGateFailures) {
|
|
1589
|
+
const remaining = listWorktrees(resetPrefix, failure.repoRoot, resetOpId, batchState.batchId);
|
|
1590
|
+
const remainingPaths = new Set(remaining.map(wt => wt.path));
|
|
1591
|
+
const stillStale = failure.staleWorktrees.filter(p => remainingPaths.has(p));
|
|
1592
|
+
if (stillStale.length > 0) {
|
|
1593
|
+
retriedGateFailures.push({
|
|
1594
|
+
repoRoot: failure.repoRoot,
|
|
1595
|
+
repoId: failure.repoId,
|
|
1596
|
+
staleWorktrees: stillStale,
|
|
1597
|
+
});
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
if (retriedGateFailures.length === 0) {
|
|
1602
|
+
execLog("batch", batchState.batchId,
|
|
1603
|
+
`tier0: cleanup gate retry succeeded — all stale worktrees removed`,
|
|
1604
|
+
{ cleanupScopeKey },
|
|
1605
|
+
);
|
|
1606
|
+
onNotify(
|
|
1607
|
+
`✅ Tier 0: Cleanup gate retry succeeded at wave ${waveIdx + 1} — continuing`,
|
|
1608
|
+
"info",
|
|
1609
|
+
);
|
|
1610
|
+
|
|
1611
|
+
// Emit success event
|
|
1612
|
+
emitTier0Event(stateRoot, {
|
|
1613
|
+
...buildTier0EventBase("tier0_recovery_success", batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount + 1, cleanupBudget.maxRetries),
|
|
1614
|
+
repoId: null, // wave-scoped
|
|
1615
|
+
resolution: `Cleanup gate retry succeeded — all stale worktrees removed at wave ${waveIdx + 1}`,
|
|
1616
|
+
scopeKey: cleanupScopeKey,
|
|
1617
|
+
});
|
|
1618
|
+
|
|
1619
|
+
persistRuntimeState("tier0-cleanup-retry-success", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1620
|
+
// Fall through to continue the wave loop (don't break)
|
|
1621
|
+
} else {
|
|
1622
|
+
// Retry failed — fall through to pausing
|
|
1623
|
+
const gatePolicyResult = computeCleanupGatePolicy(waveIdx, retriedGateFailures);
|
|
1624
|
+
|
|
1625
|
+
execLog("batch", batchState.batchId,
|
|
1626
|
+
`tier0: cleanup gate retry failed — still ${retriedGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0)} stale worktree(s), pausing batch`,
|
|
1627
|
+
gatePolicyResult.logDetails,
|
|
1628
|
+
);
|
|
1629
|
+
|
|
1630
|
+
const stillStaleCount = retriedGateFailures.reduce((n, f) => n + f.staleWorktrees.length, 0);
|
|
1631
|
+
const cleanupRetryError = `Cleanup gate retry failed — ${stillStaleCount} stale worktree(s) remain`;
|
|
1632
|
+
const cleanupRetrySuggestion = `Post-merge cleanup retry did not remove all stale worktrees. Manually remove the remaining ${stillStaleCount} worktree(s) and prune git state.`;
|
|
1633
|
+
const cleanupRetryAffected = retriedGateFailures.flatMap(f => f.staleWorktrees);
|
|
1634
|
+
// Emit exhausted event (retry attempted but failed)
|
|
1635
|
+
emitTier0Event(stateRoot, {
|
|
1636
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount + 1, cleanupBudget.maxRetries),
|
|
1637
|
+
repoId: null, // wave-scoped
|
|
1638
|
+
error: cleanupRetryError,
|
|
1639
|
+
scopeKey: cleanupScopeKey,
|
|
1640
|
+
affectedTaskIds: cleanupRetryAffected,
|
|
1641
|
+
suggestion: cleanupRetrySuggestion,
|
|
1642
|
+
});
|
|
1643
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount + 1, cleanupBudget.maxRetries,
|
|
1644
|
+
cleanupRetryError, cleanupRetryAffected, cleanupRetrySuggestion,
|
|
1645
|
+
{ repoId: null, scopeKey: cleanupScopeKey },
|
|
1646
|
+
);
|
|
1647
|
+
|
|
1648
|
+
batchState.phase = gatePolicyResult.targetPhase;
|
|
1649
|
+
batchState.errors.push(gatePolicyResult.errorMessage);
|
|
1650
|
+
persistRuntimeState(gatePolicyResult.persistTrigger, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1651
|
+
onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
|
|
1652
|
+
preserveWorktreesForResume = true;
|
|
1653
|
+
break;
|
|
1654
|
+
}
|
|
1655
|
+
} else {
|
|
1656
|
+
// Cleanup retry budget exhausted — pause immediately
|
|
1657
|
+
const gatePolicyResult = computeCleanupGatePolicy(waveIdx, cleanupGateFailures);
|
|
1658
|
+
|
|
1659
|
+
execLog("batch", batchState.batchId, `cleanup gate failed — pausing batch (retry budget exhausted)`, gatePolicyResult.logDetails);
|
|
1660
|
+
|
|
1661
|
+
// Emit exhausted event (budget already consumed from prior waves)
|
|
1662
|
+
const cleanupBudgetError = `Cleanup gate retry budget exhausted (${cleanupRetryCount}/${cleanupBudget.maxRetries})`;
|
|
1663
|
+
const cleanupBudgetSuggestion = `Cleanup gate retry budget was already consumed. Manually remove stale worktrees and prune git state.`;
|
|
1664
|
+
const cleanupBudgetAffected = cleanupGateFailures.flatMap(f => f.staleWorktrees);
|
|
1665
|
+
emitTier0Event(stateRoot, {
|
|
1666
|
+
...buildTier0EventBase("tier0_recovery_exhausted", batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount, cleanupBudget.maxRetries),
|
|
1667
|
+
repoId: null, // wave-scoped
|
|
1668
|
+
error: cleanupBudgetError,
|
|
1669
|
+
scopeKey: cleanupScopeKey,
|
|
1670
|
+
affectedTaskIds: cleanupBudgetAffected,
|
|
1671
|
+
suggestion: cleanupBudgetSuggestion,
|
|
1672
|
+
});
|
|
1673
|
+
emitTier0Escalation(stateRoot, batchState.batchId, waveIdx, "cleanup_gate", cleanupRetryCount, cleanupBudget.maxRetries,
|
|
1674
|
+
cleanupBudgetError, cleanupBudgetAffected, cleanupBudgetSuggestion,
|
|
1675
|
+
{ repoId: null, scopeKey: cleanupScopeKey },
|
|
1676
|
+
);
|
|
1677
|
+
|
|
1678
|
+
batchState.phase = gatePolicyResult.targetPhase;
|
|
1679
|
+
batchState.errors.push(gatePolicyResult.errorMessage);
|
|
1680
|
+
persistRuntimeState(gatePolicyResult.persistTrigger, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1681
|
+
onNotify(gatePolicyResult.notifyMessage, gatePolicyResult.notifyLevel);
|
|
1682
|
+
preserveWorktreesForResume = true;
|
|
1683
|
+
break;
|
|
1684
|
+
}
|
|
821
1685
|
}
|
|
822
1686
|
}
|
|
823
1687
|
}
|
|
@@ -1144,22 +2008,22 @@ export async function executeOrchBatch(
|
|
|
1144
2008
|
// are warnings that preserve the orch branch for manual integration.
|
|
1145
2009
|
// Gate: only run for terminal phases (completed/failed). Paused/stopped batches
|
|
1146
2010
|
// are not yet done — integration would mutate refs prematurely.
|
|
1147
|
-
|
|
2011
|
+
//
|
|
2012
|
+
// TP-043: "supervised" and "auto" integration modes are now owned by the
|
|
2013
|
+
// supervisor agent (which stays alive through post-batch integration).
|
|
2014
|
+
// The legacy engine fast-forward only runs for "auto" mode when no
|
|
2015
|
+
// supervisor is active (fallback). For "supervised" mode, the supervisor
|
|
2016
|
+
// always handles integration.
|
|
1148
2017
|
const mergedTaskCount = batchState.succeededTasks;
|
|
1149
2018
|
const isTerminalPhase = batchState.phase === "completed" || batchState.phase === "failed";
|
|
1150
2019
|
if (isTerminalPhase && !preserveWorktreesForResume && batchState.orchBranch && mergedTaskCount > 0) {
|
|
1151
|
-
if (orchConfig.orchestrator.integration === "auto") {
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
onNotify,
|
|
1159
|
-
);
|
|
1160
|
-
}
|
|
1161
|
-
// Manual mode (default) or auto-integration skipped: show integration guidance
|
|
1162
|
-
if (!autoIntegrated) {
|
|
2020
|
+
if (orchConfig.orchestrator.integration === "supervised" || orchConfig.orchestrator.integration === "auto") {
|
|
2021
|
+
// TP-043: Supervisor-managed integration modes. The supervisor
|
|
2022
|
+
// agent handles integration after batch_complete event. The engine
|
|
2023
|
+
// does NOT perform legacy fast-forward here — defer to supervisor.
|
|
2024
|
+
execLog("batch", batchState.batchId, `integration deferred to supervisor (mode: ${orchConfig.orchestrator.integration})`);
|
|
2025
|
+
} else {
|
|
2026
|
+
// Manual mode (default): show integration guidance
|
|
1163
2027
|
onNotify(
|
|
1164
2028
|
ORCH_MESSAGES.orchIntegrationManual(batchState.orchBranch, batchState.baseBranch, mergedTaskCount),
|
|
1165
2029
|
"info",
|
|
@@ -1170,6 +2034,9 @@ export async function executeOrchBatch(
|
|
|
1170
2034
|
// ── TS-009: Persist terminal state ──
|
|
1171
2035
|
persistRuntimeState("batch-terminal", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1172
2036
|
|
|
2037
|
+
// ── TP-040: Emit batch terminal event (R002: unified via helper) ─
|
|
2038
|
+
emitTerminalEvent();
|
|
2039
|
+
|
|
1173
2040
|
// ── TP-031: Emit diagnostic reports (JSONL + markdown) ──
|
|
1174
2041
|
// Non-fatal: errors are logged but never crash batch finalization.
|
|
1175
2042
|
emitDiagnosticReports(assembleDiagnosticInput(orchConfig, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, stateRoot));
|