taskplane 0.22.1 → 0.22.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1510,6 +1510,9 @@ export async function executeOrchBatch(
|
|
|
1510
1510
|
failedLane: mixedOutcomeLanes[0].laneNumber,
|
|
1511
1511
|
failureReason,
|
|
1512
1512
|
};
|
|
1513
|
+
// Update the already-pushed references so persisted state reflects "partial"
|
|
1514
|
+
allMergeResults[allMergeResults.length - 1] = mergeResult;
|
|
1515
|
+
batchState.mergeResults[batchState.mergeResults.length - 1] = mergeResult;
|
|
1513
1516
|
}
|
|
1514
1517
|
|
|
1515
1518
|
// Emit overall merge result notification
|
|
@@ -1567,6 +1570,10 @@ export async function executeOrchBatch(
|
|
|
1567
1570
|
`Automatic partial-branch merge is disabled to avoid dropping succeeded commits.`,
|
|
1568
1571
|
totalDurationMs: 0,
|
|
1569
1572
|
};
|
|
1573
|
+
// Keep mergeResults in sync even when no mergeable lane exists.
|
|
1574
|
+
// Downstream retry/update paths assume the current wave has an entry.
|
|
1575
|
+
allMergeResults.push(mergeResult);
|
|
1576
|
+
batchState.mergeResults.push(mergeResult);
|
|
1570
1577
|
onNotify(
|
|
1571
1578
|
ORCH_MESSAGES.orchMergeFailed(waveIdx + 1, mergeResult.failedLane, mergeResult.failureReason || "unknown"),
|
|
1572
1579
|
"error",
|
|
@@ -15,16 +15,16 @@ import { ORCH_MESSAGES, computeIntegrateCleanupResult } from "./messages.ts";
|
|
|
15
15
|
import type { IntegrateCleanupRepoFindings } from "./messages.ts";
|
|
16
16
|
import { computeWaveAssignments } from "./waves.ts";
|
|
17
17
|
import { createOrchWidget, formatDependencyGraph, formatWavePlan } from "./formatting.ts";
|
|
18
|
-
import { deleteBatchState, loadBatchState, detectOrphanSessions, parseOrchSessionNames } from "./persistence.ts";
|
|
18
|
+
import { deleteBatchState, loadBatchState, saveBatchState, detectOrphanSessions, parseOrchSessionNames } from "./persistence.ts";
|
|
19
19
|
import { deleteStaleBranches, listWorktrees, resolveWorktreeBasePath, formatPreflightResults, runPreflight } from "./worktree.ts";
|
|
20
|
-
import { executeLane } from "./execution.ts";
|
|
20
|
+
import { computeTransitiveDependents, executeLane } from "./execution.ts";
|
|
21
21
|
import { executeOrchBatch } from "./engine.ts";
|
|
22
22
|
import { formatDiscoveryResults, runDiscovery } from "./discovery.ts";
|
|
23
23
|
import { formatOrchSessions, listOrchSessions } from "./sessions.ts";
|
|
24
24
|
import { getCurrentBranch, runGit } from "./git.ts";
|
|
25
25
|
import { hasConfigFiles, resolveConfigRoot, loadOrchestratorConfig, loadSupervisorConfig, loadTaskRunnerConfig } from "./config.ts";
|
|
26
26
|
import { resolveOperatorId } from "./naming.ts";
|
|
27
|
-
import { resumeOrchBatch } from "./resume.ts";
|
|
27
|
+
import { reconstructAllocatedLanes, resumeOrchBatch } from "./resume.ts";
|
|
28
28
|
import { buildExecutionContext } from "./workspace.ts";
|
|
29
29
|
import { openSettingsTui } from "./settings-tui.ts";
|
|
30
30
|
import { loadProjectConfig } from "./config-loader.ts";
|
|
@@ -970,6 +970,7 @@ export function startBatchInWorker(
|
|
|
970
970
|
wkData.workspaceRoot,
|
|
971
971
|
wkData.agentRoot,
|
|
972
972
|
wkData.force ?? false,
|
|
973
|
+
onSupervisorAlert ?? null,
|
|
973
974
|
)
|
|
974
975
|
: () => executeOrchBatch(
|
|
975
976
|
wkData.args ?? "",
|
|
@@ -982,6 +983,8 @@ export function startBatchInWorker(
|
|
|
982
983
|
wsConfig,
|
|
983
984
|
wkData.workspaceRoot,
|
|
984
985
|
wkData.agentRoot,
|
|
986
|
+
null, // onEngineEvent
|
|
987
|
+
onSupervisorAlert ?? null,
|
|
985
988
|
);
|
|
986
989
|
startBatchAsync(fallbackFn, batchState, ctx, updateWidget, onTerminal);
|
|
987
990
|
return null;
|
|
@@ -2330,6 +2333,449 @@ export default function (pi: ExtensionAPI) {
|
|
|
2330
2333
|
return messages.join("\n");
|
|
2331
2334
|
}
|
|
2332
2335
|
|
|
2336
|
+
// ── TP-077: Supervisor Recovery Tools ────────────────────────────
|
|
2337
|
+
|
|
2338
|
+
/**
|
|
2339
|
+
* Core logic for orch_retry_task. Resets a failed task to pending for re-execution.
|
|
2340
|
+
*
|
|
2341
|
+
* Modifies persisted batch state on disk and updates in-memory state.
|
|
2342
|
+
* The engine picks up the state change on its next poll cycle.
|
|
2343
|
+
*/
|
|
2344
|
+
function doOrchRetryTask(taskId: string, ctx: ExtensionContext): string {
|
|
2345
|
+
// TP-077 R001-1: Reject while engine is actively running (no IPC retry path)
|
|
2346
|
+
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
2347
|
+
if (activePhases.has(orchBatchState.phase)) {
|
|
2348
|
+
return `❌ Cannot retry task while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
2349
|
+
}
|
|
2350
|
+
|
|
2351
|
+
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
2352
|
+
|
|
2353
|
+
// Load persisted state
|
|
2354
|
+
let state: PersistedBatchState | null = null;
|
|
2355
|
+
try {
|
|
2356
|
+
state = loadBatchState(stateRoot);
|
|
2357
|
+
} catch (err) {
|
|
2358
|
+
return `❌ Failed to load batch state: ${err instanceof Error ? err.message : String(err)}`;
|
|
2359
|
+
}
|
|
2360
|
+
|
|
2361
|
+
if (!state) {
|
|
2362
|
+
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
// Find the task
|
|
2366
|
+
const taskRecord = state.tasks.find(t => t.taskId === taskId);
|
|
2367
|
+
if (!taskRecord) {
|
|
2368
|
+
const knownIds = state.tasks.map(t => t.taskId).join(", ");
|
|
2369
|
+
return `❌ Task "${taskId}" not found in batch ${state.batchId}.\nKnown tasks: ${knownIds || "(none)"}`;
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
// Validate: only failed or stalled tasks can be retried
|
|
2373
|
+
if (taskRecord.status !== "failed" && taskRecord.status !== "stalled") {
|
|
2374
|
+
return `❌ Cannot retry task "${taskId}" — current status is "${taskRecord.status}". Only failed or stalled tasks can be retried.`;
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
const prevStatus = taskRecord.status;
|
|
2378
|
+
|
|
2379
|
+
// Reset task to pending
|
|
2380
|
+
taskRecord.status = "pending";
|
|
2381
|
+
taskRecord.exitReason = "";
|
|
2382
|
+
taskRecord.doneFileFound = false;
|
|
2383
|
+
taskRecord.startedAt = null;
|
|
2384
|
+
taskRecord.endedAt = null;
|
|
2385
|
+
taskRecord.exitDiagnostic = undefined;
|
|
2386
|
+
taskRecord.partialProgressCommits = undefined;
|
|
2387
|
+
taskRecord.partialProgressBranch = undefined;
|
|
2388
|
+
|
|
2389
|
+
// Adjust counters: only decrement failedTasks if the task was in a failure state
|
|
2390
|
+
if (prevStatus === "failed" || prevStatus === "stalled") {
|
|
2391
|
+
state.failedTasks = Math.max(0, state.failedTasks - 1);
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2394
|
+
// Recompute blocked dependents — the retried task is no longer a failure,
|
|
2395
|
+
// so tasks that were blocked solely by it should be unblocked.
|
|
2396
|
+
const remainingFailures = new Set<string>();
|
|
2397
|
+
for (const t of state.tasks) {
|
|
2398
|
+
if (t.status === "failed" || t.status === "stalled") {
|
|
2399
|
+
remainingFailures.add(t.taskId);
|
|
2400
|
+
}
|
|
2401
|
+
}
|
|
2402
|
+
if (orchBatchState.dependencyGraph && orchBatchState.batchId === state.batchId) {
|
|
2403
|
+
const newBlocked = computeTransitiveDependents(remainingFailures, orchBatchState.dependencyGraph);
|
|
2404
|
+
state.blockedTaskIds = [...newBlocked].sort();
|
|
2405
|
+
state.blockedTasks = newBlocked.size;
|
|
2406
|
+
} else if (remainingFailures.size === 0) {
|
|
2407
|
+
state.blockedTaskIds = [];
|
|
2408
|
+
state.blockedTasks = 0;
|
|
2409
|
+
}
|
|
2410
|
+
|
|
2411
|
+
// TP-077 R001-3: Phase transition — terminal "failed" → "stopped" (resumable with force)
|
|
2412
|
+
// "stopped" and "paused" are kept as-is (already resumable).
|
|
2413
|
+
if (state.phase === "failed") {
|
|
2414
|
+
state.phase = "stopped";
|
|
2415
|
+
}
|
|
2416
|
+
|
|
2417
|
+
// Update timestamp
|
|
2418
|
+
state.updatedAt = Date.now();
|
|
2419
|
+
|
|
2420
|
+
// Persist
|
|
2421
|
+
try {
|
|
2422
|
+
saveBatchState(JSON.stringify(state, null, 2), stateRoot);
|
|
2423
|
+
} catch (err) {
|
|
2424
|
+
return `❌ Failed to persist state after retry: ${err instanceof Error ? err.message : String(err)}`;
|
|
2425
|
+
}
|
|
2426
|
+
|
|
2427
|
+
// Sync in-memory state if batch IDs match
|
|
2428
|
+
if (orchBatchState.batchId === state.batchId) {
|
|
2429
|
+
orchBatchState.failedTasks = state.failedTasks;
|
|
2430
|
+
orchBatchState.blockedTasks = state.blockedTasks;
|
|
2431
|
+
orchBatchState.blockedTaskIds = new Set(state.blockedTaskIds);
|
|
2432
|
+
if (state.phase === "stopped" && orchBatchState.phase === "failed") {
|
|
2433
|
+
orchBatchState.phase = "stopped";
|
|
2434
|
+
}
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
updateOrchWidget();
|
|
2438
|
+
|
|
2439
|
+
const resumeHint = state.phase === "stopped"
|
|
2440
|
+
? "Use orch_resume(force=true) to re-execute the batch."
|
|
2441
|
+
: "Use orch_resume() to re-execute the batch.";
|
|
2442
|
+
return `✅ Task "${taskId}" reset to pending for re-execution.\n` +
|
|
2443
|
+
` Previous status: ${prevStatus}\n` +
|
|
2444
|
+
` Batch phase: ${state.phase} | Failed: ${state.failedTasks}/${state.totalTasks}\n` +
|
|
2445
|
+
` ${resumeHint}`;
|
|
2446
|
+
}
|
|
2447
|
+
|
|
2448
|
+
/**
|
|
2449
|
+
* Core logic for orch_skip_task. Marks a task as skipped and unblocks dependents.
|
|
2450
|
+
*
|
|
2451
|
+
* Modifies persisted batch state on disk and updates in-memory state.
|
|
2452
|
+
* The engine picks up the state change on its next poll cycle.
|
|
2453
|
+
*/
|
|
2454
|
+
function doOrchSkipTask(taskId: string, ctx: ExtensionContext): string {
|
|
2455
|
+
// TP-077 R001-1: Reject while engine is actively running (no IPC skip path)
|
|
2456
|
+
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
2457
|
+
if (activePhases.has(orchBatchState.phase)) {
|
|
2458
|
+
return `❌ Cannot skip task while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
2459
|
+
}
|
|
2460
|
+
|
|
2461
|
+
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
2462
|
+
|
|
2463
|
+
// Load persisted state
|
|
2464
|
+
let state: PersistedBatchState | null = null;
|
|
2465
|
+
try {
|
|
2466
|
+
state = loadBatchState(stateRoot);
|
|
2467
|
+
} catch (err) {
|
|
2468
|
+
return `❌ Failed to load batch state: ${err instanceof Error ? err.message : String(err)}`;
|
|
2469
|
+
}
|
|
2470
|
+
|
|
2471
|
+
if (!state) {
|
|
2472
|
+
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
2473
|
+
}
|
|
2474
|
+
|
|
2475
|
+
// Find the task
|
|
2476
|
+
const taskRecord = state.tasks.find(t => t.taskId === taskId);
|
|
2477
|
+
if (!taskRecord) {
|
|
2478
|
+
const knownIds = state.tasks.map(t => t.taskId).join(", ");
|
|
2479
|
+
return `❌ Task "${taskId}" not found in batch ${state.batchId}.\nKnown tasks: ${knownIds || "(none)"}`;
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2482
|
+
// Validate: only failed, stalled, or pending tasks can be skipped
|
|
2483
|
+
if (taskRecord.status !== "failed" && taskRecord.status !== "stalled" && taskRecord.status !== "pending") {
|
|
2484
|
+
return `❌ Cannot skip task "${taskId}" — current status is "${taskRecord.status}". Only failed, stalled, or pending tasks can be skipped.`;
|
|
2485
|
+
}
|
|
2486
|
+
|
|
2487
|
+
const prevStatus = taskRecord.status;
|
|
2488
|
+
const wasFailed = prevStatus === "failed" || prevStatus === "stalled";
|
|
2489
|
+
|
|
2490
|
+
// Mark as skipped
|
|
2491
|
+
taskRecord.status = "skipped";
|
|
2492
|
+
taskRecord.exitReason = "Skipped by supervisor";
|
|
2493
|
+
taskRecord.endedAt = Date.now();
|
|
2494
|
+
|
|
2495
|
+
// Adjust counters
|
|
2496
|
+
state.skippedTasks = (state.skippedTasks ?? 0) + 1;
|
|
2497
|
+
if (wasFailed) {
|
|
2498
|
+
state.failedTasks = Math.max(0, state.failedTasks - 1);
|
|
2499
|
+
}
|
|
2500
|
+
|
|
2501
|
+
// Unblock dependents: recompute which tasks should remain blocked.
|
|
2502
|
+
// After skipping this task, collect the set of remaining failures to
|
|
2503
|
+
// recompute transitive blocked set from the dependency graph.
|
|
2504
|
+
const prevBlocked = new Set(state.blockedTaskIds ?? []);
|
|
2505
|
+
const unblockedTasks: string[] = [];
|
|
2506
|
+
|
|
2507
|
+
const remainingFailures = new Set<string>();
|
|
2508
|
+
for (const t of state.tasks) {
|
|
2509
|
+
if ((t.status === "failed" || t.status === "stalled") && t.taskId !== taskId) {
|
|
2510
|
+
remainingFailures.add(t.taskId);
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
|
|
2514
|
+
// Use in-memory dependency graph if available (batch IDs must match)
|
|
2515
|
+
if (orchBatchState.dependencyGraph && orchBatchState.batchId === state.batchId) {
|
|
2516
|
+
const newBlocked = computeTransitiveDependents(remainingFailures, orchBatchState.dependencyGraph);
|
|
2517
|
+
|
|
2518
|
+
// Find tasks that were blocked but are now unblocked
|
|
2519
|
+
for (const id of prevBlocked) {
|
|
2520
|
+
if (!newBlocked.has(id)) {
|
|
2521
|
+
unblockedTasks.push(id);
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
state.blockedTaskIds = [...newBlocked].sort();
|
|
2526
|
+
state.blockedTasks = newBlocked.size;
|
|
2527
|
+
} else {
|
|
2528
|
+
// No dependency graph available — conservatively remove the skipped
|
|
2529
|
+
// task from blocked list and let the engine re-evaluate on resume.
|
|
2530
|
+
prevBlocked.delete(taskId);
|
|
2531
|
+
state.blockedTaskIds = [...prevBlocked];
|
|
2532
|
+
state.blockedTasks = prevBlocked.size;
|
|
2533
|
+
}
|
|
2534
|
+
|
|
2535
|
+
// TP-077 R001-3: Phase transition — "failed" → "stopped" (resumable with force)
|
|
2536
|
+
if (state.phase === "failed") {
|
|
2537
|
+
state.phase = "stopped";
|
|
2538
|
+
}
|
|
2539
|
+
|
|
2540
|
+
// Update timestamp
|
|
2541
|
+
state.updatedAt = Date.now();
|
|
2542
|
+
|
|
2543
|
+
// Persist
|
|
2544
|
+
try {
|
|
2545
|
+
saveBatchState(JSON.stringify(state, null, 2), stateRoot);
|
|
2546
|
+
} catch (err) {
|
|
2547
|
+
return `❌ Failed to persist state after skip: ${err instanceof Error ? err.message : String(err)}`;
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
// Sync in-memory state if batch IDs match
|
|
2551
|
+
if (orchBatchState.batchId === state.batchId) {
|
|
2552
|
+
orchBatchState.failedTasks = state.failedTasks;
|
|
2553
|
+
orchBatchState.skippedTasks = state.skippedTasks;
|
|
2554
|
+
orchBatchState.blockedTasks = state.blockedTasks;
|
|
2555
|
+
orchBatchState.blockedTaskIds = new Set(state.blockedTaskIds);
|
|
2556
|
+
if (state.phase === "stopped" && orchBatchState.phase === "failed") {
|
|
2557
|
+
orchBatchState.phase = "stopped";
|
|
2558
|
+
}
|
|
2559
|
+
}
|
|
2560
|
+
|
|
2561
|
+
updateOrchWidget();
|
|
2562
|
+
|
|
2563
|
+
const lines = [
|
|
2564
|
+
`✅ Task "${taskId}" marked as skipped.`,
|
|
2565
|
+
` Previous status: ${prevStatus}`,
|
|
2566
|
+
` Batch phase: ${state.phase} | Failed: ${state.failedTasks}, Skipped: ${state.skippedTasks}, Blocked: ${state.blockedTasks} / ${state.totalTasks} total`,
|
|
2567
|
+
];
|
|
2568
|
+
|
|
2569
|
+
if (unblockedTasks.length > 0) {
|
|
2570
|
+
lines.push(` Unblocked tasks: ${unblockedTasks.join(", ")}`);
|
|
2571
|
+
}
|
|
2572
|
+
|
|
2573
|
+
lines.push(" The engine will re-evaluate dependent tasks on next resume cycle.");
|
|
2574
|
+
|
|
2575
|
+
return lines.join("\n");
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
// ── TP-078: Force Merge Tool ─────────────────────────────────────
|
|
2579
|
+
|
|
2580
|
+
/**
|
|
2581
|
+
* Core logic for orch_force_merge. Unblocks mixed-outcome merge failures by
|
|
2582
|
+
* skipping failed tasks, clearing the failed merge entry, and pausing so
|
|
2583
|
+
* resume re-attempts the real merge.
|
|
2584
|
+
*
|
|
2585
|
+
* This is the supervisor's escape hatch when a wave merge was rejected because
|
|
2586
|
+
* some lanes had both succeeded and failed tasks (mixed-outcome). The tool:
|
|
2587
|
+
* 1. Validates the batch is paused/stopped/failed and the wave merge status is "partial"
|
|
2588
|
+
* 2. Verifies the partial failure is specifically the mixed-outcome rejection
|
|
2589
|
+
* 3. If skipFailed=true, marks failed/stalled tasks in the wave as "skipped"
|
|
2590
|
+
* 4. Clears the failed merge entry and sets phase to "paused"
|
|
2591
|
+
* 5. `orch_resume()` re-runs the merge using real git merge logic
|
|
2592
|
+
*/
|
|
2593
|
+
function doOrchForceMerge(waveIndex: number | undefined, skipFailed: boolean, ctx: ExtensionContext): string {
|
|
2594
|
+
// Reject while engine is actively running
|
|
2595
|
+
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
2596
|
+
if (activePhases.has(orchBatchState.phase)) {
|
|
2597
|
+
return `❌ Cannot force merge while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
2598
|
+
}
|
|
2599
|
+
|
|
2600
|
+
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
2601
|
+
|
|
2602
|
+
// Load persisted state
|
|
2603
|
+
let state: PersistedBatchState | null = null;
|
|
2604
|
+
try {
|
|
2605
|
+
state = loadBatchState(stateRoot);
|
|
2606
|
+
} catch (err) {
|
|
2607
|
+
return `❌ Failed to load batch state: ${err instanceof Error ? err.message : String(err)}`;
|
|
2608
|
+
}
|
|
2609
|
+
|
|
2610
|
+
if (!state) {
|
|
2611
|
+
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
2612
|
+
}
|
|
2613
|
+
|
|
2614
|
+
// Force-merge is a recovery action for non-running failed/paused batches.
|
|
2615
|
+
const resumablePhases = new Set(["paused", "stopped", "failed"]);
|
|
2616
|
+
if (!resumablePhases.has(state.phase)) {
|
|
2617
|
+
return `❌ Cannot force merge when batch phase is "${state.phase}". ` +
|
|
2618
|
+
`Force merge is only valid for paused/stopped/failed batches.`;
|
|
2619
|
+
}
|
|
2620
|
+
|
|
2621
|
+
// Determine target wave index (0-based). Default to currentWaveIndex.
|
|
2622
|
+
const targetWave = waveIndex ?? state.currentWaveIndex;
|
|
2623
|
+
|
|
2624
|
+
// Validate wave index
|
|
2625
|
+
if (targetWave < 0 || targetWave >= state.totalWaves) {
|
|
2626
|
+
return `❌ Invalid wave index ${targetWave}. Batch has ${state.totalWaves} wave(s) (0-based: 0..${state.totalWaves - 1}).`;
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
// Find the merge result for the target wave
|
|
2630
|
+
// Walk in reverse to find the latest entry for this wave
|
|
2631
|
+
let mergeResultIdx = -1;
|
|
2632
|
+
for (let i = state.mergeResults.length - 1; i >= 0; i--) {
|
|
2633
|
+
if (state.mergeResults[i].waveIndex === targetWave) {
|
|
2634
|
+
mergeResultIdx = i;
|
|
2635
|
+
break;
|
|
2636
|
+
}
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
// Validate: there must be a merge failure (partial or failed) for this wave
|
|
2640
|
+
if (mergeResultIdx === -1) {
|
|
2641
|
+
return `❌ No merge result found for wave ${targetWave}. Force merge is only needed when a merge failed or was rejected due to mixed-outcome lanes.`;
|
|
2642
|
+
}
|
|
2643
|
+
|
|
2644
|
+
const mergeEntry = state.mergeResults[mergeResultIdx];
|
|
2645
|
+
if (mergeEntry.status === "succeeded") {
|
|
2646
|
+
return `✅ Wave ${targetWave} merge already succeeded. No force merge needed.`;
|
|
2647
|
+
}
|
|
2648
|
+
|
|
2649
|
+
// Only allow force merge for mixed-outcome failures (partial status).
|
|
2650
|
+
// Other failures (conflicts, build failures, repo divergence) need different resolution.
|
|
2651
|
+
if (mergeEntry.status !== "partial") {
|
|
2652
|
+
return `❌ Wave ${targetWave} merge failed with status "${mergeEntry.status}": ${mergeEntry.failureReason || "unknown reason"}.\n` +
|
|
2653
|
+
`Force merge only applies to mixed-outcome lanes (partial). This failure needs manual resolution.`;
|
|
2654
|
+
}
|
|
2655
|
+
|
|
2656
|
+
const failureReason = mergeEntry.failureReason || "";
|
|
2657
|
+
const failureReasonLower = failureReason.toLowerCase();
|
|
2658
|
+
const isMixedOutcomePartial =
|
|
2659
|
+
failureReasonLower.includes("both succeeded and failed tasks") ||
|
|
2660
|
+
failureReasonLower.includes("mixed-outcome") ||
|
|
2661
|
+
failureReasonLower.includes("automatic partial-branch merge is disabled");
|
|
2662
|
+
if (!isMixedOutcomePartial) {
|
|
2663
|
+
return `❌ Wave ${targetWave} has partial merge status, but the failure reason does not match mixed-outcome lanes.\n` +
|
|
2664
|
+
`Reason: ${failureReason || "unknown"}\n` +
|
|
2665
|
+
`Force merge is only valid for the mixed-outcome lane guard. Resolve this merge failure manually.`;
|
|
2666
|
+
}
|
|
2667
|
+
|
|
2668
|
+
// Collect tasks in the target wave
|
|
2669
|
+
const waveTasks = state.wavePlan[targetWave] ?? [];
|
|
2670
|
+
const failedInWave: string[] = [];
|
|
2671
|
+
const succeededInWave: string[] = [];
|
|
2672
|
+
|
|
2673
|
+
for (const taskId of waveTasks) {
|
|
2674
|
+
const task = state.tasks.find(t => t.taskId === taskId);
|
|
2675
|
+
if (!task) continue;
|
|
2676
|
+
if (task.status === "failed" || task.status === "stalled") {
|
|
2677
|
+
failedInWave.push(taskId);
|
|
2678
|
+
} else if (task.status === "succeeded") {
|
|
2679
|
+
succeededInWave.push(taskId);
|
|
2680
|
+
}
|
|
2681
|
+
}
|
|
2682
|
+
|
|
2683
|
+
if (succeededInWave.length === 0) {
|
|
2684
|
+
return `❌ No succeeded tasks in wave ${targetWave}. Force merge requires at least one succeeded task whose commits can be merged.`;
|
|
2685
|
+
}
|
|
2686
|
+
|
|
2687
|
+
// If skipFailed is true, mark failed/stalled tasks as skipped
|
|
2688
|
+
const skippedTasks: string[] = [];
|
|
2689
|
+
if (skipFailed && failedInWave.length > 0) {
|
|
2690
|
+
for (const taskId of failedInWave) {
|
|
2691
|
+
const task = state.tasks.find(t => t.taskId === taskId);
|
|
2692
|
+
if (!task) continue;
|
|
2693
|
+
const prevStatus = task.status;
|
|
2694
|
+
task.status = "skipped";
|
|
2695
|
+
task.exitReason = "Skipped by orch_force_merge";
|
|
2696
|
+
task.endedAt = Date.now();
|
|
2697
|
+
skippedTasks.push(taskId);
|
|
2698
|
+
|
|
2699
|
+
// Adjust counters
|
|
2700
|
+
if (prevStatus === "failed" || prevStatus === "stalled") {
|
|
2701
|
+
state.failedTasks = Math.max(0, state.failedTasks - 1);
|
|
2702
|
+
}
|
|
2703
|
+
state.skippedTasks = (state.skippedTasks ?? 0) + 1;
|
|
2704
|
+
}
|
|
2705
|
+
|
|
2706
|
+
// Recompute blocked tasks if dependency graph is available
|
|
2707
|
+
const remainingFailures = new Set<string>();
|
|
2708
|
+
for (const t of state.tasks) {
|
|
2709
|
+
if ((t.status === "failed" || t.status === "stalled")) {
|
|
2710
|
+
remainingFailures.add(t.taskId);
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
if (orchBatchState.dependencyGraph && orchBatchState.batchId === state.batchId) {
|
|
2715
|
+
const newBlocked = computeTransitiveDependents(remainingFailures, orchBatchState.dependencyGraph);
|
|
2716
|
+
state.blockedTaskIds = [...newBlocked].sort();
|
|
2717
|
+
state.blockedTasks = newBlocked.size;
|
|
2718
|
+
} else if (remainingFailures.size === 0) {
|
|
2719
|
+
// No remaining failures — clear all blocked state
|
|
2720
|
+
state.blockedTaskIds = [];
|
|
2721
|
+
state.blockedTasks = 0;
|
|
2722
|
+
}
|
|
2723
|
+
} else if (!skipFailed && failedInWave.length > 0) {
|
|
2724
|
+
return `❌ Wave ${targetWave} has ${failedInWave.length} failed task(s): ${failedInWave.join(", ")}.\n` +
|
|
2725
|
+
`Use skipFailed=true to skip them, or use orch_skip_task to skip them individually first.`;
|
|
2726
|
+
}
|
|
2727
|
+
|
|
2728
|
+
// Clear the failed merge result so resume will re-attempt the merge.
|
|
2729
|
+
// With failed tasks now skipped, the merge should succeed (no mixed outcomes).
|
|
2730
|
+
state.mergeResults.splice(mergeResultIdx, 1);
|
|
2731
|
+
|
|
2732
|
+
// Phase transition to "paused" so orch_resume will re-run the merge phase.
|
|
2733
|
+
// "paused" is the standard resumable state (not "stopped" which needs force).
|
|
2734
|
+
state.phase = "paused";
|
|
2735
|
+
|
|
2736
|
+
// Clear merge-related errors
|
|
2737
|
+
state.errors = state.errors.filter(e => !e.includes("mixed") && !e.includes("merge") && !e.includes("Merge"));
|
|
2738
|
+
state.lastError = null;
|
|
2739
|
+
|
|
2740
|
+
// Update timestamp
|
|
2741
|
+
state.updatedAt = Date.now();
|
|
2742
|
+
|
|
2743
|
+
// Persist
|
|
2744
|
+
try {
|
|
2745
|
+
saveBatchState(JSON.stringify(state, null, 2), stateRoot);
|
|
2746
|
+
} catch (err) {
|
|
2747
|
+
return `❌ Failed to persist state after force merge: ${err instanceof Error ? err.message : String(err)}`;
|
|
2748
|
+
}
|
|
2749
|
+
|
|
2750
|
+
// Sync in-memory state if batch IDs match
|
|
2751
|
+
if (orchBatchState.batchId === state.batchId) {
|
|
2752
|
+
orchBatchState.failedTasks = state.failedTasks;
|
|
2753
|
+
orchBatchState.skippedTasks = state.skippedTasks ?? 0;
|
|
2754
|
+
orchBatchState.blockedTasks = state.blockedTasks;
|
|
2755
|
+
orchBatchState.blockedTaskIds = new Set(state.blockedTaskIds);
|
|
2756
|
+
orchBatchState.phase = "paused";
|
|
2757
|
+
}
|
|
2758
|
+
|
|
2759
|
+
updateOrchWidget();
|
|
2760
|
+
|
|
2761
|
+
const lines = [
|
|
2762
|
+
`✅ Force merge prepared for wave ${targetWave}.`,
|
|
2763
|
+
` Failed merge result cleared — resume will re-attempt the merge.`,
|
|
2764
|
+
` Succeeded tasks: ${succeededInWave.join(", ")}`,
|
|
2765
|
+
];
|
|
2766
|
+
|
|
2767
|
+
if (skippedTasks.length > 0) {
|
|
2768
|
+
lines.push(` Skipped tasks (were failed): ${skippedTasks.join(", ")}`);
|
|
2769
|
+
}
|
|
2770
|
+
|
|
2771
|
+
lines.push(` Batch phase: paused | Failed: ${state.failedTasks}, Skipped: ${state.skippedTasks ?? 0} / ${state.totalTasks} total`);
|
|
2772
|
+
|
|
2773
|
+
const resumeHint = "Use orch_resume() to re-run the merge with failed tasks skipped.";
|
|
2774
|
+
lines.push(` ${resumeHint}`);
|
|
2775
|
+
|
|
2776
|
+
return lines.join("\n");
|
|
2777
|
+
}
|
|
2778
|
+
|
|
2333
2779
|
/**
|
|
2334
2780
|
* Core logic for orch-integrate. Returns a result message string.
|
|
2335
2781
|
* On error, returns an object with error flag.
|
|
@@ -3080,6 +3526,111 @@ export default function (pi: ExtensionAPI) {
|
|
|
3080
3526
|
},
|
|
3081
3527
|
});
|
|
3082
3528
|
|
|
3529
|
+
// ── TP-077: Supervisor Recovery Tools ────────────────────────────
|
|
3530
|
+
|
|
3531
|
+
pi.registerTool({
|
|
3532
|
+
name: "orch_retry_task",
|
|
3533
|
+
label: "Retry Failed Task",
|
|
3534
|
+
description:
|
|
3535
|
+
"Retry a specific failed task. Resets the task to pending status so it will " +
|
|
3536
|
+
"be re-executed on the next resume cycle. Only works for tasks with 'failed' status.",
|
|
3537
|
+
promptSnippet: "orch_retry_task(taskId) — retry a specific failed task",
|
|
3538
|
+
promptGuidelines: [
|
|
3539
|
+
"Call orch_retry_task to reset a failed task for re-execution.",
|
|
3540
|
+
"The task must have 'failed' status — running, succeeded, or pending tasks cannot be retried.",
|
|
3541
|
+
"After retrying, use orch_resume(force=true) to re-execute the batch if it's paused.",
|
|
3542
|
+
"Use this when a task failed due to a transient issue (context pressure, API error) that may succeed on retry.",
|
|
3543
|
+
],
|
|
3544
|
+
parameters: Type.Object({
|
|
3545
|
+
taskId: Type.String({
|
|
3546
|
+
description: "Task ID to retry (e.g., 'TP-003')",
|
|
3547
|
+
}),
|
|
3548
|
+
}),
|
|
3549
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
3550
|
+
try {
|
|
3551
|
+
const result = doOrchRetryTask(params.taskId, ctx);
|
|
3552
|
+
return { content: [{ type: "text" as const, text: result }], details: undefined };
|
|
3553
|
+
} catch (err) {
|
|
3554
|
+
return {
|
|
3555
|
+
content: [{ type: "text" as const, text: `Error retrying task: ${err instanceof Error ? err.message : String(err)}` }],
|
|
3556
|
+
details: undefined,
|
|
3557
|
+
};
|
|
3558
|
+
}
|
|
3559
|
+
},
|
|
3560
|
+
});
|
|
3561
|
+
|
|
3562
|
+
pi.registerTool({
|
|
3563
|
+
name: "orch_skip_task",
|
|
3564
|
+
label: "Skip Task",
|
|
3565
|
+
description:
|
|
3566
|
+
"Skip a failed or pending task and unblock its dependents. " +
|
|
3567
|
+
"The task is marked as 'skipped' and will not be executed. " +
|
|
3568
|
+
"Dependent tasks are unblocked for execution.",
|
|
3569
|
+
promptSnippet: "orch_skip_task(taskId) — skip a task and unblock dependents",
|
|
3570
|
+
promptGuidelines: [
|
|
3571
|
+
"Call orch_skip_task to skip a task and unblock any tasks that depend on it.",
|
|
3572
|
+
"The task must have 'failed' or 'pending' status — running or succeeded tasks cannot be skipped.",
|
|
3573
|
+
"Use this when a task cannot succeed and you want to continue the batch without it.",
|
|
3574
|
+
"Skipping a task removes it from the blocker set, potentially unblocking downstream tasks.",
|
|
3575
|
+
"The engine re-evaluates dependencies on the next resume cycle.",
|
|
3576
|
+
],
|
|
3577
|
+
parameters: Type.Object({
|
|
3578
|
+
taskId: Type.String({
|
|
3579
|
+
description: "Task ID to skip (e.g., 'TP-003')",
|
|
3580
|
+
}),
|
|
3581
|
+
}),
|
|
3582
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
3583
|
+
try {
|
|
3584
|
+
const result = doOrchSkipTask(params.taskId, ctx);
|
|
3585
|
+
return { content: [{ type: "text" as const, text: result }], details: undefined };
|
|
3586
|
+
} catch (err) {
|
|
3587
|
+
return {
|
|
3588
|
+
content: [{ type: "text" as const, text: `Error skipping task: ${err instanceof Error ? err.message : String(err)}` }],
|
|
3589
|
+
details: undefined,
|
|
3590
|
+
};
|
|
3591
|
+
}
|
|
3592
|
+
},
|
|
3593
|
+
});
|
|
3594
|
+
|
|
3595
|
+
// ── TP-078: Force Merge Tool ─────────────────────────────────────
|
|
3596
|
+
|
|
3597
|
+
pi.registerTool({
|
|
3598
|
+
name: "orch_force_merge",
|
|
3599
|
+
label: "Force Merge Wave",
|
|
3600
|
+
description:
|
|
3601
|
+
"Force merge a wave that was rejected due to mixed-outcome lanes (succeeded and failed tasks " +
|
|
3602
|
+
"on the same lane). Updates the merge result to 'succeeded' so the batch can continue. " +
|
|
3603
|
+
"Optionally skips failed tasks in the wave.",
|
|
3604
|
+
promptSnippet: "orch_force_merge(waveIndex?, skipFailed?) — force merge a wave with mixed results",
|
|
3605
|
+
promptGuidelines: [
|
|
3606
|
+
"Call orch_force_merge when a wave merge was rejected because lanes had both succeeded and failed tasks.",
|
|
3607
|
+
"The batch must be paused, stopped, or failed with a 'partial' merge result for the target wave.",
|
|
3608
|
+
"Set skipFailed=true to automatically skip all failed tasks in the wave (recommended).",
|
|
3609
|
+
"If skipFailed is false and failed tasks exist, you must skip them individually with orch_skip_task first.",
|
|
3610
|
+
"After force merging, use orch_resume(force=true) to continue the batch.",
|
|
3611
|
+
"waveIndex is 0-based. Omit it to target the current wave.",
|
|
3612
|
+
],
|
|
3613
|
+
parameters: Type.Object({
|
|
3614
|
+
waveIndex: Type.Optional(Type.Number({
|
|
3615
|
+
description: "0-based wave index to force merge. Defaults to the current wave.",
|
|
3616
|
+
})),
|
|
3617
|
+
skipFailed: Type.Optional(Type.Boolean({
|
|
3618
|
+
description: "If true, automatically skip all failed tasks in the wave before merging. Defaults to false.",
|
|
3619
|
+
})),
|
|
3620
|
+
}),
|
|
3621
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
3622
|
+
try {
|
|
3623
|
+
const result = doOrchForceMerge(params.waveIndex, params.skipFailed ?? false, ctx);
|
|
3624
|
+
return { content: [{ type: "text" as const, text: result }], details: undefined };
|
|
3625
|
+
} catch (err) {
|
|
3626
|
+
return {
|
|
3627
|
+
content: [{ type: "text" as const, text: `Error force merging: ${err instanceof Error ? err.message : String(err)}` }],
|
|
3628
|
+
details: undefined,
|
|
3629
|
+
};
|
|
3630
|
+
}
|
|
3631
|
+
},
|
|
3632
|
+
});
|
|
3633
|
+
|
|
3083
3634
|
// ── Settings TUI ─────────────────────────────────────────────────
|
|
3084
3635
|
|
|
3085
3636
|
pi.registerCommand("taskplane-settings", {
|
|
@@ -1202,6 +1202,7 @@ export async function mergeWave(
|
|
|
1202
1202
|
testingCommands?: Record<string, string>,
|
|
1203
1203
|
repoId?: string,
|
|
1204
1204
|
healthMonitor?: MergeHealthMonitor | null,
|
|
1205
|
+
forceMixedOutcome?: boolean,
|
|
1205
1206
|
): Promise<MergeWaveResult> {
|
|
1206
1207
|
const startTime = Date.now();
|
|
1207
1208
|
const tmuxPrefix = config.orchestrator.tmux_prefix;
|
|
@@ -1221,6 +1222,10 @@ export async function mergeWave(
|
|
|
1221
1222
|
//
|
|
1222
1223
|
// This allows succeeded+skipped lanes (e.g., stop-wave skip of remaining tasks)
|
|
1223
1224
|
// to merge their committed work, while excluding mixed succeeded+failed lanes.
|
|
1225
|
+
//
|
|
1226
|
+
// TP-078: When forceMixedOutcome is true, lanes with both succeeded and
|
|
1227
|
+
// failed/stalled tasks are also considered mergeable. This allows the
|
|
1228
|
+
// orch_force_merge tool to merge succeeded commits from mixed-outcome lanes.
|
|
1224
1229
|
const mergeableLanes = completedLanes.filter(lane => {
|
|
1225
1230
|
const outcome = laneOutcomeByNumber.get(lane.laneNumber);
|
|
1226
1231
|
if (!outcome) return false;
|
|
@@ -1230,6 +1235,11 @@ export async function mergeWave(
|
|
|
1230
1235
|
t => t.status === "failed" || t.status === "stalled",
|
|
1231
1236
|
);
|
|
1232
1237
|
|
|
1238
|
+
if (forceMixedOutcome) {
|
|
1239
|
+
// In force mode, merge any lane with at least one succeeded task
|
|
1240
|
+
return hasSucceeded;
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1233
1243
|
return hasSucceeded && !hasHardFailure;
|
|
1234
1244
|
});
|
|
1235
1245
|
|
|
@@ -2127,6 +2137,7 @@ export async function mergeWaveByRepo(
|
|
|
2127
2137
|
agentRoot?: string,
|
|
2128
2138
|
testingCommands?: Record<string, string>,
|
|
2129
2139
|
healthMonitor?: MergeHealthMonitor | null,
|
|
2140
|
+
forceMixedOutcome?: boolean,
|
|
2130
2141
|
): Promise<MergeWaveResult> {
|
|
2131
2142
|
const startTime = Date.now();
|
|
2132
2143
|
|
|
@@ -2137,6 +2148,7 @@ export async function mergeWaveByRepo(
|
|
|
2137
2148
|
}
|
|
2138
2149
|
|
|
2139
2150
|
// Filter to mergeable lanes (same criteria as mergeWave).
|
|
2151
|
+
// TP-078: When forceMixedOutcome is true, lanes with mixed outcomes are also included.
|
|
2140
2152
|
const mergeableLanes = completedLanes.filter(lane => {
|
|
2141
2153
|
const outcome = laneOutcomeByNumber.get(lane.laneNumber);
|
|
2142
2154
|
if (!outcome) return false;
|
|
@@ -2144,6 +2156,7 @@ export async function mergeWaveByRepo(
|
|
|
2144
2156
|
const hasHardFailure = outcome.tasks.some(
|
|
2145
2157
|
t => t.status === "failed" || t.status === "stalled",
|
|
2146
2158
|
);
|
|
2159
|
+
if (forceMixedOutcome) return hasSucceeded;
|
|
2147
2160
|
return hasSucceeded && !hasHardFailure;
|
|
2148
2161
|
});
|
|
2149
2162
|
|
|
@@ -2184,6 +2197,7 @@ export async function mergeWaveByRepo(
|
|
|
2184
2197
|
testingCommands,
|
|
2185
2198
|
undefined, // repoId
|
|
2186
2199
|
healthMonitor,
|
|
2200
|
+
forceMixedOutcome,
|
|
2187
2201
|
);
|
|
2188
2202
|
// Attach empty repoResults for consistent shape
|
|
2189
2203
|
return { ...result, repoResults: [] };
|
|
@@ -2241,6 +2255,7 @@ export async function mergeWaveByRepo(
|
|
|
2241
2255
|
testingCommands,
|
|
2242
2256
|
group.repoId,
|
|
2243
2257
|
healthMonitor,
|
|
2258
|
+
forceMixedOutcome,
|
|
2244
2259
|
);
|
|
2245
2260
|
|
|
2246
2261
|
// Accumulate lane results
|
|
@@ -1377,6 +1377,9 @@ export async function resumeOrchBatch(
|
|
|
1377
1377
|
// For waves where some tasks are already done, we filter them out.
|
|
1378
1378
|
|
|
1379
1379
|
let preserveWorktreesForResume = false;
|
|
1380
|
+
const persistedStatusByTaskId = new Map(
|
|
1381
|
+
persistedState.tasks.map((task) => [task.taskId, task.status] as const),
|
|
1382
|
+
);
|
|
1380
1383
|
|
|
1381
1384
|
for (let waveIdx = resumePoint.resumeWaveIndex; waveIdx < persistedState.wavePlan.length; waveIdx++) {
|
|
1382
1385
|
// Check pause signal
|
|
@@ -1390,10 +1393,12 @@ export async function resumeOrchBatch(
|
|
|
1390
1393
|
batchState.currentWaveIndex = waveIdx;
|
|
1391
1394
|
persistRuntimeState("wave-index-change", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discovery, stateRoot);
|
|
1392
1395
|
|
|
1393
|
-
// Get wave tasks, filtering out completed/failed/blocked ones.
|
|
1396
|
+
// Get wave tasks, filtering out completed/failed/skipped/blocked ones.
|
|
1397
|
+
// Persisted "skipped" tasks are terminal and must never be re-executed.
|
|
1394
1398
|
let waveTasks = persistedState.wavePlan[waveIdx].filter(
|
|
1395
1399
|
taskId => !completedTaskSet.has(taskId) &&
|
|
1396
1400
|
!failedTaskSet.has(taskId) &&
|
|
1401
|
+
persistedStatusByTaskId.get(taskId) !== "skipped" &&
|
|
1397
1402
|
!batchState.blockedTaskIds.has(taskId),
|
|
1398
1403
|
);
|
|
1399
1404
|
|
|
@@ -1425,26 +1430,71 @@ export async function resumeOrchBatch(
|
|
|
1425
1430
|
);
|
|
1426
1431
|
const mergeRetryLanes = reconstructAllocatedLanes(waveLaneRecords, persistedState.tasks);
|
|
1427
1432
|
|
|
1428
|
-
// Build synthetic WaveExecutionResult
|
|
1433
|
+
// Build synthetic WaveExecutionResult from persisted terminal task states.
|
|
1434
|
+
// Crucial for orch_force_merge: tasks intentionally marked "skipped" must
|
|
1435
|
+
// remain skipped here (not failed), otherwise mixed-outcome detection would
|
|
1436
|
+
// trigger again and block the forced merge recovery path.
|
|
1429
1437
|
const succeededTaskIds = persistedState.wavePlan[waveIdx].filter(
|
|
1430
1438
|
taskId => completedTaskSet.has(taskId),
|
|
1431
1439
|
);
|
|
1432
|
-
const
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
status
|
|
1440
|
+
const skippedTaskIds = persistedState.wavePlan[waveIdx].filter(
|
|
1441
|
+
taskId => persistedStatusByTaskId.get(taskId) === "skipped",
|
|
1442
|
+
);
|
|
1443
|
+
const failedTaskIds = persistedState.wavePlan[waveIdx].filter(
|
|
1444
|
+
taskId => {
|
|
1445
|
+
const status = persistedStatusByTaskId.get(taskId);
|
|
1446
|
+
return status === "failed" || status === "stalled";
|
|
1447
|
+
},
|
|
1448
|
+
);
|
|
1449
|
+
|
|
1450
|
+
const syntheticLaneResults: LaneExecutionResult[] = mergeRetryLanes.map((lane) => {
|
|
1451
|
+
const laneTasks = lane.tasks.map((t) => {
|
|
1452
|
+
const persistedStatus = persistedStatusByTaskId.get(t.taskId);
|
|
1453
|
+
let status: LaneTaskStatus;
|
|
1454
|
+
if (completedTaskSet.has(t.taskId) || persistedStatus === "succeeded") {
|
|
1455
|
+
status = "succeeded";
|
|
1456
|
+
} else if (persistedStatus === "skipped") {
|
|
1457
|
+
status = "skipped";
|
|
1458
|
+
} else if (persistedStatus === "failed") {
|
|
1459
|
+
status = "failed";
|
|
1460
|
+
} else if (persistedStatus === "stalled") {
|
|
1461
|
+
status = "stalled";
|
|
1462
|
+
} else {
|
|
1463
|
+
status = "failed";
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
return {
|
|
1467
|
+
taskId: t.taskId,
|
|
1468
|
+
status,
|
|
1469
|
+
startTime: Date.now(),
|
|
1470
|
+
endTime: Date.now(),
|
|
1471
|
+
exitReason:
|
|
1472
|
+
status === "succeeded" ? "Task completed (merge retry)"
|
|
1473
|
+
: status === "skipped" ? "Task skipped (merge retry)"
|
|
1474
|
+
: status === "stalled" ? "Task stalled (merge retry)"
|
|
1475
|
+
: "Task failed (merge retry)",
|
|
1476
|
+
sessionName: lane.tmuxSessionName,
|
|
1477
|
+
doneFileFound: status === "succeeded",
|
|
1478
|
+
};
|
|
1479
|
+
});
|
|
1480
|
+
|
|
1481
|
+
const laneHasHardFailure = laneTasks.some(
|
|
1482
|
+
(t) => t.status === "failed" || t.status === "stalled",
|
|
1483
|
+
);
|
|
1484
|
+
const laneHasSucceeded = laneTasks.some((t) => t.status === "succeeded");
|
|
1485
|
+
const overallStatus = laneHasHardFailure
|
|
1486
|
+
? (laneHasSucceeded ? "partial" : "failed")
|
|
1487
|
+
: "succeeded";
|
|
1488
|
+
|
|
1489
|
+
return {
|
|
1490
|
+
laneNumber: lane.laneNumber,
|
|
1491
|
+
laneId: lane.laneId,
|
|
1492
|
+
tasks: laneTasks,
|
|
1493
|
+
overallStatus,
|
|
1438
1494
|
startTime: Date.now(),
|
|
1439
1495
|
endTime: Date.now(),
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
doneFileFound: completedTaskSet.has(t.taskId),
|
|
1443
|
-
})),
|
|
1444
|
-
overallStatus: lane.tasks.every(t => completedTaskSet.has(t.taskId)) ? "succeeded" as const : "partial" as const,
|
|
1445
|
-
startTime: Date.now(),
|
|
1446
|
-
endTime: Date.now(),
|
|
1447
|
-
}));
|
|
1496
|
+
};
|
|
1497
|
+
});
|
|
1448
1498
|
|
|
1449
1499
|
const syntheticWaveResult: WaveExecutionResult = {
|
|
1450
1500
|
waveIndex: waveIdx + 1,
|
|
@@ -1453,8 +1503,8 @@ export async function resumeOrchBatch(
|
|
|
1453
1503
|
laneResults: syntheticLaneResults,
|
|
1454
1504
|
policyApplied: orchConfig.failure.on_task_failure,
|
|
1455
1505
|
stoppedEarly: false,
|
|
1456
|
-
failedTaskIds
|
|
1457
|
-
skippedTaskIds
|
|
1506
|
+
failedTaskIds,
|
|
1507
|
+
skippedTaskIds,
|
|
1458
1508
|
succeededTaskIds,
|
|
1459
1509
|
blockedTaskIds: [],
|
|
1460
1510
|
laneCount: mergeRetryLanes.length,
|
|
@@ -1713,6 +1763,8 @@ export async function resumeOrchBatch(
|
|
|
1713
1763
|
`Lane(s) ${mixedIds} contain both succeeded and failed tasks. ` +
|
|
1714
1764
|
`Automatic partial-branch merge is disabled to avoid dropping succeeded commits.`;
|
|
1715
1765
|
mergeResult = { ...mergeResult, status: "partial", failedLane: mixedOutcomeLanes[0].laneNumber, failureReason };
|
|
1766
|
+
// Update the already-pushed reference so persisted state reflects "partial"
|
|
1767
|
+
batchState.mergeResults[batchState.mergeResults.length - 1] = mergeResult;
|
|
1716
1768
|
}
|
|
1717
1769
|
|
|
1718
1770
|
// TP-032 R006-3: Exclude verification_new_failure lanes from success count
|
|
@@ -1752,6 +1804,9 @@ export async function resumeOrchBatch(
|
|
|
1752
1804
|
`Automatic partial-branch merge is disabled to avoid dropping succeeded commits.`,
|
|
1753
1805
|
totalDurationMs: 0,
|
|
1754
1806
|
};
|
|
1807
|
+
// Keep mergeResults in sync even when no mergeable lane exists.
|
|
1808
|
+
// Downstream retry/update paths assume the current wave has an entry.
|
|
1809
|
+
batchState.mergeResults.push(mergeResult);
|
|
1755
1810
|
onNotify(
|
|
1756
1811
|
ORCH_MESSAGES.orchMergeFailed(waveIdx + 1, mergeResult.failedLane, mergeResult.failureReason || "unknown"),
|
|
1757
1812
|
"error",
|
|
@@ -739,8 +739,8 @@ When you receive an alert, follow this sequence:
|
|
|
739
739
|
### Autonomy Rules for Alert Response
|
|
740
740
|
|
|
741
741
|
- **Do NOT ask the operator for permission** on routine recovery actions:
|
|
742
|
-
- Retrying a failed task (`orch_resume(force=true)`)
|
|
743
|
-
- Skipping
|
|
742
|
+
- Retrying a failed task (`orch_retry_task(taskId)` then `orch_resume(force=true)`)
|
|
743
|
+
- Skipping a failed task and its dependents (`orch_skip_task(taskId)` then `orch_resume(force=true)`)
|
|
744
744
|
- Reading logs and batch state for diagnosis
|
|
745
745
|
|
|
746
746
|
- **DO escalate to the operator** for genuinely ambiguous situations:
|
|
@@ -758,6 +758,17 @@ You have these orchestrator tools available:
|
|
|
758
758
|
- `orch_abort(hard?)` — Abort the batch
|
|
759
759
|
- `orch_integrate(mode?, force?)` — Integrate completed batch
|
|
760
760
|
- `orch_start(target)` — Start a new batch
|
|
761
|
+
- `orch_retry_task(taskId)` — Reset a failed/stalled task to pending for re-execution
|
|
762
|
+
- `orch_skip_task(taskId)` — Skip a task and unblock its dependents
|
|
763
|
+
- `orch_force_merge(waveIndex?, skipFailed?)` — Force merge a wave with mixed results (skips failed tasks if skipFailed=true)
|
|
764
|
+
|
|
765
|
+
**Recovery workflow:**
|
|
766
|
+
1. Diagnose with `orch_status()` and reading logs
|
|
767
|
+
2. Decide: retry (`orch_retry_task`), skip (`orch_skip_task`), or force merge (`orch_force_merge`)
|
|
768
|
+
3. Resume: `orch_resume(force=true)` to continue the batch
|
|
769
|
+
|
|
770
|
+
**Note:** `orch_retry_task`, `orch_skip_task`, and `orch_force_merge` require the batch to be paused/stopped first.
|
|
771
|
+
If the batch is actively running, call `orch_pause()` first.
|
|
761
772
|
|
|
762
773
|
Plus general tools: `read`, `write`, `edit`, `bash`, `grep`, `find`, `ls`
|
|
763
774
|
for inspecting files, running git commands, and editing batch state.
|
|
@@ -771,6 +782,220 @@ typically requires `orch_resume(force=true)` after checking batch state.
|
|
|
771
782
|
|
|
772
783
|
---
|
|
773
784
|
|
|
785
|
+
## 13b. Recovery Playbooks (TP-078)
|
|
786
|
+
|
|
787
|
+
When you receive an alert, follow the playbook for that alert category.
|
|
788
|
+
Each playbook is a **decision tree** — follow the branches based on what
|
|
789
|
+
you observe. Do not skip steps; each observation narrows the diagnosis.
|
|
790
|
+
|
|
791
|
+
### Playbook A: Task Failure
|
|
792
|
+
|
|
793
|
+
**Trigger:** `task-failure` alert — a task failed after the engine exhausted
|
|
794
|
+
deterministic recovery (retries, context resets).
|
|
795
|
+
|
|
796
|
+
```
|
|
797
|
+
TASK FAILED: {taskId}
|
|
798
|
+
│
|
|
799
|
+
├─ 1. Read STATUS.md from the task's worktree
|
|
800
|
+
│ Path: .worktrees/{opId}-{batchId}/lane-{N}/{taskFolder}/STATUS.md
|
|
801
|
+
│
|
|
802
|
+
├─ 2. Check: Did the worker complete all steps?
|
|
803
|
+
│ Look at STATUS.md "Current Step" and checkbox completion
|
|
804
|
+
│ │
|
|
805
|
+
│ ├─ YES (all steps checked, .DONE missing — race condition)
|
|
806
|
+
│ │ → orch_retry_task(taskId)
|
|
807
|
+
│ │ → orch_resume(force=true)
|
|
808
|
+
│ │ → Report: "Task {taskId} appears to have completed but .DONE was
|
|
809
|
+
│ │ not created (likely race condition). Retrying."
|
|
810
|
+
│ │
|
|
811
|
+
│ └─ NO (incomplete steps — genuine failure)
|
|
812
|
+
│ │
|
|
813
|
+
│ ├─ 3. Check exit reason in batch state or STATUS.md
|
|
814
|
+
│ │ Read `.pi/batch-state.json` → tasks[].exitReason
|
|
815
|
+
│ │ │
|
|
816
|
+
│ │ ├─ Context pressure / API error / timeout
|
|
817
|
+
│ │ │ → Transient failure. orch_retry_task(taskId)
|
|
818
|
+
│ │ │ → orch_resume(force=true)
|
|
819
|
+
│ │ │ → Report: "Task {taskId} failed due to {reason}. Retrying."
|
|
820
|
+
│ │ │
|
|
821
|
+
│ │ ├─ Test failure / compile error / logic error
|
|
822
|
+
│ │ │ │
|
|
823
|
+
│ │ │ ├─ 4. Is this the first failure of this task?
|
|
824
|
+
│ │ │ │ Check: has it been retried before?
|
|
825
|
+
│ │ │ │ (Look for exitDiagnostic or retry count in state)
|
|
826
|
+
│ │ │ │ │
|
|
827
|
+
│ │ │ │ ├─ FIRST FAILURE
|
|
828
|
+
│ │ │ │ │ → orch_retry_task(taskId)
|
|
829
|
+
│ │ │ │ │ → orch_resume(force=true)
|
|
830
|
+
│ │ │ │ │ → Report: "Retrying {taskId} — first failure,
|
|
831
|
+
│ │ │ │ │ may succeed with fresh context."
|
|
832
|
+
│ │ │ │ │
|
|
833
|
+
│ │ │ │ ├─ SECOND FAILURE (same error pattern)
|
|
834
|
+
│ │ │ │ │ → orch_retry_task(taskId)
|
|
835
|
+
│ │ │ │ │ → orch_resume(force=true)
|
|
836
|
+
│ │ │ │ │ → Report: "Retrying {taskId} — second attempt.
|
|
837
|
+
│ │ │ │ │ Will escalate if it fails again."
|
|
838
|
+
│ │ │ │ │
|
|
839
|
+
│ │ │ │ └─ THIRD+ FAILURE
|
|
840
|
+
│ │ │ │ → ESCALATE to operator
|
|
841
|
+
│ │ │ │ → Report: "Task {taskId} has failed {N} times.
|
|
842
|
+
│ │ │ │ Error: {exitReason}. Recommend skipping or
|
|
843
|
+
│ │ │ │ manual intervention."
|
|
844
|
+
│ │ │ │ → If autonomous mode: orch_skip_task(taskId)
|
|
845
|
+
│ │ │ │ then orch_resume(force=true)
|
|
846
|
+
│ │ │ │
|
|
847
|
+
│ │ │ └─ (unknown error type)
|
|
848
|
+
│ │ │ → ESCALATE to operator
|
|
849
|
+
│ │ │ → Report: "Task {taskId} failed with unexpected error.
|
|
850
|
+
│ │ │ Recommend investigation before retrying."
|
|
851
|
+
│ │ │
|
|
852
|
+
│ │ └─ No exit reason recorded
|
|
853
|
+
│ │ → orch_retry_task(taskId)
|
|
854
|
+
│ │ → orch_resume(force=true)
|
|
855
|
+
│ │ → Report: "Task {taskId} failed without exit reason
|
|
856
|
+
│ │ (session may have died). Retrying."
|
|
857
|
+
│ │
|
|
858
|
+
│ └─ (STATUS.md not accessible — worktree cleaned up)
|
|
859
|
+
│ → orch_retry_task(taskId)
|
|
860
|
+
│ → orch_resume(force=true)
|
|
861
|
+
│ → Report: "Task {taskId} failed, worktree unavailable.
|
|
862
|
+
│ Retrying with fresh worktree."
|
|
863
|
+
```
|
|
864
|
+
|
|
865
|
+
### Playbook B: Merge Failure
|
|
866
|
+
|
|
867
|
+
**Trigger:** `merge-failure` alert — wave merge failed and the batch paused.
|
|
868
|
+
Common cause: mixed-outcome lanes (succeeded + failed tasks on the same lane).
|
|
869
|
+
|
|
870
|
+
```
|
|
871
|
+
MERGE FAILED: wave {waveIndex}
|
|
872
|
+
│
|
|
873
|
+
├─ 1. Check merge result in batch state
|
|
874
|
+
│ Read `.pi/batch-state.json` → mergeResults[]
|
|
875
|
+
│ Find the entry for the failed wave
|
|
876
|
+
│ │
|
|
877
|
+
│ ├─ Status: "partial" (mixed-outcome lanes)
|
|
878
|
+
│ │ │
|
|
879
|
+
│ │ ├─ 2. Identify failed tasks in the wave
|
|
880
|
+
│ │ │ Read wavePlan[waveIndex] → task IDs
|
|
881
|
+
│ │ │ Check each task's status in tasks[]
|
|
882
|
+
│ │ │
|
|
883
|
+
│ │ ├─ 3. For each failed task, decide: retry or skip?
|
|
884
|
+
│ │ │ │
|
|
885
|
+
│ │ │ ├─ Task has partial progress (commits ahead of base)
|
|
886
|
+
│ │ │ │ → May be worth retrying
|
|
887
|
+
│ │ │ │ → orch_retry_task(taskId) for each
|
|
888
|
+
│ │ │ │ → orch_resume(force=true)
|
|
889
|
+
│ │ │ │
|
|
890
|
+
│ │ │ └─ Task genuinely cannot succeed / already retried
|
|
891
|
+
│ │ │ → Skip it and force merge
|
|
892
|
+
│ │ │ → orch_force_merge(waveIndex, skipFailed=true)
|
|
893
|
+
│ │ │ → orch_resume(force=true)
|
|
894
|
+
│ │ │ → Report: "Force merged wave {N}. Skipped tasks:
|
|
895
|
+
│ │ │ {list}. Succeeded tasks merged: {list}."
|
|
896
|
+
│ │ │
|
|
897
|
+
│ │ └─ 4. SHORTCUT (when diagnosis is clear)
|
|
898
|
+
│ │ If all failed tasks are genuinely failed (not race conditions):
|
|
899
|
+
│ │ → orch_force_merge(waveIndex, skipFailed=true)
|
|
900
|
+
│ │ → orch_resume(force=true)
|
|
901
|
+
│ │ This is the most common recovery path.
|
|
902
|
+
│ │
|
|
903
|
+
│ ├─ Status: "failed" (merge agent failure)
|
|
904
|
+
│ │ │
|
|
905
|
+
│ │ ├─ 2. Check merge result JSON files
|
|
906
|
+
│ │ │ ls .pi/merge-result-w{N}-lane{K}-*.json
|
|
907
|
+
│ │ │ │
|
|
908
|
+
│ │ │ ├─ Result file shows CONFLICT_UNRESOLVED
|
|
909
|
+
│ │ │ │ → ESCALATE to operator
|
|
910
|
+
│ │ │ │ → Report: "Merge conflicts in wave {N} that the merge
|
|
911
|
+
│ │ │ │ agent couldn't resolve. Manual resolution needed."
|
|
912
|
+
│ │ │ │ → Provide conflict file list
|
|
913
|
+
│ │ │ │
|
|
914
|
+
│ │ │ ├─ Result file shows BUILD_FAILURE
|
|
915
|
+
│ │ │ │ → Tests failed after merge. May indicate conflicting changes.
|
|
916
|
+
│ │ │ │ → ESCALATE to operator
|
|
917
|
+
│ │ │ │ → Report: "Tests failed after merging wave {N}.
|
|
918
|
+
│ │ │ │ Changes may be incompatible."
|
|
919
|
+
│ │ │ │
|
|
920
|
+
│ │ │ ├─ No result file (merge agent timed out/died)
|
|
921
|
+
│ │ │ │ → Check if lane branch was actually merged:
|
|
922
|
+
│ │ │ │ git log orch/{orchBranch}..task/{laneBranch}
|
|
923
|
+
│ │ │ │ → If empty (merged): update batch state manually
|
|
924
|
+
│ │ │ │ → If not merged: orch_resume(force=true) to retry
|
|
925
|
+
│ │ │ │
|
|
926
|
+
│ │ │ └─ Result file shows SUCCESS
|
|
927
|
+
│ │ │ → Merge succeeded but engine didn't pick it up
|
|
928
|
+
│ │ │ → Update mergeResults in batch state to "succeeded"
|
|
929
|
+
│ │ │ → orch_resume(force=true)
|
|
930
|
+
│ │ │
|
|
931
|
+
│ │ └─ 3. If all else fails
|
|
932
|
+
│ │ → ESCALATE to operator with full diagnostic
|
|
933
|
+
│ │
|
|
934
|
+
│ └─ (No merge result entry)
|
|
935
|
+
│ → Wave tasks completed but merge was never attempted
|
|
936
|
+
│ → orch_resume(force=true) to trigger merge
|
|
937
|
+
│ → Report: "Merge for wave {N} was not attempted. Resuming."
|
|
938
|
+
```
|
|
939
|
+
|
|
940
|
+
### Playbook C: Batch Complete
|
|
941
|
+
|
|
942
|
+
**Trigger:** `batch-complete` alert — all waves finished (with or without failures).
|
|
943
|
+
|
|
944
|
+
```
|
|
945
|
+
BATCH COMPLETE: {batchId}
|
|
946
|
+
│
|
|
947
|
+
├─ 1. Read batch state summary
|
|
948
|
+
│ Check: succeededTasks, failedTasks, skippedTasks, totalTasks
|
|
949
|
+
│ │
|
|
950
|
+
│ ├─ ALL SUCCEEDED (failedTasks=0, skippedTasks=0)
|
|
951
|
+
│ │ → Report: "✅ Batch complete! All {N} tasks succeeded across
|
|
952
|
+
│ │ {W} waves. Ready to integrate."
|
|
953
|
+
│ │ → Suggest: orch_integrate() to bring changes to working branch
|
|
954
|
+
│ │
|
|
955
|
+
│ ├─ SOME FAILED (failedTasks > 0)
|
|
956
|
+
│ │ │
|
|
957
|
+
│ │ ├─ 2. List failed tasks with reasons
|
|
958
|
+
│ │ │ For each failed task:
|
|
959
|
+
│ │ │ - Task ID and title (from PROMPT.md header)
|
|
960
|
+
│ │ │ - Exit reason (from batch state)
|
|
961
|
+
│ │ │ - Wave and lane info
|
|
962
|
+
│ │ │
|
|
963
|
+
│ │ ├─ 3. Report with context
|
|
964
|
+
│ │ │ → "⚠️ Batch complete with {F} failure(s) out of {N} tasks.
|
|
965
|
+
│ │ │ Succeeded: {S}, Skipped: {K}, Failed: {F}
|
|
966
|
+
│ │ │ Failed tasks: {list with reasons}
|
|
967
|
+
│ │ │ The succeeded work is ready to integrate."
|
|
968
|
+
│ │ │
|
|
969
|
+
│ │ └─ 4. Suggest next steps
|
|
970
|
+
│ │ → "You can integrate the succeeded work now with orch_integrate()
|
|
971
|
+
│ │ and handle the failed tasks separately."
|
|
972
|
+
│ │ → If tasks have partial progress: "Some failed tasks have
|
|
973
|
+
│ │ partial commits that could be preserved."
|
|
974
|
+
│ │
|
|
975
|
+
│ └─ SOME SKIPPED (skippedTasks > 0, failedTasks = 0)
|
|
976
|
+
│ → Report: "✅ Batch complete. {S} succeeded, {K} skipped.
|
|
977
|
+
│ Skipped tasks: {list}. Ready to integrate."
|
|
978
|
+
│ → Suggest: orch_integrate()
|
|
979
|
+
```
|
|
980
|
+
|
|
981
|
+
### Quick Reference: Recovery Action Matrix
|
|
982
|
+
|
|
983
|
+
| Alert | Diagnosis | Action | Autonomy |
|
|
984
|
+
|-------|-----------|--------|----------|
|
|
985
|
+
| task-failure | Race condition (.DONE missing) | `orch_retry_task` → `orch_resume` | Automatic |
|
|
986
|
+
| task-failure | Transient error (API, context) | `orch_retry_task` → `orch_resume` | Automatic |
|
|
987
|
+
| task-failure | Genuine error, 1st-2nd attempt | `orch_retry_task` → `orch_resume` | Automatic |
|
|
988
|
+
| task-failure | Genuine error, 3rd+ attempt | Escalate (or `orch_skip_task` in autonomous) | Supervised: escalate |
|
|
989
|
+
| task-failure | Unknown error | Escalate | Always escalate |
|
|
990
|
+
| merge-failure | Mixed-outcome lanes | `orch_force_merge(skipFailed=true)` → `orch_resume` | Automatic |
|
|
991
|
+
| merge-failure | Unresolved conflicts | Escalate | Always escalate |
|
|
992
|
+
| merge-failure | Build failure after merge | Escalate | Always escalate |
|
|
993
|
+
| merge-failure | Agent timeout, no result | `orch_resume(force=true)` to retry | Automatic |
|
|
994
|
+
| batch-complete | All succeeded | Report → suggest `orch_integrate` | Report only |
|
|
995
|
+
| batch-complete | Some failed | Report with failure details | Report only |
|
|
996
|
+
|
|
997
|
+
---
|
|
998
|
+
|
|
774
999
|
## 14. Your Startup Checklist
|
|
775
1000
|
|
|
776
1001
|
When you activate at the start of a batch:
|