taskplane 0.21.10 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/taskplane/engine-worker.ts +9 -0
- package/extensions/taskplane/engine.ts +171 -2
- package/extensions/taskplane/extension.ts +60 -46
- package/extensions/taskplane/resume.ts +165 -1
- package/extensions/taskplane/supervisor-primer.md +68 -0
- package/extensions/taskplane/types.ts +114 -0
- package/package.json +1 -1
|
@@ -21,6 +21,7 @@ import type {
|
|
|
21
21
|
OrchBatchPhase,
|
|
22
22
|
OrchBatchRuntimeState,
|
|
23
23
|
OrchestratorConfig,
|
|
24
|
+
SupervisorAlert,
|
|
24
25
|
TaskRunnerConfig,
|
|
25
26
|
WorkspaceConfig,
|
|
26
27
|
WorkspaceRepoConfig,
|
|
@@ -35,6 +36,7 @@ export type WorkerToMainMessage =
|
|
|
35
36
|
| { type: "notify"; msg: string; level: "info" | "warning" | "error" }
|
|
36
37
|
| { type: "monitor-update"; state: MonitorState }
|
|
37
38
|
| { type: "engine-event"; event: EngineEvent }
|
|
39
|
+
| { type: "supervisor-alert"; alert: SupervisorAlert }
|
|
38
40
|
| { type: "state-sync"; state: SerializedBatchState }
|
|
39
41
|
| { type: "complete"; state: SerializedBatchState }
|
|
40
42
|
| { type: "error"; message: string };
|
|
@@ -246,6 +248,11 @@ if (process.env.TASKPLANE_ENGINE_FORK === "1" && typeof process.send === "functi
|
|
|
246
248
|
send({ type: "engine-event", event });
|
|
247
249
|
};
|
|
248
250
|
|
|
251
|
+
// TP-076: Supervisor alert callback — sends structured alerts to main thread
|
|
252
|
+
const onSupervisorAlert = (alert: import("./types.ts").SupervisorAlert) => {
|
|
253
|
+
send({ type: "supervisor-alert", alert });
|
|
254
|
+
};
|
|
255
|
+
|
|
249
256
|
// ── Execute engine ───────────────────────────────────────────
|
|
250
257
|
const enginePromise = data.mode === "resume"
|
|
251
258
|
? resumeOrchBatch(
|
|
@@ -259,6 +266,7 @@ if (process.env.TASKPLANE_ENGINE_FORK === "1" && typeof process.send === "functi
|
|
|
259
266
|
data.workspaceRoot,
|
|
260
267
|
data.agentRoot,
|
|
261
268
|
data.force ?? false,
|
|
269
|
+
onSupervisorAlert,
|
|
262
270
|
)
|
|
263
271
|
: executeOrchBatch(
|
|
264
272
|
data.args ?? "",
|
|
@@ -272,6 +280,7 @@ if (process.env.TASKPLANE_ENGINE_FORK === "1" && typeof process.send === "functi
|
|
|
272
280
|
data.workspaceRoot,
|
|
273
281
|
data.agentRoot,
|
|
274
282
|
onEngineEvent,
|
|
283
|
+
onSupervisorAlert,
|
|
275
284
|
);
|
|
276
285
|
|
|
277
286
|
enginePromise
|
|
@@ -18,8 +18,8 @@ import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-rep
|
|
|
18
18
|
import { resolveOperatorId } from "./naming.ts";
|
|
19
19
|
import { applyPartialProgressToOutcomes, buildTier0EventBase, deleteBatchState, emitEngineEvent, emitTier0Event, loadBatchHistory, loadBatchState, persistRuntimeState, saveBatchHistory, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
20
20
|
import { listOrchSessions } from "./sessions.ts";
|
|
21
|
-
import { buildEngineEventBase, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
|
|
22
|
-
import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, TaskRunnerConfig, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
21
|
+
import { buildBatchProgressSnapshot, buildEngineEventBase, defaultResilienceState, FATAL_DISCOVERY_CODES, generateBatchId, TIER0_RETRYABLE_CLASSIFICATIONS, TIER0_RETRY_BUDGETS, tier0ScopeKey, tier0WaveScopeKey } from "./types.ts";
|
|
22
|
+
import type { AllocatedLane, AllocatedTask, BatchHistorySummary, BatchTaskSummary, BatchWaveSummary, DiscoveryResult, EngineEventCallback, EscalationContext, LaneExecutionResult, LaneTaskOutcome, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, SupervisorAlert, SupervisorAlertCallback, TaskRunnerConfig, Tier0EscalationPattern, Tier0RecoveryPattern, TokenCounts, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
23
23
|
import { buildDependencyGraph, computeWaves, resolveBaseBranch, resolveRepoRoot, validateGraph } from "./waves.ts";
|
|
24
24
|
import { deleteBranchBestEffort, forceCleanupWorktree, formatPreflightResults, listWorktrees, preserveFailedLaneProgress, removeAllWorktrees, removeWorktree, runPreflight, safeResetWorktree, sleepSync } from "./worktree.ts";
|
|
25
25
|
import { runPreflightCleanup, formatPreflightCleanup } from "./cleanup.ts";
|
|
@@ -744,6 +744,7 @@ async function attemptStaleWorktreeRecovery(
|
|
|
744
744
|
* @param workspaceRoot - Workspace root for resolving task area paths (defaults to cwd)
|
|
745
745
|
* @param agentRoot - Agent root for config resolution
|
|
746
746
|
* @param onEngineEvent - Optional callback for engine lifecycle events (TP-040)
|
|
747
|
+
* @param onSupervisorAlert - Optional callback for supervisor alerts (TP-076)
|
|
747
748
|
*/
|
|
748
749
|
export async function executeOrchBatch(
|
|
749
750
|
args: string,
|
|
@@ -757,6 +758,7 @@ export async function executeOrchBatch(
|
|
|
757
758
|
workspaceRoot?: string,
|
|
758
759
|
agentRoot?: string,
|
|
759
760
|
onEngineEvent?: EngineEventCallback | null,
|
|
761
|
+
onSupervisorAlert?: SupervisorAlertCallback | null,
|
|
760
762
|
): Promise<void> {
|
|
761
763
|
const repoRoot = cwd;
|
|
762
764
|
// State files (.pi/batch-state.json, lane-state, etc.) belong in the workspace root,
|
|
@@ -768,6 +770,21 @@ export async function executeOrchBatch(
|
|
|
768
770
|
// batchState.batchId is read at call time (it's set in Phase 1).
|
|
769
771
|
const emitEvent: typeof emitEngineEvent = (sr, event, cb) => emitEngineEvent(sr, event, cb);
|
|
770
772
|
|
|
773
|
+
// ── TP-076: Supervisor alert emission helper ─────────────────
|
|
774
|
+
// Wraps the optional callback with a null guard for terse call sites.
|
|
775
|
+
const emitAlert = (alert: SupervisorAlert): void => {
|
|
776
|
+
if (onSupervisorAlert) {
|
|
777
|
+
try {
|
|
778
|
+
onSupervisorAlert(alert);
|
|
779
|
+
} catch (err: unknown) {
|
|
780
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
781
|
+
execLog("batch", batchState.batchId, `supervisor alert callback failed: ${msg}`, {
|
|
782
|
+
alertCategory: alert.category,
|
|
783
|
+
});
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
};
|
|
787
|
+
|
|
771
788
|
// ── TP-040 R002: Terminal event emission helper ──────────────
|
|
772
789
|
// Routes all early-return and terminal paths through consistent event
|
|
773
790
|
// emission so external consumers always receive a deterministic terminal
|
|
@@ -1309,6 +1326,34 @@ export async function executeOrchBatch(
|
|
|
1309
1326
|
reason: outcome?.exitReason || "unknown",
|
|
1310
1327
|
partialProgress: (outcome?.partialProgressCommits ?? 0) > 0,
|
|
1311
1328
|
}, onEngineEvent);
|
|
1329
|
+
|
|
1330
|
+
// ── TP-076: Emit supervisor alert for task failure ──────
|
|
1331
|
+
const laneForTask = latestAllocatedLanes.find(l => l.tasks.some(t => t.taskId === taskId));
|
|
1332
|
+
const exitReason = outcome?.exitReason || "unknown";
|
|
1333
|
+
const hasPartialProgress = (outcome?.partialProgressCommits ?? 0) > 0;
|
|
1334
|
+
emitAlert({
|
|
1335
|
+
category: "task-failure",
|
|
1336
|
+
summary:
|
|
1337
|
+
`⚠️ Task failure: ${taskId}\n` +
|
|
1338
|
+
` Exit reason: ${exitReason}\n` +
|
|
1339
|
+
` Lane: ${laneForTask?.laneId ?? "unknown"} (lane ${laneForTask?.laneNumber ?? "?"})\n` +
|
|
1340
|
+
` Partial progress preserved: ${hasPartialProgress ? "yes" : "no"}\n` +
|
|
1341
|
+
` Batch: wave ${waveIdx + 1}/${batchState.totalWaves}, ` +
|
|
1342
|
+
`${batchState.succeededTasks} succeeded, ${batchState.failedTasks} failed\n\n` +
|
|
1343
|
+
`Available actions:\n` +
|
|
1344
|
+
` - orch_status() to inspect current state\n` +
|
|
1345
|
+
` - orch_resume(force=true) to retry\n` +
|
|
1346
|
+
` - Read STATUS.md and lane logs for diagnosis`,
|
|
1347
|
+
context: {
|
|
1348
|
+
taskId,
|
|
1349
|
+
laneId: laneForTask?.laneId,
|
|
1350
|
+
laneNumber: laneForTask?.laneNumber,
|
|
1351
|
+
waveIndex: waveIdx,
|
|
1352
|
+
exitReason,
|
|
1353
|
+
partialProgress: hasPartialProgress,
|
|
1354
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1355
|
+
},
|
|
1356
|
+
});
|
|
1312
1357
|
}
|
|
1313
1358
|
|
|
1314
1359
|
// ── TS-009: Persist state after wave execution ──
|
|
@@ -1575,6 +1620,27 @@ export async function executeOrchBatch(
|
|
|
1575
1620
|
persistWarning,
|
|
1576
1621
|
"error",
|
|
1577
1622
|
);
|
|
1623
|
+
|
|
1624
|
+
// ── TP-076: Emit supervisor alert for rollback safe-stop ──
|
|
1625
|
+
const rollbackError = `Safe-stop at wave ${waveIdx + 1}: verification rollback failed.${persistWarning}`;
|
|
1626
|
+
emitAlert({
|
|
1627
|
+
category: "merge-failure",
|
|
1628
|
+
summary:
|
|
1629
|
+
`⚠️ Merge failed for wave ${waveIdx + 1} — verification rollback failed\n` +
|
|
1630
|
+
` Batch force-paused for manual recovery.\n` +
|
|
1631
|
+
` ${persistWarning ? persistWarning.trim() : "Check .pi/verification/ for recovery commands."}\n\n` +
|
|
1632
|
+
`Available actions:\n` +
|
|
1633
|
+
` - Check .pi/verification/ transaction records for recovery commands\n` +
|
|
1634
|
+
` - orch_status() to inspect current state\n` +
|
|
1635
|
+
` - orch_resume(force=true) after manual recovery`,
|
|
1636
|
+
context: {
|
|
1637
|
+
waveIndex: waveIdx,
|
|
1638
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1639
|
+
mergeError: rollbackError,
|
|
1640
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1641
|
+
},
|
|
1642
|
+
});
|
|
1643
|
+
|
|
1578
1644
|
preserveWorktreesForResume = true;
|
|
1579
1645
|
break;
|
|
1580
1646
|
}
|
|
@@ -1660,6 +1726,26 @@ export async function executeOrchBatch(
|
|
|
1660
1726
|
persistRuntimeState("merge-rollback-safe-stop", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1661
1727
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
1662
1728
|
|
|
1729
|
+
// ── TP-076: Emit supervisor alert for merge safe-stop ──
|
|
1730
|
+
emitAlert({
|
|
1731
|
+
category: "merge-failure",
|
|
1732
|
+
summary:
|
|
1733
|
+
`⚠️ Merge failed for wave ${waveIdx + 1} — rollback failure, batch force-paused\n` +
|
|
1734
|
+
` Merge policy: safe-stop (rollback failed)\n` +
|
|
1735
|
+
` Failed lane: ${mergeResult.failedLane ?? "unknown"}\n` +
|
|
1736
|
+
` Error: ${retryOutcome.errorMessage}\n\n` +
|
|
1737
|
+
`Available actions:\n` +
|
|
1738
|
+
` - Investigate failed merge, check .pi/verification/ for recovery commands\n` +
|
|
1739
|
+
` - orch_status() to inspect current state\n` +
|
|
1740
|
+
` - orch_resume(force=true) after manual recovery`,
|
|
1741
|
+
context: {
|
|
1742
|
+
waveIndex: waveIdx,
|
|
1743
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1744
|
+
mergeError: retryOutcome.errorMessage,
|
|
1745
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1746
|
+
},
|
|
1747
|
+
});
|
|
1748
|
+
|
|
1663
1749
|
// Emit merge safe-stop event (treated as exhausted — no further automatic recovery possible)
|
|
1664
1750
|
const mergeSafeStopSuggestion = "Merge rollback failed — batch force-paused for manual recovery. Check .pi/verification/ for recovery commands.";
|
|
1665
1751
|
emitTier0Event(stateRoot, {
|
|
@@ -1714,6 +1800,28 @@ export async function executeOrchBatch(
|
|
|
1714
1800
|
batchState.errors.push(exhaustionMsg);
|
|
1715
1801
|
persistRuntimeState("merge-retry-exhausted", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1716
1802
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
1803
|
+
|
|
1804
|
+
// ── TP-076: Emit supervisor alert for merge retry exhausted ──
|
|
1805
|
+
emitAlert({
|
|
1806
|
+
category: "merge-failure",
|
|
1807
|
+
summary:
|
|
1808
|
+
`⚠️ Merge failed for wave ${waveIdx + 1} — retry exhausted\n` +
|
|
1809
|
+
` Classification: ${retryOutcome.classification ?? "unknown"}\n` +
|
|
1810
|
+
` Attempts: ${retryOutcome.lastDecision.currentAttempt}/${retryOutcome.lastDecision.maxAttempts}\n` +
|
|
1811
|
+
` Failed lane: ${mergeResult.failedLane ?? "unknown"}\n` +
|
|
1812
|
+
` Error: ${exhaustionMsg}\n\n` +
|
|
1813
|
+
`Available actions:\n` +
|
|
1814
|
+
` - Investigate merge failure and retry manually\n` +
|
|
1815
|
+
` - orch_status() to inspect current state\n` +
|
|
1816
|
+
` - orch_resume(force=true) after fixing the issue`,
|
|
1817
|
+
context: {
|
|
1818
|
+
waveIndex: waveIdx,
|
|
1819
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1820
|
+
mergeError: exhaustionMsg,
|
|
1821
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1822
|
+
},
|
|
1823
|
+
});
|
|
1824
|
+
|
|
1717
1825
|
preserveWorktreesForResume = true;
|
|
1718
1826
|
break;
|
|
1719
1827
|
} else {
|
|
@@ -1730,6 +1838,27 @@ export async function executeOrchBatch(
|
|
|
1730
1838
|
batchState.errors.push(policyResult.errorMessage + classNote);
|
|
1731
1839
|
persistRuntimeState(policyResult.persistTrigger, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
1732
1840
|
onNotify(policyResult.notifyMessage + classNote, policyResult.notifyLevel);
|
|
1841
|
+
|
|
1842
|
+
// ── TP-076: Emit supervisor alert for merge failure (no-retry policy) ──
|
|
1843
|
+
emitAlert({
|
|
1844
|
+
category: "merge-failure",
|
|
1845
|
+
summary:
|
|
1846
|
+
`⚠️ Merge failed for wave ${waveIdx + 1}\n` +
|
|
1847
|
+
` Policy: ${policyResult.policy}${classNote}\n` +
|
|
1848
|
+
` Failed lane: ${mergeResult.failedLane ?? "unknown"}\n` +
|
|
1849
|
+
` Error: ${mergeResult.failureReason || "unknown"}\n\n` +
|
|
1850
|
+
`Available actions:\n` +
|
|
1851
|
+
` - Investigate failed merge\n` +
|
|
1852
|
+
` - orch_status() to inspect current state\n` +
|
|
1853
|
+
` - orch_resume(force=true) after fixing the issue`,
|
|
1854
|
+
context: {
|
|
1855
|
+
waveIndex: waveIdx,
|
|
1856
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1857
|
+
mergeError: mergeResult.failureReason || "unknown",
|
|
1858
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1859
|
+
},
|
|
1860
|
+
});
|
|
1861
|
+
|
|
1733
1862
|
// DO NOT cleanup/reset worktrees — preserve state for debugging/resume
|
|
1734
1863
|
preserveWorktreesForResume = true;
|
|
1735
1864
|
break;
|
|
@@ -2390,6 +2519,46 @@ export async function executeOrchBatch(
|
|
|
2390
2519
|
// ── TS-009: Persist terminal state ──
|
|
2391
2520
|
persistRuntimeState("batch-terminal", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discoveryRef, stateRoot);
|
|
2392
2521
|
|
|
2522
|
+
// ── TP-076: Emit supervisor alert for batch completion ──────
|
|
2523
|
+
if (batchState.phase === "completed" || batchState.phase === "failed") {
|
|
2524
|
+
const batchDurationMs = batchState.endedAt ? batchState.endedAt - batchState.startedAt : 0;
|
|
2525
|
+
const durationStr = batchDurationMs > 0
|
|
2526
|
+
? `${Math.floor(batchDurationMs / 60000)}m ${Math.round((batchDurationMs % 60000) / 1000)}s`
|
|
2527
|
+
: "unknown";
|
|
2528
|
+
if (batchState.phase === "completed" && batchState.failedTasks === 0) {
|
|
2529
|
+
emitAlert({
|
|
2530
|
+
category: "batch-complete",
|
|
2531
|
+
summary:
|
|
2532
|
+
`✅ Batch ${batchState.batchId} completed\n` +
|
|
2533
|
+
` ${batchState.succeededTasks}/${batchState.totalTasks} tasks succeeded\n` +
|
|
2534
|
+
` ${batchState.totalWaves} wave(s), duration: ${durationStr}\n` +
|
|
2535
|
+
` Merged to orch branch: ${batchState.orchBranch}\n\n` +
|
|
2536
|
+
`Ready for integration. Run orch_integrate() or review first.`,
|
|
2537
|
+
context: {
|
|
2538
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
2539
|
+
batchDurationMs,
|
|
2540
|
+
},
|
|
2541
|
+
});
|
|
2542
|
+
} else {
|
|
2543
|
+
emitAlert({
|
|
2544
|
+
category: "batch-complete",
|
|
2545
|
+
summary:
|
|
2546
|
+
`⚠️ Batch ${batchState.batchId} finished with failures\n` +
|
|
2547
|
+
` ${batchState.succeededTasks} succeeded, ${batchState.failedTasks} failed, ` +
|
|
2548
|
+
`${batchState.skippedTasks} skipped, ${batchState.blockedTasks} blocked\n` +
|
|
2549
|
+
` Duration: ${durationStr}\n\n` +
|
|
2550
|
+
`Available actions:\n` +
|
|
2551
|
+
` - orch_status() to review final state\n` +
|
|
2552
|
+
` - orch_integrate() if succeeded work should be kept\n` +
|
|
2553
|
+
` - orch_resume(force=true) to retry failed tasks`,
|
|
2554
|
+
context: {
|
|
2555
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
2556
|
+
batchDurationMs,
|
|
2557
|
+
},
|
|
2558
|
+
});
|
|
2559
|
+
}
|
|
2560
|
+
}
|
|
2561
|
+
|
|
2393
2562
|
// ── TP-040: Emit batch terminal event (R002: unified via helper) ─
|
|
2394
2563
|
emitTerminalEvent();
|
|
2395
2564
|
|
|
@@ -935,6 +935,7 @@ export function startBatchInWorker(
|
|
|
935
935
|
updateWidget: () => void,
|
|
936
936
|
onMonitorUpdate?: (state: import("./types.ts").MonitorState) => void,
|
|
937
937
|
onTerminal?: () => void,
|
|
938
|
+
onSupervisorAlert?: (alert: import("./types.ts").SupervisorAlert) => void,
|
|
938
939
|
): ChildProcess | null {
|
|
939
940
|
const workerPath = resolveEngineWorkerPath();
|
|
940
941
|
|
|
@@ -1011,6 +1012,11 @@ export function startBatchInWorker(
|
|
|
1011
1012
|
case "engine-event":
|
|
1012
1013
|
break;
|
|
1013
1014
|
|
|
1015
|
+
// ── TP-076: Supervisor alert handling ────────────────
|
|
1016
|
+
case "supervisor-alert":
|
|
1017
|
+
onSupervisorAlert?.(msg.alert);
|
|
1018
|
+
break;
|
|
1019
|
+
|
|
1014
1020
|
case "state-sync":
|
|
1015
1021
|
applySerializedState(batchState, msg.state);
|
|
1016
1022
|
updateWidget();
|
|
@@ -1050,6 +1056,28 @@ export function startBatchInWorker(
|
|
|
1050
1056
|
"error",
|
|
1051
1057
|
);
|
|
1052
1058
|
updateWidget();
|
|
1059
|
+
// ── TP-076: Alert supervisor about engine process error ──
|
|
1060
|
+
onSupervisorAlert?.({
|
|
1061
|
+
category: "task-failure",
|
|
1062
|
+
summary:
|
|
1063
|
+
`🔴 Engine process error — batch ${batchState.batchId} marked as failed\n` +
|
|
1064
|
+
` Error: ${err.message}\n\n` +
|
|
1065
|
+
`This is a critical engine failure. The batch cannot continue.\n` +
|
|
1066
|
+
`Available actions:\n` +
|
|
1067
|
+
` - orch_status() to inspect state\n` +
|
|
1068
|
+
` - orch_resume(force=true) to retry from last checkpoint`,
|
|
1069
|
+
context: {
|
|
1070
|
+
batchProgress: batchState.totalTasks > 0 ? {
|
|
1071
|
+
succeededTasks: batchState.succeededTasks,
|
|
1072
|
+
failedTasks: batchState.failedTasks,
|
|
1073
|
+
skippedTasks: batchState.skippedTasks,
|
|
1074
|
+
blockedTasks: batchState.blockedTasks,
|
|
1075
|
+
totalTasks: batchState.totalTasks,
|
|
1076
|
+
currentWave: batchState.currentWaveIndex + 1,
|
|
1077
|
+
totalWaves: batchState.totalWaves,
|
|
1078
|
+
} : undefined,
|
|
1079
|
+
},
|
|
1080
|
+
});
|
|
1053
1081
|
settle();
|
|
1054
1082
|
});
|
|
1055
1083
|
|
|
@@ -1065,6 +1093,28 @@ export function startBatchInWorker(
|
|
|
1065
1093
|
"error",
|
|
1066
1094
|
);
|
|
1067
1095
|
updateWidget();
|
|
1096
|
+
// ── TP-076: Alert supervisor about unexpected engine exit ──
|
|
1097
|
+
onSupervisorAlert?.({
|
|
1098
|
+
category: "task-failure",
|
|
1099
|
+
summary:
|
|
1100
|
+
`🔴 Engine process died unexpectedly (exit code ${code})\n` +
|
|
1101
|
+
` Batch ${batchState.batchId} marked as failed.\n\n` +
|
|
1102
|
+
`This is a critical engine failure. The batch cannot continue.\n` +
|
|
1103
|
+
`Available actions:\n` +
|
|
1104
|
+
` - orch_status() to inspect state\n` +
|
|
1105
|
+
` - orch_resume(force=true) to retry from last checkpoint`,
|
|
1106
|
+
context: {
|
|
1107
|
+
batchProgress: batchState.totalTasks > 0 ? {
|
|
1108
|
+
succeededTasks: batchState.succeededTasks,
|
|
1109
|
+
failedTasks: batchState.failedTasks,
|
|
1110
|
+
skippedTasks: batchState.skippedTasks,
|
|
1111
|
+
blockedTasks: batchState.blockedTasks,
|
|
1112
|
+
totalTasks: batchState.totalTasks,
|
|
1113
|
+
currentWave: batchState.currentWaveIndex + 1,
|
|
1114
|
+
totalWaves: batchState.totalWaves,
|
|
1115
|
+
} : undefined,
|
|
1116
|
+
},
|
|
1117
|
+
});
|
|
1068
1118
|
}
|
|
1069
1119
|
settle();
|
|
1070
1120
|
});
|
|
@@ -1893,33 +1943,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
1893
1943
|
};
|
|
1894
1944
|
transitionToRoutingMode(pi, supervisorState, postBatchContext);
|
|
1895
1945
|
},
|
|
1896
|
-
//
|
|
1897
|
-
() =>
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
repoRoot,
|
|
1902
|
-
orchBatchState,
|
|
1903
|
-
(message, level) => {
|
|
1904
|
-
ctx.ui.notify(message, level);
|
|
1905
|
-
updateOrchWidget();
|
|
1906
|
-
},
|
|
1907
|
-
(monState: MonitorState) => {
|
|
1908
|
-
const changed = !latestMonitorState ||
|
|
1909
|
-
latestMonitorState.totalDone !== monState.totalDone ||
|
|
1910
|
-
latestMonitorState.totalFailed !== monState.totalFailed ||
|
|
1911
|
-
latestMonitorState.lanes.some((l, i) =>
|
|
1912
|
-
l.currentTaskId !== monState.lanes[i]?.currentTaskId ||
|
|
1913
|
-
l.currentStep !== monState.lanes[i]?.currentStep ||
|
|
1914
|
-
l.completedChecks !== monState.lanes[i]?.completedChecks,
|
|
1915
|
-
);
|
|
1916
|
-
latestMonitorState = monState;
|
|
1917
|
-
if (changed) updateOrchWidget();
|
|
1918
|
-
},
|
|
1919
|
-
execCtx!.workspaceConfig,
|
|
1920
|
-
execCtx!.workspaceRoot,
|
|
1921
|
-
execCtx!.pointer?.agentRoot,
|
|
1922
|
-
),
|
|
1946
|
+
// ── TP-076: Supervisor alert handler — injects alerts as user messages ──
|
|
1947
|
+
(alert) => {
|
|
1948
|
+
if (!supervisorState.active) return; // Don't send orphaned messages
|
|
1949
|
+
ctx.sendUserMessage(alert.summary, { deliverAs: "followUp" });
|
|
1950
|
+
},
|
|
1923
1951
|
);
|
|
1924
1952
|
|
|
1925
1953
|
// Activate supervisor agent
|
|
@@ -2135,25 +2163,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
2135
2163
|
};
|
|
2136
2164
|
transitionToRoutingMode(pi, supervisorState, postBatchContext);
|
|
2137
2165
|
},
|
|
2138
|
-
//
|
|
2139
|
-
() =>
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
orchBatchState,
|
|
2144
|
-
(message, level) => {
|
|
2145
|
-
ctx.ui.notify(message, level);
|
|
2146
|
-
updateOrchWidget();
|
|
2147
|
-
},
|
|
2148
|
-
(monState: MonitorState) => {
|
|
2149
|
-
latestMonitorState = monState;
|
|
2150
|
-
updateOrchWidget();
|
|
2151
|
-
},
|
|
2152
|
-
execCtx!.workspaceConfig,
|
|
2153
|
-
execCtx!.workspaceRoot,
|
|
2154
|
-
execCtx!.pointer?.agentRoot,
|
|
2155
|
-
force,
|
|
2156
|
-
),
|
|
2166
|
+
// ── TP-076: Supervisor alert handler — injects alerts as user messages ──
|
|
2167
|
+
(alert) => {
|
|
2168
|
+
if (!supervisorState.active) return; // Don't send orphaned messages
|
|
2169
|
+
ctx.sendUserMessage(alert.summary, { deliverAs: "followUp" });
|
|
2170
|
+
},
|
|
2157
2171
|
);
|
|
2158
2172
|
|
|
2159
2173
|
// Activate supervisor agent on resume
|
|
@@ -16,7 +16,7 @@ import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolic
|
|
|
16
16
|
import type { CleanupGateRepoFailure } from "./messages.ts";
|
|
17
17
|
import { resolveOperatorId } from "./naming.ts";
|
|
18
18
|
import { applyPartialProgressToOutcomes, deleteBatchState, hasTaskDoneMarker, loadBatchState, persistRuntimeState, seedPendingOutcomesForAllocatedLanes, syncTaskOutcomesFromMonitor, upsertTaskOutcome } from "./persistence.ts";
|
|
19
|
-
import { defaultResilienceState, StateFileError } from "./types.ts";
|
|
19
|
+
import { buildBatchProgressSnapshot, defaultResilienceState, StateFileError } from "./types.ts";
|
|
20
20
|
import type { AllocatedLane, AllocatedTask, LaneExecutionResult, LaneTaskOutcome, LaneTaskStatus, MergeWaveResult, OrchBatchPhase, OrchBatchRuntimeState, OrchestratorConfig, ParsedTask, PersistedBatchState, PersistedLaneRecord, ReconciledTaskState, ResumeEligibility, ResumePoint, TaskRunnerConfig, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
|
|
21
21
|
import { buildDependencyGraph, resolveBaseBranch, resolveRepoRoot } from "./waves.ts";
|
|
22
22
|
import { deleteBranchBestEffort, forceCleanupWorktree, listWorktrees, preserveFailedLaneProgress, removeAllWorktrees, removeWorktree, safeResetWorktree, sleepSync } from "./worktree.ts";
|
|
@@ -749,6 +749,7 @@ export async function resumeOrchBatch(
|
|
|
749
749
|
workspaceRoot?: string,
|
|
750
750
|
agentRoot?: string,
|
|
751
751
|
force: boolean = false,
|
|
752
|
+
onSupervisorAlert?: import("./types.ts").SupervisorAlertCallback | null,
|
|
752
753
|
): Promise<void> {
|
|
753
754
|
const repoRoot = cwd;
|
|
754
755
|
// State files (.pi/batch-state.json, lane-state, etc.) belong in the workspace root,
|
|
@@ -756,6 +757,20 @@ export async function resumeOrchBatch(
|
|
|
756
757
|
const stateRoot = workspaceRoot ?? cwd;
|
|
757
758
|
const prefix = orchConfig.orchestrator.tmux_prefix;
|
|
758
759
|
|
|
760
|
+
// ── TP-076: Supervisor alert emission helper ─────────────────
|
|
761
|
+
const emitAlert = (alert: import("./types.ts").SupervisorAlert): void => {
|
|
762
|
+
if (onSupervisorAlert) {
|
|
763
|
+
try {
|
|
764
|
+
onSupervisorAlert(alert);
|
|
765
|
+
} catch (err: unknown) {
|
|
766
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
767
|
+
execLog("resume", "unknown", `supervisor alert callback failed: ${msg}`, {
|
|
768
|
+
alertCategory: alert.category,
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
};
|
|
773
|
+
|
|
759
774
|
// ── 1. Load persisted state ──────────────────────────────────
|
|
760
775
|
let persistedState: PersistedBatchState | null;
|
|
761
776
|
try {
|
|
@@ -1569,6 +1584,37 @@ export async function resumeOrchBatch(
|
|
|
1569
1584
|
batchState.blockedTaskIds.add(blocked);
|
|
1570
1585
|
}
|
|
1571
1586
|
|
|
1587
|
+
// ── TP-076: Emit supervisor alerts for task failures ────
|
|
1588
|
+
for (const taskId of waveResult.failedTaskIds) {
|
|
1589
|
+
const outcome = allTaskOutcomes.find(o => o.taskId === taskId);
|
|
1590
|
+
const laneForTask = latestAllocatedLanes.find(l => l.tasks.some(t => t.taskId === taskId));
|
|
1591
|
+
const exitReason = outcome?.exitReason || "unknown";
|
|
1592
|
+
const hasPartialProgress = (outcome?.partialProgressCommits ?? 0) > 0;
|
|
1593
|
+
emitAlert({
|
|
1594
|
+
category: "task-failure",
|
|
1595
|
+
summary:
|
|
1596
|
+
`⚠️ Task failure: ${taskId}\n` +
|
|
1597
|
+
` Exit reason: ${exitReason}\n` +
|
|
1598
|
+
` Lane: ${laneForTask?.laneId ?? "unknown"} (lane ${laneForTask?.laneNumber ?? "?"})\n` +
|
|
1599
|
+
` Partial progress preserved: ${hasPartialProgress ? "yes" : "no"}\n` +
|
|
1600
|
+
` Batch: wave ${waveIdx + 1}/${batchState.totalWaves}, ` +
|
|
1601
|
+
`${batchState.succeededTasks} succeeded, ${batchState.failedTasks} failed\n\n` +
|
|
1602
|
+
`Available actions:\n` +
|
|
1603
|
+
` - orch_status() to inspect current state\n` +
|
|
1604
|
+
` - orch_resume(force=true) to retry\n` +
|
|
1605
|
+
` - Read STATUS.md and lane logs for diagnosis`,
|
|
1606
|
+
context: {
|
|
1607
|
+
taskId,
|
|
1608
|
+
laneId: laneForTask?.laneId,
|
|
1609
|
+
laneNumber: laneForTask?.laneNumber,
|
|
1610
|
+
waveIndex: waveIdx,
|
|
1611
|
+
exitReason,
|
|
1612
|
+
partialProgress: hasPartialProgress,
|
|
1613
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1614
|
+
},
|
|
1615
|
+
});
|
|
1616
|
+
}
|
|
1617
|
+
|
|
1572
1618
|
persistRuntimeState("wave-execution-complete", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discovery, stateRoot);
|
|
1573
1619
|
|
|
1574
1620
|
const elapsedSec = Math.round((waveResult.endedAt - waveResult.startedAt) / 1000);
|
|
@@ -1750,6 +1796,26 @@ export async function resumeOrchBatch(
|
|
|
1750
1796
|
persistWarning,
|
|
1751
1797
|
"error",
|
|
1752
1798
|
);
|
|
1799
|
+
|
|
1800
|
+
// ── TP-076: Emit supervisor alert for rollback safe-stop ──
|
|
1801
|
+
emitAlert({
|
|
1802
|
+
category: "merge-failure",
|
|
1803
|
+
summary:
|
|
1804
|
+
`⚠️ Merge failed for wave ${waveIdx + 1} — verification rollback failed\n` +
|
|
1805
|
+
` Batch force-paused for manual recovery.\n` +
|
|
1806
|
+
` Check .pi/verification/ for recovery commands.\n\n` +
|
|
1807
|
+
`Available actions:\n` +
|
|
1808
|
+
` - Check .pi/verification/ transaction records\n` +
|
|
1809
|
+
` - orch_status() to inspect current state\n` +
|
|
1810
|
+
` - orch_resume(force=true) after manual recovery`,
|
|
1811
|
+
context: {
|
|
1812
|
+
waveIndex: waveIdx,
|
|
1813
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1814
|
+
mergeError: `Safe-stop: verification rollback failed at wave ${waveIdx + 1}`,
|
|
1815
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1816
|
+
},
|
|
1817
|
+
});
|
|
1818
|
+
|
|
1753
1819
|
preserveWorktreesForResume = true;
|
|
1754
1820
|
break;
|
|
1755
1821
|
}
|
|
@@ -1805,6 +1871,24 @@ export async function resumeOrchBatch(
|
|
|
1805
1871
|
batchState.errors.push(retryOutcome.errorMessage);
|
|
1806
1872
|
persistRuntimeState("merge-rollback-safe-stop", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discovery, stateRoot);
|
|
1807
1873
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
1874
|
+
|
|
1875
|
+
// ── TP-076: Emit supervisor alert for merge safe-stop ──
|
|
1876
|
+
emitAlert({
|
|
1877
|
+
category: "merge-failure",
|
|
1878
|
+
summary:
|
|
1879
|
+
`⚠️ Merge failed for wave ${waveIdx + 1} — rollback failure\n` +
|
|
1880
|
+
` Error: ${retryOutcome.errorMessage}\n\n` +
|
|
1881
|
+
`Available actions:\n` +
|
|
1882
|
+
` - orch_status() to inspect current state\n` +
|
|
1883
|
+
` - orch_resume(force=true) after manual recovery`,
|
|
1884
|
+
context: {
|
|
1885
|
+
waveIndex: waveIdx,
|
|
1886
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1887
|
+
mergeError: retryOutcome.errorMessage,
|
|
1888
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1889
|
+
},
|
|
1890
|
+
});
|
|
1891
|
+
|
|
1808
1892
|
preserveWorktreesForResume = true;
|
|
1809
1893
|
break;
|
|
1810
1894
|
} else if (retryOutcome.kind === "exhausted") {
|
|
@@ -1824,6 +1908,26 @@ export async function resumeOrchBatch(
|
|
|
1824
1908
|
batchState.errors.push(exhaustionMsg);
|
|
1825
1909
|
persistRuntimeState("merge-retry-exhausted", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discovery, stateRoot);
|
|
1826
1910
|
onNotify(retryOutcome.notifyMessage, "error");
|
|
1911
|
+
|
|
1912
|
+
// ── TP-076: Emit supervisor alert for merge retry exhausted ──
|
|
1913
|
+
emitAlert({
|
|
1914
|
+
category: "merge-failure",
|
|
1915
|
+
summary:
|
|
1916
|
+
`⚠️ Merge failed for wave ${waveIdx + 1} — retry exhausted\n` +
|
|
1917
|
+
` Classification: ${retryOutcome.classification ?? "unknown"}\n` +
|
|
1918
|
+
` Error: ${exhaustionMsg}\n\n` +
|
|
1919
|
+
`Available actions:\n` +
|
|
1920
|
+
` - Investigate merge failure and retry manually\n` +
|
|
1921
|
+
` - orch_status() to inspect current state\n` +
|
|
1922
|
+
` - orch_resume(force=true) after fixing the issue`,
|
|
1923
|
+
context: {
|
|
1924
|
+
waveIndex: waveIdx,
|
|
1925
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1926
|
+
mergeError: exhaustionMsg,
|
|
1927
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1928
|
+
},
|
|
1929
|
+
});
|
|
1930
|
+
|
|
1827
1931
|
preserveWorktreesForResume = true;
|
|
1828
1932
|
break;
|
|
1829
1933
|
} else {
|
|
@@ -1840,6 +1944,26 @@ export async function resumeOrchBatch(
|
|
|
1840
1944
|
batchState.errors.push(policyResult.errorMessage + classNote);
|
|
1841
1945
|
persistRuntimeState(policyResult.persistTrigger, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discovery, stateRoot);
|
|
1842
1946
|
onNotify(policyResult.notifyMessage + classNote, policyResult.notifyLevel);
|
|
1947
|
+
|
|
1948
|
+
// ── TP-076: Emit supervisor alert for merge failure (no-retry policy) ──
|
|
1949
|
+
emitAlert({
|
|
1950
|
+
category: "merge-failure",
|
|
1951
|
+
summary:
|
|
1952
|
+
`⚠️ Merge failed for wave ${waveIdx + 1}\n` +
|
|
1953
|
+
` Policy: ${policyResult.policy}${classNote}\n` +
|
|
1954
|
+
` Error: ${mergeResult.failureReason || "unknown"}\n\n` +
|
|
1955
|
+
`Available actions:\n` +
|
|
1956
|
+
` - Investigate failed merge\n` +
|
|
1957
|
+
` - orch_status() to inspect current state\n` +
|
|
1958
|
+
` - orch_resume(force=true) after fixing the issue`,
|
|
1959
|
+
context: {
|
|
1960
|
+
waveIndex: waveIdx,
|
|
1961
|
+
laneNumber: mergeResult.failedLane ?? undefined,
|
|
1962
|
+
mergeError: mergeResult.failureReason || "unknown",
|
|
1963
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
1964
|
+
},
|
|
1965
|
+
});
|
|
1966
|
+
|
|
1843
1967
|
preserveWorktreesForResume = true;
|
|
1844
1968
|
break;
|
|
1845
1969
|
}
|
|
@@ -2132,6 +2256,46 @@ export async function resumeOrchBatch(
|
|
|
2132
2256
|
|
|
2133
2257
|
persistRuntimeState("batch-terminal", batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, discovery, stateRoot);
|
|
2134
2258
|
|
|
2259
|
+
// ── TP-076: Emit supervisor alert for batch completion ──────
|
|
2260
|
+
if (batchState.phase === "completed" || batchState.phase === "failed") {
|
|
2261
|
+
const batchDurationMs = batchState.endedAt ? batchState.endedAt - batchState.startedAt : 0;
|
|
2262
|
+
const durationStr = batchDurationMs > 0
|
|
2263
|
+
? `${Math.floor(batchDurationMs / 60000)}m ${Math.round((batchDurationMs % 60000) / 1000)}s`
|
|
2264
|
+
: "unknown";
|
|
2265
|
+
if (batchState.phase === "completed" && batchState.failedTasks === 0) {
|
|
2266
|
+
emitAlert({
|
|
2267
|
+
category: "batch-complete",
|
|
2268
|
+
summary:
|
|
2269
|
+
`✅ Batch ${batchState.batchId} completed\n` +
|
|
2270
|
+
` ${batchState.succeededTasks}/${batchState.totalTasks} tasks succeeded\n` +
|
|
2271
|
+
` ${batchState.totalWaves} wave(s), duration: ${durationStr}\n` +
|
|
2272
|
+
` Merged to orch branch: ${batchState.orchBranch}\n\n` +
|
|
2273
|
+
`Ready for integration. Run orch_integrate() or review first.`,
|
|
2274
|
+
context: {
|
|
2275
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
2276
|
+
batchDurationMs,
|
|
2277
|
+
},
|
|
2278
|
+
});
|
|
2279
|
+
} else {
|
|
2280
|
+
emitAlert({
|
|
2281
|
+
category: "batch-complete",
|
|
2282
|
+
summary:
|
|
2283
|
+
`⚠️ Batch ${batchState.batchId} finished with failures\n` +
|
|
2284
|
+
` ${batchState.succeededTasks} succeeded, ${batchState.failedTasks} failed, ` +
|
|
2285
|
+
`${batchState.skippedTasks} skipped, ${batchState.blockedTasks} blocked\n` +
|
|
2286
|
+
` Duration: ${durationStr}\n\n` +
|
|
2287
|
+
`Available actions:\n` +
|
|
2288
|
+
` - orch_status() to review final state\n` +
|
|
2289
|
+
` - orch_integrate() if succeeded work should be kept\n` +
|
|
2290
|
+
` - orch_resume(force=true) to retry failed tasks`,
|
|
2291
|
+
context: {
|
|
2292
|
+
batchProgress: buildBatchProgressSnapshot(batchState),
|
|
2293
|
+
batchDurationMs,
|
|
2294
|
+
},
|
|
2295
|
+
});
|
|
2296
|
+
}
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2135
2299
|
// ── TP-031: Emit diagnostic reports (JSONL + markdown) ──
|
|
2136
2300
|
// Non-fatal: errors are logged but never crash batch finalization.
|
|
2137
2301
|
emitDiagnosticReports(assembleDiagnosticInput(orchConfig, batchState, wavePlan, latestAllocatedLanes, allTaskOutcomes, stateRoot));
|
|
@@ -703,6 +703,74 @@ In ALL modes, you log every action to the audit trail.
|
|
|
703
703
|
|
|
704
704
|
---
|
|
705
705
|
|
|
706
|
+
## 13a. Autonomous Alert Handling (TP-076)
|
|
707
|
+
|
|
708
|
+
The engine sends you **structured alerts** via IPC when significant events
|
|
709
|
+
occur. These alerts arrive as conversation messages — you don't need to poll
|
|
710
|
+
or check status manually. The engine wakes you up when you're needed.
|
|
711
|
+
|
|
712
|
+
### Alert Categories
|
|
713
|
+
|
|
714
|
+
| Category | Emoji | When |
|
|
715
|
+
|----------|-------|------|
|
|
716
|
+
| `task-failure` | ⚠️ | A task failed after deterministic recovery was exhausted |
|
|
717
|
+
| `merge-failure` | ⚠️ | Wave merge failed and batch paused |
|
|
718
|
+
| `batch-complete` | ✅/⚠️ | Batch finished (all waves done, with or without failures) |
|
|
719
|
+
|
|
720
|
+
### Alert Format
|
|
721
|
+
|
|
722
|
+
Each alert contains:
|
|
723
|
+
- **Summary**: Human-readable text describing what happened and what actions
|
|
724
|
+
are available. This is what you see in the conversation.
|
|
725
|
+
- **Context**: Structured data (taskId, laneId, waveIndex, exitReason,
|
|
726
|
+
batchProgress, etc.) embedded in the message for your reference.
|
|
727
|
+
|
|
728
|
+
### Response Protocol
|
|
729
|
+
|
|
730
|
+
When you receive an alert, follow this sequence:
|
|
731
|
+
|
|
732
|
+
1. **Acknowledge** — "I see the failure. Investigating."
|
|
733
|
+
2. **Diagnose** — Call `orch_status()`, read STATUS.md, check logs
|
|
734
|
+
3. **Decide** — Based on diagnosis, choose an action
|
|
735
|
+
4. **Act** — Execute the recovery (resume, retry, skip, abort)
|
|
736
|
+
5. **Report** — Tell the operator what happened and what was done
|
|
737
|
+
6. **Learn** — If this is a recurring pattern, note it for future improvement
|
|
738
|
+
|
|
739
|
+
### Autonomy Rules for Alert Response
|
|
740
|
+
|
|
741
|
+
- **Do NOT ask the operator for permission** on routine recovery actions:
|
|
742
|
+
- Retrying a failed task (`orch_resume(force=true)`)
|
|
743
|
+
- Skipping dependents of a failed task
|
|
744
|
+
- Reading logs and batch state for diagnosis
|
|
745
|
+
|
|
746
|
+
- **DO escalate to the operator** for genuinely ambiguous situations:
|
|
747
|
+
- The same task has failed multiple times with different errors
|
|
748
|
+
- An unknown error type you haven't seen before
|
|
749
|
+
- Destructive actions (aborting a batch with partial work)
|
|
750
|
+
- Repeated merge failures on the same wave
|
|
751
|
+
|
|
752
|
+
### Available Tools for Recovery
|
|
753
|
+
|
|
754
|
+
You have these orchestrator tools available:
|
|
755
|
+
- `orch_status()` — Check current batch state
|
|
756
|
+
- `orch_resume(force=true)` — Resume/retry from last checkpoint
|
|
757
|
+
- `orch_pause()` — Pause the batch gracefully
|
|
758
|
+
- `orch_abort(hard?)` — Abort the batch
|
|
759
|
+
- `orch_integrate(mode?, force?)` — Integrate completed batch
|
|
760
|
+
- `orch_start(target)` — Start a new batch
|
|
761
|
+
|
|
762
|
+
Plus general tools: `read`, `write`, `edit`, `bash`, `grep`, `find`, `ls`
|
|
763
|
+
for inspecting files, running git commands, and editing batch state.
|
|
764
|
+
|
|
765
|
+
### Critical Engine Alerts
|
|
766
|
+
|
|
767
|
+
If the engine process itself crashes (process error or unexpected exit), you
|
|
768
|
+
receive a critical alert with category `task-failure` and a 🔴 emoji. These
|
|
769
|
+
indicate an infrastructure-level failure, not a task-level failure. Recovery
|
|
770
|
+
typically requires `orch_resume(force=true)` after checking batch state.
|
|
771
|
+
|
|
772
|
+
---
|
|
773
|
+
|
|
706
774
|
## 14. Your Startup Checklist
|
|
707
775
|
|
|
708
776
|
When you activate at the start of a batch:
|
|
@@ -1780,6 +1780,120 @@ export interface EngineEvent {
|
|
|
1780
1780
|
*/
|
|
1781
1781
|
export type EngineEventCallback = (event: EngineEvent) => void;
|
|
1782
1782
|
|
|
1783
|
+
|
|
1784
|
+
// ── Supervisor Alert Types (TP-076) ──────────────────────────────────
|
|
1785
|
+
|
|
1786
|
+
/**
|
|
1787
|
+
* Alert category for supervisor notifications.
|
|
1788
|
+
*
|
|
1789
|
+
* Matches the alert categories in the autonomous supervisor spec:
|
|
1790
|
+
* - `task-failure`: A task failed after deterministic recovery was exhausted
|
|
1791
|
+
* - `merge-failure`: Wave merge failed and batch paused
|
|
1792
|
+
* - `batch-complete`: Batch finished (all waves done)
|
|
1793
|
+
*
|
|
1794
|
+
* Note: `stall` detection is deferred to a future phase (requires
|
|
1795
|
+
* last-activity tracking not yet built).
|
|
1796
|
+
*
|
|
1797
|
+
* @since TP-076
|
|
1798
|
+
*/
|
|
1799
|
+
export type SupervisorAlertCategory = "task-failure" | "merge-failure" | "batch-complete";
|
|
1800
|
+
|
|
1801
|
+
/**
|
|
1802
|
+
* Structured context payload for supervisor alerts.
|
|
1803
|
+
*
|
|
1804
|
+
* All fields are IPC-serializable (no functions, no circular refs, no Maps/Sets).
|
|
1805
|
+
* Each alert category populates the relevant subset of optional fields.
|
|
1806
|
+
*
|
|
1807
|
+
* @since TP-076
|
|
1808
|
+
*/
|
|
1809
|
+
export interface SupervisorAlertContext {
|
|
1810
|
+
/** Task ID (for task-failure alerts) */
|
|
1811
|
+
taskId?: string;
|
|
1812
|
+
/** Lane ID, e.g., "lane-1" (for task-failure alerts) */
|
|
1813
|
+
laneId?: string;
|
|
1814
|
+
/** Lane number (for task-failure and merge-failure alerts) */
|
|
1815
|
+
laneNumber?: number;
|
|
1816
|
+
/** Wave index, 0-based (for merge-failure and batch-complete alerts) */
|
|
1817
|
+
waveIndex?: number;
|
|
1818
|
+
/** Exit reason string (for task-failure alerts) */
|
|
1819
|
+
exitReason?: string;
|
|
1820
|
+
/** Whether partial progress was preserved (for task-failure alerts) */
|
|
1821
|
+
partialProgress?: boolean;
|
|
1822
|
+
/** Batch progress summary */
|
|
1823
|
+
batchProgress?: {
|
|
1824
|
+
succeededTasks: number;
|
|
1825
|
+
failedTasks: number;
|
|
1826
|
+
skippedTasks: number;
|
|
1827
|
+
blockedTasks: number;
|
|
1828
|
+
totalTasks: number;
|
|
1829
|
+
currentWave: number;
|
|
1830
|
+
totalWaves: number;
|
|
1831
|
+
};
|
|
1832
|
+
/** Merge failure reason (for merge-failure alerts) */
|
|
1833
|
+
mergeError?: string;
|
|
1834
|
+
/** Batch duration in milliseconds (for batch-complete alerts) */
|
|
1835
|
+
batchDurationMs?: number;
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
/**
|
|
1839
|
+
* Structured supervisor alert message.
|
|
1840
|
+
*
|
|
1841
|
+
* Emitted by the engine (child process) via IPC when the supervisor
|
|
1842
|
+
* needs to be notified of an event requiring attention or acknowledgement.
|
|
1843
|
+
*
|
|
1844
|
+
* Design:
|
|
1845
|
+
* - All fields are plain JSON-serializable values (IPC-safe).
|
|
1846
|
+
* - `category` determines the alert type and which `context` fields are populated.
|
|
1847
|
+
* - `summary` is a pre-formatted, human-readable string suitable for direct
|
|
1848
|
+
* display to the supervisor LLM as a conversation message.
|
|
1849
|
+
* - `context` provides structured data for programmatic consumption.
|
|
1850
|
+
*
|
|
1851
|
+
* @since TP-076
|
|
1852
|
+
*/
|
|
1853
|
+
export interface SupervisorAlert {
|
|
1854
|
+
/** Alert category — determines handling behavior */
|
|
1855
|
+
category: SupervisorAlertCategory;
|
|
1856
|
+
/** Human-readable summary suitable for display as a chat message */
|
|
1857
|
+
summary: string;
|
|
1858
|
+
/** Structured context data (all fields IPC-serializable) */
|
|
1859
|
+
context: SupervisorAlertContext;
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
/**
|
|
1863
|
+
* Callback type for supervisor alert emission.
|
|
1864
|
+
*
|
|
1865
|
+
* The engine (child process) calls this when it needs to alert the
|
|
1866
|
+
* supervisor about a significant event. The main thread handler
|
|
1867
|
+
* converts the alert into a `sendUserMessage` call to wake the
|
|
1868
|
+
* supervisor LLM.
|
|
1869
|
+
*
|
|
1870
|
+
* @since TP-076
|
|
1871
|
+
*/
|
|
1872
|
+
export type SupervisorAlertCallback = (alert: SupervisorAlert) => void;
|
|
1873
|
+
|
|
1874
|
+
/**
|
|
1875
|
+
* Build a batch progress snapshot from runtime state.
|
|
1876
|
+
*
|
|
1877
|
+
* Pure function — extracts the current progress counters from
|
|
1878
|
+
* OrchBatchRuntimeState into the IPC-serializable format used
|
|
1879
|
+
* by SupervisorAlertContext.batchProgress.
|
|
1880
|
+
*
|
|
1881
|
+
* @since TP-076
|
|
1882
|
+
*/
|
|
1883
|
+
export function buildBatchProgressSnapshot(
|
|
1884
|
+
batchState: OrchBatchRuntimeState,
|
|
1885
|
+
): NonNullable<SupervisorAlertContext["batchProgress"]> {
|
|
1886
|
+
return {
|
|
1887
|
+
succeededTasks: batchState.succeededTasks,
|
|
1888
|
+
failedTasks: batchState.failedTasks,
|
|
1889
|
+
skippedTasks: batchState.skippedTasks,
|
|
1890
|
+
blockedTasks: batchState.blockedTasks,
|
|
1891
|
+
totalTasks: batchState.totalTasks,
|
|
1892
|
+
currentWave: batchState.currentWaveIndex + 1, // 1-based for display
|
|
1893
|
+
totalWaves: batchState.totalWaves,
|
|
1894
|
+
};
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1783
1897
|
/**
|
|
1784
1898
|
* Build the base fields for an engine event.
|
|
1785
1899
|
*
|