taskplane 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -437,6 +437,10 @@ function loadTelemetryData(batchState) {
437
437
  for (const event of events) {
438
438
  switch (event.type) {
439
439
  case "message_end": {
440
+ // A successful message_end means any prior retry resolved.
441
+ // Clear retryActive to prevent stale retry badges from persisting
442
+ // across batches or after transient API errors recover.
443
+ acc.retryActive = false;
440
444
  const usage = event.message?.usage;
441
445
  if (usage) {
442
446
  acc.inputTokens += usage.input || 0;
@@ -11,7 +11,7 @@ import type { MonitorUpdateCallback } from "./execution.ts";
11
11
  // classifyExit no longer called directly — Tier 0 uses exitDiagnostic.classification
12
12
  // from the diagnostic-reports pipeline (populated by assembleDiagnosticInput).
13
13
  import { getCurrentBranch, runGit } from "./git.ts";
14
- import { mergeWaveByRepo } from "./merge.ts";
14
+ import { mergeWaveByRepo, MergeHealthMonitor } from "./merge.ts";
15
15
  import { applyMergeRetryLoop, computeCleanupGatePolicy, computeMergeFailurePolicy, extractFailedRepoId, formatRepoMergeSummary, ORCH_MESSAGES } from "./messages.ts";
16
16
  import type { CleanupGateRepoFailure } from "./messages.ts";
17
17
  import { assembleDiagnosticInput, emitDiagnosticReports } from "./diagnostic-reports.ts";
@@ -1358,19 +1358,41 @@ export async function executeOrchBatch(
1358
1358
  laneCount: mergeableLaneCount,
1359
1359
  }, onEngineEvent);
1360
1360
 
1361
- mergeResult = await mergeWaveByRepo(
1362
- waveResult.allocatedLanes,
1363
- waveResult,
1364
- waveIdx + 1,
1365
- orchConfig,
1366
- repoRoot,
1367
- batchState.batchId,
1368
- batchState.orchBranch,
1369
- workspaceConfig,
1361
+ // TP-056: Start merge health monitor during merge phase
1362
+ const mergeHealthMonitor = new MergeHealthMonitor({
1370
1363
  stateRoot,
1371
- agentRoot,
1372
- runnerConfig.testing_commands,
1373
- );
1364
+ batchId: batchState.batchId,
1365
+ waveIndex: waveIdx,
1366
+ phase: batchState.phase,
1367
+ onDeadSession: (sessionName, laneNumber) => {
1368
+ execLog("batch", batchState.batchId, `merge health monitor detected dead session`, {
1369
+ sessionName,
1370
+ laneNumber,
1371
+ waveIndex: waveIdx,
1372
+ });
1373
+ },
1374
+ });
1375
+ mergeHealthMonitor.start();
1376
+
1377
+ try {
1378
+ mergeResult = await mergeWaveByRepo(
1379
+ waveResult.allocatedLanes,
1380
+ waveResult,
1381
+ waveIdx + 1,
1382
+ orchConfig,
1383
+ repoRoot,
1384
+ batchState.batchId,
1385
+ batchState.orchBranch,
1386
+ workspaceConfig,
1387
+ stateRoot,
1388
+ agentRoot,
1389
+ runnerConfig.testing_commands,
1390
+ mergeHealthMonitor,
1391
+ );
1392
+ } finally {
1393
+ // TP-056: Always stop the health monitor when merge phase ends
1394
+ mergeHealthMonitor.stop();
1395
+ }
1374
1396
  allMergeResults.push(mergeResult);
1375
1397
  batchState.mergeResults.push(mergeResult);
1376
1398
 
@@ -8,12 +8,13 @@ import { join, dirname, resolve, relative } from "path";
8
8
 
9
9
  import { buildLaneEnvVars, buildTmuxSpawnArgs, execLog, generateTelemetryPaths, resolveRpcWrapperPath, resolveTelemOpId, tmuxHasSession, tmuxKillSession, toTmuxPath } from "./execution.ts";
10
10
  import { resolveOperatorId } from "./naming.ts";
11
- import { MERGE_POLL_INTERVAL_MS, MERGE_RESULT_GRACE_MS, MERGE_RESULT_READ_RETRIES, MERGE_RESULT_READ_RETRY_DELAY_MS, MERGE_SPAWN_RETRY_MAX, MERGE_TIMEOUT_MAX_RETRIES, MERGE_TIMEOUT_MS, MergeError, VALID_MERGE_STATUSES } from "./types.ts";
12
- import type { AllocatedLane, LaneExecutionResult, MergeLaneResult, MergeResult, MergeResultStatus, MergeWaveResult, OrchestratorConfig, RepoMergeOutcome, TaskRunnerConfig, TransactionRecord, TransactionStatus, VerificationBaselineResult, WaveExecutionResult, WorkspaceConfig } from "./types.ts";
11
+ import { MERGE_POLL_INTERVAL_MS, MERGE_RESULT_GRACE_MS, MERGE_RESULT_READ_RETRIES, MERGE_RESULT_READ_RETRY_DELAY_MS, MERGE_SPAWN_RETRY_MAX, MERGE_TIMEOUT_MAX_RETRIES, MERGE_TIMEOUT_MS, MERGE_HEALTH_POLL_INTERVAL_MS, MERGE_HEALTH_WARNING_THRESHOLD_MS, MERGE_HEALTH_STUCK_THRESHOLD_MS, MERGE_HEALTH_CAPTURE_LINES, MergeError, VALID_MERGE_STATUSES, buildEngineEventBase } from "./types.ts";
12
+ import type { AllocatedLane, LaneExecutionResult, MergeLaneResult, MergeResult, MergeResultStatus, MergeWaveResult, OrchestratorConfig, RepoMergeOutcome, TaskRunnerConfig, TransactionRecord, TransactionStatus, VerificationBaselineResult, WaveExecutionResult, WorkspaceConfig, MergeHealthStatus, MergeHealthEventType, MergeSessionSnapshot, MergeSessionHealthState, EngineEvent, OrchBatchPhase } from "./types.ts";
13
13
  import { resolveBaseBranch, resolveRepoRoot } from "./waves.ts";
14
14
  import { generateMergeWorktreePath, sleepAsync, sleepSync } from "./worktree.ts";
15
15
  import { getCurrentBranch, runGit } from "./git.ts";
16
16
  import { ORCH_MESSAGES } from "./messages.ts";
17
+ import { emitEngineEvent } from "./persistence.ts";
17
18
  import { loadOrchestratorConfig } from "./config.ts";
18
19
  import { captureBaseline, diffFingerprints, runVerificationCommands, parseTestOutput, deduplicateFingerprints } from "./verification.ts";
19
20
  import type { VerificationBaseline, FingerprintDiff, TestFingerprint } from "./verification.ts";
@@ -1019,6 +1020,7 @@ export async function mergeWave(
1019
1020
  agentRoot?: string,
1020
1021
  testingCommands?: Record<string, string>,
1021
1022
  repoId?: string,
1023
+ healthMonitor?: MergeHealthMonitor | null,
1022
1024
  ): Promise<MergeWaveResult> {
1023
1025
  const startTime = Date.now();
1024
1026
  const tmuxPrefix = config.orchestrator.tmux_prefix;
@@ -1319,13 +1321,19 @@ export async function mergeWave(
1319
1321
 
1320
1322
  // Re-spawn merge agent for the retry
1321
1323
  await spawnMergeAgent(sessionName, repoRoot, mergeWorkDir, requestFilePath, config, stateRoot, agentRoot);
1324
+ // TP-056: Re-register with health monitor after respawn
1325
+ if (healthMonitor) healthMonitor.addSession(sessionName, lane.laneNumber, resultFilePath);
1322
1326
  } else {
1323
1327
  // First attempt: spawn merge agent
1324
1328
  await spawnMergeAgent(sessionName, repoRoot, mergeWorkDir, requestFilePath, config, stateRoot, agentRoot);
1329
+ // TP-056: Register session with health monitor
1330
+ if (healthMonitor) healthMonitor.addSession(sessionName, lane.laneNumber, resultFilePath);
1325
1331
  }
1326
1332
 
1327
1333
  try {
1328
1334
  mergeResult = await waitForMergeResult(resultFilePath, sessionName, currentTimeoutMs);
1335
+ // TP-056: Deregister session from health monitor on completion
1336
+ if (healthMonitor) healthMonitor.removeSession(sessionName);
1329
1337
  lastTimeoutError = null;
1330
1338
  break; // Success — exit retry loop
1331
1339
  } catch (waitErr: unknown) {
@@ -1336,9 +1344,13 @@ export async function mergeWave(
1336
1344
  ) {
1337
1345
  // Timeout — will retry on next loop iteration
1338
1346
  lastTimeoutError = waitErr;
1347
+ // TP-056: Deregister before retry (will re-register on respawn)
1348
+ if (healthMonitor) healthMonitor.removeSession(sessionName);
1339
1349
  continue;
1340
1350
  }
1341
1351
  // Non-timeout error or final retry exhausted — propagate
1352
+ // TP-056: Deregister session from health monitor on error
1353
+ if (healthMonitor) healthMonitor.removeSession(sessionName);
1342
1354
  throw waitErr;
1343
1355
  }
1344
1356
  }
@@ -1933,6 +1945,7 @@ export async function mergeWaveByRepo(
1933
1945
  stateRoot?: string,
1934
1946
  agentRoot?: string,
1935
1947
  testingCommands?: Record<string, string>,
1948
+ healthMonitor?: MergeHealthMonitor | null,
1936
1949
  ): Promise<MergeWaveResult> {
1937
1950
  const startTime = Date.now();
1938
1951
 
@@ -1988,6 +2001,8 @@ export async function mergeWaveByRepo(
1988
2001
  stateRoot,
1989
2002
  agentRoot,
1990
2003
  testingCommands,
2004
+ undefined, // repoId
2005
+ healthMonitor,
1991
2006
  );
1992
2007
  // Attach empty repoResults for consistent shape
1993
2008
  return { ...result, repoResults: [] };
@@ -2044,6 +2059,7 @@ export async function mergeWaveByRepo(
2044
2059
  agentRoot,
2045
2060
  testingCommands,
2046
2061
  group.repoId,
2062
+ healthMonitor,
2047
2063
  );
2048
2064
 
2049
2065
  // Accumulate lane results
@@ -2154,6 +2170,8 @@ export async function mergeWaveByRepo(
2154
2170
  return aggregateResult;
2155
2171
  }
2156
2172
 
2173
+
2174
+
2157
2175
  // ── Auto-Integration ─────────────────────────────────────────────────
2158
2176
 
2159
2177
  /**
@@ -2256,3 +2274,341 @@ export function attemptAutoIntegration(
2256
2274
  return true;
2257
2275
  }
2258
2276
 
2277
+ // ── Merge Health Monitor (TP-056) ────────────────────────────────────
2278
+
2279
+ /**
2280
+ * Capture the last N lines of a tmux pane for activity detection.
2281
+ *
2282
+ * Uses `tmux capture-pane` with `-p` (stdout) and `-S -N` (last N lines).
2283
+ * Returns null if the session doesn't exist or capture fails.
2284
+ *
2285
+ * @param sessionName - TMUX session name
2286
+ * @param lines - Number of lines to capture from the bottom
2287
+ * @returns Captured text, or null on failure
2288
+ *
2289
+ * @since TP-056
2290
+ */
2291
+ export function captureMergePaneOutput(
2292
+ sessionName: string,
2293
+ lines: number = MERGE_HEALTH_CAPTURE_LINES,
2294
+ ): string | null {
2295
+ try {
2296
+ const result = spawnSync("tmux", [
2297
+ "capture-pane",
2298
+ "-t", sessionName,
2299
+ "-p", // print to stdout
2300
+ "-S", `-${lines}`, // last N lines
2301
+ ], { encoding: "utf-8", timeout: 5_000 });
2302
+
2303
+ if (result.status !== 0) {
2304
+ return null;
2305
+ }
2306
+
2307
+ return result.stdout ?? null;
2308
+ } catch {
2309
+ return null;
2310
+ }
2311
+ }
2312
+
2313
+ /**
2314
+ * Classify the health of a merge session based on session liveness
2315
+ * and pane output activity.
2316
+ *
2317
+ * Pure function — no side effects. Takes the current state and produces
2318
+ * a health classification.
2319
+ *
2320
+ * @param sessionAlive - Whether the tmux session is alive
2321
+ * @param hasResultFile - Whether the merge result file exists
2322
+ * @param currentOutput - Current pane capture (null if session dead or capture failed)
2323
+ * @param healthState - Tracked health state for this session
2324
+ * @param now - Current epoch ms
2325
+ * @returns Updated health status
2326
+ *
2327
+ * @since TP-056
2328
+ */
2329
+ export function classifyMergeHealth(
2330
+ sessionAlive: boolean,
2331
+ hasResultFile: boolean,
2332
+ currentOutput: string | null,
2333
+ healthState: MergeSessionHealthState,
2334
+ now: number,
2335
+ ): MergeHealthStatus {
2336
+ // Dead session with no result file → immediate detection
2337
+ if (!sessionAlive && !hasResultFile) {
2338
+ return "dead";
2339
+ }
2340
+
2341
+ // Session dead but result file exists → merge completed, healthy
2342
+ if (!sessionAlive && hasResultFile) {
2343
+ return "healthy";
2344
+ }
2345
+
2346
+ // Session alive — check activity
2347
+ const lastContent = healthState.lastSnapshot?.content ?? null;
2348
+ const outputChanged = currentOutput !== null
2349
+ && (lastContent === null || currentOutput !== lastContent);
2350
+
2351
+ if (outputChanged) {
2352
+ return "healthy";
2353
+ }
2354
+
2355
+ // No output change — compute stale duration
2356
+ const staleDuration = now - healthState.lastActivityAt;
2357
+
2358
+ if (staleDuration >= MERGE_HEALTH_STUCK_THRESHOLD_MS) {
2359
+ return "stuck";
2360
+ }
2361
+
2362
+ if (staleDuration >= MERGE_HEALTH_WARNING_THRESHOLD_MS) {
2363
+ return "warning";
2364
+ }
2365
+
2366
+ return "healthy";
2367
+ }
2368
+
2369
+ /**
2370
+ * Active merge session health monitor.
2371
+ *
2372
+ * Runs on its own polling interval during the merge phase, checking each
2373
+ * active merge session for liveness and activity. Emits structured events
2374
+ * for the supervisor to consume.
2375
+ *
2376
+ * Design principles (from PROMPT.md):
2377
+ * - Does NOT kill sessions autonomously — emits events for operator decision
2378
+ * - Runs independently of the merge result poll
2379
+ * - Stores session snapshots in memory (ephemeral, not persisted)
2380
+ * - Emits structured events to the unified events.jsonl
2381
+ *
2382
+ * @since TP-056
2383
+ */
2384
+ export class MergeHealthMonitor {
2385
+ /** Per-session health state, keyed by session name */
2386
+ private sessions: Map<string, MergeSessionHealthState> = new Map();
2387
+
2388
+ /** Timer handle for the polling loop */
2389
+ private pollTimer: ReturnType<typeof setInterval> | null = null;
2390
+
2391
+ /** Whether the monitor is currently running */
2392
+ private _running = false;
2393
+
2394
+ /** Callback invoked when a dead session is detected (for early exit signaling) */
2395
+ private _onDeadSession: ((sessionName: string, laneNumber: number) => void) | null = null;
2396
+
2397
+ /** Event emission context */
2398
+ private stateRoot: string;
2399
+ private batchId: string;
2400
+ private waveIndex: number;
2401
+ private phase: OrchBatchPhase;
2402
+
2403
+ /** Polling interval override (for testing) */
2404
+ private pollIntervalMs: number;
2405
+
2406
+ constructor(opts: {
2407
+ stateRoot: string;
2408
+ batchId: string;
2409
+ waveIndex: number;
2410
+ phase: OrchBatchPhase;
2411
+ pollIntervalMs?: number;
2412
+ onDeadSession?: (sessionName: string, laneNumber: number) => void;
2413
+ }) {
2414
+ this.stateRoot = opts.stateRoot;
2415
+ this.batchId = opts.batchId;
2416
+ this.waveIndex = opts.waveIndex;
2417
+ this.phase = opts.phase;
2418
+ this.pollIntervalMs = opts.pollIntervalMs ?? MERGE_HEALTH_POLL_INTERVAL_MS;
2419
+ this._onDeadSession = opts.onDeadSession ?? null;
2420
+ }
2421
+
2422
+ /** Whether the monitor is currently running */
2423
+ get running(): boolean {
2424
+ return this._running;
2425
+ }
2426
+
2427
+ /**
2428
+ * Register a merge session for monitoring.
2429
+ *
2430
+ * @param sessionName - TMUX session name
2431
+ * @param laneNumber - Lane number the session belongs to
2432
+ * @param resultPath - Path to the expected merge result file
2433
+ */
2434
+ addSession(sessionName: string, laneNumber: number, resultPath: string): void {
2435
+ const now = Date.now();
2436
+ this.sessions.set(sessionName, {
2437
+ sessionName,
2438
+ laneNumber,
2439
+ lastSnapshot: null,
2440
+ lastActivityAt: now,
2441
+ status: "healthy",
2442
+ warningEmitted: false,
2443
+ stuckEmitted: false,
2444
+ deadEmitted: false,
2445
+ });
2446
+ // Store resultPath for later lookup
2447
+ this._resultPaths.set(sessionName, resultPath);
2448
+ }
2449
+
2450
+ /** Result file paths for each session (for dead-session detection) */
2451
+ private _resultPaths: Map<string, string> = new Map();
2452
+
2453
+ /**
2454
+ * Remove a session from monitoring (e.g., merge completed for this lane).
2455
+ */
2456
+ removeSession(sessionName: string): void {
2457
+ this.sessions.delete(sessionName);
2458
+ this._resultPaths.delete(sessionName);
2459
+ }
2460
+
2461
+ /**
2462
+ * Start the health monitoring polling loop.
2463
+ */
2464
+ start(): void {
2465
+ if (this._running) return;
2466
+ this._running = true;
2467
+
2468
+ execLog("merge-health", "monitor", "merge health monitor started", {
2469
+ sessionCount: this.sessions.size,
2470
+ pollIntervalMs: this.pollIntervalMs,
2471
+ });
2472
+
2473
+ this.pollTimer = setInterval(() => {
2474
+ this.poll();
2475
+ }, this.pollIntervalMs);
2476
+ }
2477
+
2478
+ /**
2479
+ * Stop the health monitoring polling loop.
2480
+ */
2481
+ stop(): void {
2482
+ if (!this._running) return;
2483
+ this._running = false;
2484
+
2485
+ if (this.pollTimer !== null) {
2486
+ clearInterval(this.pollTimer);
2487
+ this.pollTimer = null;
2488
+ }
2489
+
2490
+ execLog("merge-health", "monitor", "merge health monitor stopped", {
2491
+ sessionCount: this.sessions.size,
2492
+ });
2493
+
2494
+ this.sessions.clear();
2495
+ this._resultPaths.clear();
2496
+ }
2497
+
2498
+ /**
2499
+ * Run a single poll cycle across all monitored sessions.
2500
+ *
2501
+ * Exposed as public for testing — normally called by the interval timer.
2502
+ */
2503
+ poll(): void {
2504
+ const now = Date.now();
2505
+
2506
+ for (const [sessionName, state] of this.sessions) {
2507
+ const sessionAlive = tmuxHasSession(sessionName);
2508
+ const resultPath = this._resultPaths.get(sessionName) ?? "";
2509
+ const hasResultFile = resultPath ? existsSync(resultPath) : false;
2510
+
2511
+ // Capture pane output for activity detection
2512
+ const currentOutput = sessionAlive
2513
+ ? captureMergePaneOutput(sessionName)
2514
+ : null;
2515
+
2516
+ // Classify health
2517
+ const newStatus = classifyMergeHealth(
2518
+ sessionAlive,
2519
+ hasResultFile,
2520
+ currentOutput,
2521
+ state,
2522
+ now,
2523
+ );
2524
+
2525
+ // Update snapshot if output changed
2526
+ if (currentOutput !== null && (
2527
+ state.lastSnapshot === null || currentOutput !== state.lastSnapshot.content
2528
+ )) {
2529
+ state.lastSnapshot = { content: currentOutput, capturedAt: now };
2530
+ state.lastActivityAt = now;
2531
+ }
2532
+
2533
+ const prevStatus = state.status;
2534
+ state.status = newStatus;
2535
+
2536
+ // Emit events based on status transitions
2537
+ this._emitHealthEvents(state, now);
2538
+
2539
+ // Signal dead session for early exit
2540
+ if (newStatus === "dead" && !state.deadEmitted) {
2541
+ state.deadEmitted = true;
2542
+ if (this._onDeadSession) {
2543
+ this._onDeadSession(sessionName, state.laneNumber);
2544
+ }
2545
+ }
2546
+ }
2547
+ }
2548
+
2549
+ /**
2550
+ * Emit health events based on current state.
2551
+ * De-duplicates: each event type emitted at most once per session.
2552
+ */
2553
+ private _emitHealthEvents(state: MergeSessionHealthState, now: number): void {
2554
+ const stalledMinutes = Math.round((now - state.lastActivityAt) / 60_000);
2555
+
2556
+ if (state.status === "warning" && !state.warningEmitted) {
2557
+ state.warningEmitted = true;
2558
+ const event: EngineEvent = {
2559
+ ...buildEngineEventBase("merge_health_warning", this.batchId, this.waveIndex, this.phase),
2560
+ laneNumber: state.laneNumber,
2561
+ sessionName: state.sessionName,
2562
+ healthStatus: "warning",
2563
+ stalledMinutes,
2564
+ reason: `Merge agent on lane ${state.laneNumber} may be stalled (no output for ${stalledMinutes} min)`,
2565
+ };
2566
+ emitEngineEvent(this.stateRoot, event);
2567
+ execLog("merge-health", state.sessionName, `⚠️ merge session possibly stalled`, {
2568
+ stalledMinutes,
2569
+ laneNumber: state.laneNumber,
2570
+ });
2571
+ }
2572
+
2573
+ if (state.status === "dead" && !state.deadEmitted) {
2574
+ // deadEmitted is set in poll() after onDeadSession callback
2575
+ const event: EngineEvent = {
2576
+ ...buildEngineEventBase("merge_health_dead", this.batchId, this.waveIndex, this.phase),
2577
+ laneNumber: state.laneNumber,
2578
+ sessionName: state.sessionName,
2579
+ healthStatus: "dead",
2580
+ reason: `Merge agent on lane ${state.laneNumber} session died without producing a result`,
2581
+ };
2582
+ emitEngineEvent(this.stateRoot, event);
2583
+ execLog("merge-health", state.sessionName, `💀 merge session dead — no result file`, {
2584
+ laneNumber: state.laneNumber,
2585
+ });
2586
+ }
2587
+
2588
+ if (state.status === "stuck" && !state.stuckEmitted) {
2589
+ state.stuckEmitted = true;
2590
+ const event: EngineEvent = {
2591
+ ...buildEngineEventBase("merge_health_stuck", this.batchId, this.waveIndex, this.phase),
2592
+ laneNumber: state.laneNumber,
2593
+ sessionName: state.sessionName,
2594
+ healthStatus: "stuck",
2595
+ stalledMinutes,
2596
+ reason: `Merge agent on lane ${state.laneNumber} appears stuck (no output for ${stalledMinutes} min). Consider killing and retrying.`,
2597
+ };
2598
+ emitEngineEvent(this.stateRoot, event);
2599
+ execLog("merge-health", state.sessionName, `🔒 merge session stuck`, {
2600
+ stalledMinutes,
2601
+ laneNumber: state.laneNumber,
2602
+ });
2603
+ }
2604
+ }
2605
+
2606
+ /**
2607
+ * Get the current health states for all monitored sessions.
2608
+ * Used for testing and inspection.
2609
+ */
2610
+ getSessionStates(): Map<string, MergeSessionHealthState> {
2611
+ return new Map(this.sessions);
2612
+ }
2613
+ }
2614
+
@@ -189,6 +189,8 @@ detects the sessionId mismatch and yields gracefully.
189
189
 
190
190
  Engine lifecycle events (wave_start, task_complete, merge_success, etc.)
191
191
  are written here as JSONL. You tail this file for proactive monitoring.
192
+ Merge health monitoring events (merge_health_warning, merge_health_dead,
193
+ merge_health_stuck) are also written here when merge agents stall or die.
192
194
 
193
195
  **Audit trail:** `.pi/supervisor/actions.jsonl`
194
196
 
@@ -244,6 +246,8 @@ Wave N starts
244
246
  | Execute | Worker makes no progress | STATUS.md unchanged for `stallTimeout` minutes |
245
247
  | Execute | API error (rate limit, overload) | Session exits, pi handles retry internally |
246
248
  | Merge | Merge agent times out | No result JSON within `merge.timeoutMinutes` |
249
+ | Merge | Merge agent stalls silently | `merge_health_warning` or `merge_health_stuck` event |
250
+ | Merge | Merge agent session dies | `merge_health_dead` event — no result file |
247
251
  | Merge | Merge conflicts too complex | Merge agent can't resolve |
248
252
  | Merge | Verification tests fail | Tests fail in merge worktree |
249
253
  | Cleanup | Windows file locks | `git worktree remove` fails |
@@ -324,6 +328,32 @@ git log --oneline orch/{branch}..task/{lane-branch} # empty = already merged
324
328
  ```
325
329
  5. Update batch state and advance.
326
330
 
331
+ ### Pattern 1b: Merge Agent Stall (TP-056)
332
+
333
+ **Symptom:** Supervisor notification: "⚠️ Merge agent on lane N may be stalled (no output for 10 min)"
334
+ or "🔒 Merge agent on lane N appears stuck (no output for 20 min)."
335
+
336
+ **How it works:** The merge health monitor (TP-056) actively polls merge agent
337
+ tmux sessions every 2 minutes during the merge phase. It checks:
338
+ - **Session liveness:** `tmux has-session` — is the session alive?
339
+ - **Activity detection:** Captures the last 10 lines of pane output and compares
340
+ with the previous snapshot. If output hasn't changed, the session may be stalled.
341
+
342
+ **Escalation tiers:**
343
+ - **Healthy:** Session alive, output changing → no action
344
+ - **Warning** (10 min no output): `merge_health_warning` event → supervisor notification
345
+ - **Dead** (session gone, no result file): `merge_health_dead` event → immediate detection
346
+ - **Stuck** (20 min no output): `merge_health_stuck` event → recommendation to kill
347
+
348
+ **Recovery:**
349
+ 1. Attach to the session to inspect: `tmux attach -t {sessionName}`
350
+ 2. If truly stuck, kill the session: `tmux kill-session -t {sessionName}`
351
+ 3. The engine detects the dead session and applies the `on_merge_failure` policy
352
+ 4. Resume with `/orch-resume` if needed
353
+
354
+ **Note:** The monitor does NOT kill sessions autonomously — it emits events for
355
+ the operator or supervisor to decide.
356
+
327
357
  ### Pattern 2: Resume Skips Wave Merge (Bug #102)
328
358
 
329
359
  **Symptom:** After `/orch-resume`, the engine says "wave N: no tasks to execute
@@ -3212,6 +3212,10 @@ interface ParsedEvent {
3212
3212
  skippedTasks?: number;
3213
3213
  blockedTasks?: number;
3214
3214
  batchDurationMs?: number;
3215
+ // ── Merge health monitoring fields (TP-056) ─────────────────
3216
+ sessionName?: string;
3217
+ healthStatus?: string;
3218
+ stalledMinutes?: number;
3215
3219
  // ── Tier0Event-specific optional fields ──────────────────────
3216
3220
  pattern?: string;
3217
3221
  attempt?: number;
@@ -3239,6 +3243,9 @@ const SIGNIFICANT_EVENT_TYPES = new Set<UnifiedEventType>([
3239
3243
  "merge_start",
3240
3244
  "merge_success",
3241
3245
  "merge_failed",
3246
+ "merge_health_warning",
3247
+ "merge_health_dead",
3248
+ "merge_health_stuck",
3242
3249
  "batch_complete",
3243
3250
  "batch_paused",
3244
3251
  "tier0_escalation",
@@ -3477,6 +3484,20 @@ export function formatEventNotification(
3477
3484
  return `⚠️ **Wave ${waveNum} merge failed**${laneInfo}: ${reason}.\n` +
3478
3485
  ` Recovery may be needed. Check the merge logs for details.`;
3479
3486
  }
3487
+ case "merge_health_warning": {
3488
+ const lane = event.laneNumber !== undefined ? event.laneNumber : "?";
3489
+ const mins = event.stalledMinutes ?? "?";
3490
+ return `⚠️ Merge agent on lane ${lane} may be stalled (no output for ${mins} min)`;
3491
+ }
3492
+ case "merge_health_dead": {
3493
+ const lane = event.laneNumber !== undefined ? event.laneNumber : "?";
3494
+ return `💀 Merge agent on lane ${lane} session died — triggering early retry`;
3495
+ }
3496
+ case "merge_health_stuck": {
3497
+ const lane = event.laneNumber !== undefined ? event.laneNumber : "?";
3498
+ const mins = event.stalledMinutes ?? "?";
3499
+ return `🔒 Merge agent on lane ${lane} appears stuck (no output for ${mins} min). Consider killing and retrying.`;
3500
+ }
3480
3501
  case "batch_complete": {
3481
3502
  const parts: string[] = [];
3482
3503
  if (event.succeededTasks !== undefined) parts.push(`${event.succeededTasks} succeeded`);
@@ -3598,6 +3619,8 @@ export function shouldNotify(
3598
3619
  eventType === "batch_complete" ||
3599
3620
  eventType === "batch_paused" ||
3600
3621
  eventType === "merge_failed" ||
3622
+ eventType === "merge_health_dead" ||
3623
+ eventType === "merge_health_stuck" ||
3601
3624
  eventType === "tier0_escalation"
3602
3625
  ) {
3603
3626
  return true;
@@ -1276,6 +1276,100 @@ export const MERGE_SPAWN_RETRY_MAX = 2;
1276
1276
  */
1277
1277
  export const MERGE_TIMEOUT_MAX_RETRIES = 2;
1278
1278
 
1279
+ // ── Merge Health Monitoring Constants (TP-056) ───────────────────────
1280
+
1281
+ /**
1282
+ * Polling interval for merge health monitor (ms).
1283
+ * Independent of the merge result poll — runs on its own cadence.
1284
+ * @since TP-056
1285
+ */
1286
+ export const MERGE_HEALTH_POLL_INTERVAL_MS = 2 * 60 * 1000; // 2 minutes
1287
+
1288
+ /**
1289
+ * Threshold (ms) after which a merge session with no new output
1290
+ * is classified as "possibly stalled" and a warning event is emitted.
1291
+ * @since TP-056
1292
+ */
1293
+ export const MERGE_HEALTH_WARNING_THRESHOLD_MS = 10 * 60 * 1000; // 10 minutes
1294
+
1295
+ /**
1296
+ * Threshold (ms) after which a merge session with no new output
1297
+ * is classified as "stuck" and a stuck event is emitted.
1298
+ * @since TP-056
1299
+ */
1300
+ export const MERGE_HEALTH_STUCK_THRESHOLD_MS = 20 * 60 * 1000; // 20 minutes
1301
+
1302
+ /**
1303
+ * Number of lines to capture from the bottom of a tmux pane
1304
+ * for activity detection via snapshot comparison.
1305
+ * @since TP-056
1306
+ */
1307
+ export const MERGE_HEALTH_CAPTURE_LINES = 10;
1308
+
1309
+ // ── Merge Health Event Types (TP-056) ────────────────────────────────
1310
+
1311
+ /**
1312
+ * Health classification for a merge session.
1313
+ *
1314
+ * - `healthy`: Session alive, output changing
1315
+ * - `warning`: Session alive, no new output for MERGE_HEALTH_WARNING_THRESHOLD_MS
1316
+ * - `dead`: Session gone, no result file
1317
+ * - `stuck`: Session alive, no new output for MERGE_HEALTH_STUCK_THRESHOLD_MS
1318
+ *
1319
+ * @since TP-056
1320
+ */
1321
+ export type MergeHealthStatus = "healthy" | "warning" | "dead" | "stuck";
1322
+
1323
+ /**
1324
+ * Engine event types for merge health monitoring.
1325
+ *
1326
+ * These extend the EngineEventType union and are emitted to the
1327
+ * unified events.jsonl for supervisor consumption.
1328
+ *
1329
+ * @since TP-056
1330
+ */
1331
+ export type MergeHealthEventType =
1332
+ | "merge_health_warning"
1333
+ | "merge_health_dead"
1334
+ | "merge_health_stuck";
1335
+
1336
+ /**
1337
+ * Snapshot of a merge session's pane output at a point in time.
1338
+ * Used for activity detection by comparing successive snapshots.
1339
+ *
1340
+ * @since TP-056
1341
+ */
1342
+ export interface MergeSessionSnapshot {
1343
+ /** Captured pane content (last N lines) */
1344
+ content: string;
1345
+ /** Epoch ms when the snapshot was taken */
1346
+ capturedAt: number;
1347
+ }
1348
+
1349
+ /**
1350
+ * Per-session health tracking state.
1351
+ *
1352
+ * @since TP-056
1353
+ */
1354
+ export interface MergeSessionHealthState {
1355
+ /** TMUX session name */
1356
+ sessionName: string;
1357
+ /** Lane number this session belongs to */
1358
+ laneNumber: number;
1359
+ /** Last captured pane snapshot */
1360
+ lastSnapshot: MergeSessionSnapshot | null;
1361
+ /** Epoch ms when the last output change was detected */
1362
+ lastActivityAt: number;
1363
+ /** Current health classification */
1364
+ status: MergeHealthStatus;
1365
+ /** Whether a warning event has been emitted (prevent duplicates) */
1366
+ warningEmitted: boolean;
1367
+ /** Whether a stuck event has been emitted (prevent duplicates) */
1368
+ stuckEmitted: boolean;
1369
+ /** Whether a dead event has been emitted (prevent duplicates) */
1370
+ deadEmitted: boolean;
1371
+ }
1372
+
1279
1373
 
1280
1374
  // ── Merge Retry Policy Matrix (TP-033 Step 2) ───────────────────────
1281
1375
 
@@ -1557,6 +1651,9 @@ export type EngineEventType =
1557
1651
  | "merge_start"
1558
1652
  | "merge_success"
1559
1653
  | "merge_failed"
1654
+ | "merge_health_warning"
1655
+ | "merge_health_dead"
1656
+ | "merge_health_stuck"
1560
1657
  | "batch_complete"
1561
1658
  | "batch_paused";
1562
1659
 
@@ -1622,6 +1719,15 @@ export interface EngineEvent {
1622
1719
  blockedTasks?: number;
1623
1720
  /** Batch duration in milliseconds (for batch_complete) */
1624
1721
  batchDurationMs?: number;
1722
+
1723
+ // ── Merge health monitoring fields (TP-056) ──────────────────
1724
+
1725
+ /** TMUX session name (for merge_health_* events) */
1726
+ sessionName?: string;
1727
+ /** Merge health status classification (for merge_health_* events) */
1728
+ healthStatus?: MergeHealthStatus;
1729
+ /** Minutes since last activity (for merge_health_warning, merge_health_stuck) */
1730
+ stalledMinutes?: number;
1625
1731
  }
1626
1732
 
1627
1733
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "taskplane",
3
- "version": "0.11.0",
3
+ "version": "0.12.0",
4
4
  "description": "AI agent orchestration for pi — parallel task execution with checkpoint discipline",
5
5
  "keywords": [
6
6
  "pi-package",