taskplane 0.30.5 → 0.30.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/taskplane/agent-bridge-extension.ts +66 -8
- package/extensions/taskplane/agent-host.ts +170 -9
- package/extensions/taskplane/config-loader.ts +9 -0
- package/extensions/taskplane/config-schema.ts +47 -1
- package/extensions/taskplane/diagnostic-reports.ts +109 -6
- package/extensions/taskplane/diagnostics.ts +3 -0
- package/extensions/taskplane/engine-identity.ts +401 -0
- package/extensions/taskplane/engine-worker.ts +59 -3
- package/extensions/taskplane/engine.ts +137 -17
- package/extensions/taskplane/execution.ts +89 -8
- package/extensions/taskplane/extension.ts +1130 -96
- package/extensions/taskplane/git.ts +74 -0
- package/extensions/taskplane/lane-runner.ts +971 -95
- package/extensions/taskplane/process-registry.ts +7 -2
- package/extensions/taskplane/resume.ts +559 -23
- package/extensions/taskplane/review-analysis.ts +450 -0
- package/extensions/taskplane/segment-recovery.ts +192 -0
- package/extensions/taskplane/supervisor-primer.md +182 -2
- package/extensions/taskplane/supervisor.ts +225 -17
- package/extensions/taskplane/types.ts +142 -4
- package/extensions/taskplane/worktree.ts +143 -2
- package/package.json +1 -1
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
StateFileError,
|
|
27
27
|
WorkspaceConfigError,
|
|
28
28
|
freshOrchBatchState,
|
|
29
|
+
generateBatchId,
|
|
29
30
|
} from "./types.ts";
|
|
30
31
|
import type {
|
|
31
32
|
AbortMode,
|
|
@@ -44,6 +45,7 @@ import {
|
|
|
44
45
|
loadBatchState,
|
|
45
46
|
saveBatchState,
|
|
46
47
|
detectOrphanSessions,
|
|
48
|
+
reconstructBatchStateFromRuntime,
|
|
47
49
|
updateBatchHistoryIntegration,
|
|
48
50
|
} from "./persistence.ts";
|
|
49
51
|
import {
|
|
@@ -53,7 +55,12 @@ import {
|
|
|
53
55
|
formatPreflightResults,
|
|
54
56
|
runPreflight,
|
|
55
57
|
} from "./worktree.ts";
|
|
56
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
batchTaskScope,
|
|
60
|
+
computeTransitiveDependents,
|
|
61
|
+
execLog,
|
|
62
|
+
resolveCanonicalTaskPaths,
|
|
63
|
+
} from "./execution.ts";
|
|
57
64
|
import { executeOrchBatch } from "./engine.ts";
|
|
58
65
|
import { formatDiscoveryResults, runDiscovery } from "./discovery.ts";
|
|
59
66
|
import { formatOrchSessions, listOrchSessions } from "./sessions.ts";
|
|
@@ -67,6 +74,7 @@ import {
|
|
|
67
74
|
} from "./config.ts";
|
|
68
75
|
import { resolveOperatorId } from "./naming.ts";
|
|
69
76
|
import { reconstructAllocatedLanes, resumeOrchBatch } from "./resume.ts";
|
|
77
|
+
import { markTaskSegmentsSkipped, resetTaskSegmentsForRetry } from "./segment-recovery.ts";
|
|
70
78
|
import { buildExecutionContext } from "./workspace.ts";
|
|
71
79
|
import { openSettingsTui } from "./settings-tui.ts";
|
|
72
80
|
import { loadProjectConfig } from "./config-loader.ts";
|
|
@@ -102,6 +110,14 @@ import {
|
|
|
102
110
|
isProcessAlive as registryIsProcessAlive,
|
|
103
111
|
isTerminalStatus,
|
|
104
112
|
} from "./process-registry.ts";
|
|
113
|
+
import {
|
|
114
|
+
assessEngineLiveness,
|
|
115
|
+
decideRecoveryOwnership,
|
|
116
|
+
findBatchesForOrchBranch,
|
|
117
|
+
markEngineExited,
|
|
118
|
+
recordOperatorConfirmedShutdown,
|
|
119
|
+
writeEngineIdentity,
|
|
120
|
+
} from "./engine-identity.ts";
|
|
105
121
|
import type { MailboxMessageType } from "./types.ts";
|
|
106
122
|
import {
|
|
107
123
|
activateSupervisor,
|
|
@@ -118,6 +134,9 @@ import {
|
|
|
118
134
|
triggerSupervisorIntegration,
|
|
119
135
|
presentBatchSummary,
|
|
120
136
|
resolveModelFromString,
|
|
137
|
+
isStaleExtensionCtx,
|
|
138
|
+
safeCtxCallFromCallback,
|
|
139
|
+
logRecoveryAction,
|
|
121
140
|
} from "./supervisor.ts";
|
|
122
141
|
import { SupervisorNoticeGate } from "./supervisor-dispatch.ts";
|
|
123
142
|
import { repairToolResultOrdering } from "./context-repair.ts";
|
|
@@ -336,6 +355,17 @@ export function resolveIntegrationContext(
|
|
|
336
355
|
|
|
337
356
|
// Source 2: CLI positional branch arg overrides or fills in
|
|
338
357
|
if (parsed.orchBranchArg) {
|
|
358
|
+
// #631: an explicit branch that differs from the persisted batch's branch
|
|
359
|
+
// must NOT inherit that batch's id — cleanup/history/ownership would then
|
|
360
|
+
// target an unrelated batch. The batch behind the selected branch (if any)
|
|
361
|
+
// is looked up from runtime artifacts by the caller.
|
|
362
|
+
if (orchBranch && batchId && orchBranch !== parsed.orchBranchArg) {
|
|
363
|
+
notices.push(
|
|
364
|
+
`ℹ️ Persisted batch ${batchId} belongs to ${orchBranch}; integrating ${parsed.orchBranchArg} instead — ` +
|
|
365
|
+
`batch-scoped cleanup/history will use the batch associated with that branch, if any.`,
|
|
366
|
+
);
|
|
367
|
+
batchId = "";
|
|
368
|
+
}
|
|
339
369
|
orchBranch = parsed.orchBranchArg;
|
|
340
370
|
}
|
|
341
371
|
|
|
@@ -1052,10 +1082,17 @@ export function startBatchAsync(
|
|
|
1052
1082
|
batchState.endedAt = Date.now();
|
|
1053
1083
|
batchState.errors.push(`Unhandled engine error: ${errMsg}`);
|
|
1054
1084
|
}
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1085
|
+
// #620: this .catch runs when the (main-thread fallback) engine promise
|
|
1086
|
+
// rejects later — an async window where ctx may be stale. Guard the UI
|
|
1087
|
+
// sink so a stale-ctx throw can't crash Pi; onTerminal still runs.
|
|
1088
|
+
safeCtxCallFromCallback(
|
|
1089
|
+
() =>
|
|
1090
|
+
ctx.ui.notify(
|
|
1091
|
+
`❌ Engine crashed with unhandled error: ${errMsg}\n` +
|
|
1092
|
+
` Batch ${batchState.batchId} marked as failed.`,
|
|
1093
|
+
"error",
|
|
1094
|
+
),
|
|
1095
|
+
"startBatchAsync.catch.notify",
|
|
1059
1096
|
);
|
|
1060
1097
|
updateWidget();
|
|
1061
1098
|
// TP-041 R002-3: Deactivate supervisor on all terminal paths
|
|
@@ -1112,6 +1149,55 @@ function resolveEngineWorkerPath(): string {
|
|
|
1112
1149
|
*
|
|
1113
1150
|
* @since TP-071
|
|
1114
1151
|
*/
|
|
1152
|
+
/** #631: true while a main-thread fallback engine is running in this process. */
|
|
1153
|
+
let fallbackEngineActive = false;
|
|
1154
|
+
export function isFallbackEngineActive(): boolean {
|
|
1155
|
+
return fallbackEngineActive;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
/**
|
|
1159
|
+
* #631: delete `batch-state.json` at `stateRoot` ONLY when it belongs to the
|
|
1160
|
+
* integrated batch (same batchId) or branch (same orchBranch). Returns true when
|
|
1161
|
+
* deleted, false when an unrelated batch's checkpoint was preserved, null when
|
|
1162
|
+
* nothing was persisted.
|
|
1163
|
+
*/
|
|
1164
|
+
export function deleteBatchStateIfOwned(
|
|
1165
|
+
stateRoot: string,
|
|
1166
|
+
batchId: string,
|
|
1167
|
+
orchBranch: string,
|
|
1168
|
+
): boolean | null {
|
|
1169
|
+
let persisted: PersistedBatchState | null = null;
|
|
1170
|
+
try {
|
|
1171
|
+
persisted = loadBatchState(stateRoot);
|
|
1172
|
+
} catch {
|
|
1173
|
+
// Unreadable state: ownership cannot be established — preserve it (a corrupt
|
|
1174
|
+
// checkpoint is still evidence an operator may need).
|
|
1175
|
+
return false;
|
|
1176
|
+
}
|
|
1177
|
+
if (!persisted) return null;
|
|
1178
|
+
const owned =
|
|
1179
|
+
(batchId && persisted.batchId === batchId) || (orchBranch && persisted.orchBranch === orchBranch);
|
|
1180
|
+
if (!owned) return false;
|
|
1181
|
+
deleteBatchState(stateRoot);
|
|
1182
|
+
return true;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
/** #631: resolve true once the child has actually terminated, false on timeout. */
|
|
1186
|
+
export function waitForChildExit(child: ChildProcess, timeoutMs: number): Promise<boolean> {
|
|
1187
|
+
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(true);
|
|
1188
|
+
return new Promise((resolve) => {
|
|
1189
|
+
const timer = setTimeout(() => {
|
|
1190
|
+
child.off("exit", onExit);
|
|
1191
|
+
resolve(false);
|
|
1192
|
+
}, timeoutMs);
|
|
1193
|
+
const onExit = () => {
|
|
1194
|
+
clearTimeout(timer);
|
|
1195
|
+
resolve(true);
|
|
1196
|
+
};
|
|
1197
|
+
child.once("exit", onExit);
|
|
1198
|
+
});
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1115
1201
|
export function startBatchInWorker(
|
|
1116
1202
|
wkData: EngineWorkerData,
|
|
1117
1203
|
batchState: import("./types.ts").OrchBatchRuntimeState,
|
|
@@ -1142,9 +1228,16 @@ export function startBatchInWorker(
|
|
|
1142
1228
|
});
|
|
1143
1229
|
} catch (spawnErr: unknown) {
|
|
1144
1230
|
const errMsg = spawnErr instanceof Error ? spawnErr.message : String(spawnErr);
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1231
|
+
// #620: stale-safe (uniform with the async callbacks below). This runs
|
|
1232
|
+
// synchronously during the command so ctx is normally fresh, but guarding
|
|
1233
|
+
// keeps the "no bare ctx.ui.notify in startBatchInWorker" invariant simple.
|
|
1234
|
+
safeCtxCallFromCallback(
|
|
1235
|
+
() =>
|
|
1236
|
+
ctx.ui.notify(
|
|
1237
|
+
`⚠️ Engine process spawn failed: ${errMsg}\n Falling back to main-thread execution.`,
|
|
1238
|
+
"warning",
|
|
1239
|
+
),
|
|
1240
|
+
"spawn.fallback.notify",
|
|
1148
1241
|
);
|
|
1149
1242
|
// Construct fallback engine function from workerData and run on main thread
|
|
1150
1243
|
const wsConfig = wkData.workspaceConfig
|
|
@@ -1159,7 +1252,7 @@ export function startBatchInWorker(
|
|
|
1159
1252
|
wkData.cwd,
|
|
1160
1253
|
batchState,
|
|
1161
1254
|
(msg: string, lvl: "info" | "warning" | "error") => {
|
|
1162
|
-
ctx.ui.notify(msg, lvl);
|
|
1255
|
+
safeCtxCallFromCallback(() => ctx.ui.notify(msg, lvl), "fallback.notify");
|
|
1163
1256
|
updateWidget();
|
|
1164
1257
|
},
|
|
1165
1258
|
(monState: import("./types.ts").MonitorState) => {
|
|
@@ -1182,7 +1275,7 @@ export function startBatchInWorker(
|
|
|
1182
1275
|
wkData.cwd,
|
|
1183
1276
|
batchState,
|
|
1184
1277
|
(msg: string, lvl: "info" | "warning" | "error") => {
|
|
1185
|
-
ctx.ui.notify(msg, lvl);
|
|
1278
|
+
safeCtxCallFromCallback(() => ctx.ui.notify(msg, lvl), "fallback.notify");
|
|
1186
1279
|
updateWidget();
|
|
1187
1280
|
},
|
|
1188
1281
|
(monState: import("./types.ts").MonitorState) => {
|
|
@@ -1197,7 +1290,38 @@ export function startBatchInWorker(
|
|
|
1197
1290
|
null, // onLaneTerminated — main-thread fallback path
|
|
1198
1291
|
null, // onLaneRespawned — main-thread fallback path
|
|
1199
1292
|
);
|
|
1200
|
-
|
|
1293
|
+
// #631: the fallback runs the engine IN THIS PROCESS. It must be just as
|
|
1294
|
+
// visible/ownable as a forked engine: publish an identity with our own pid
|
|
1295
|
+
// and mark it exited when the run settles. If identity cannot be
|
|
1296
|
+
// published, refuse to start (ownership invariant) instead of running an
|
|
1297
|
+
// invisible engine.
|
|
1298
|
+
const fbStateRoot = wkData.workspaceRoot ?? wkData.cwd;
|
|
1299
|
+
const fbBatchId = wkData.authorizedBatchId ?? null;
|
|
1300
|
+
if (
|
|
1301
|
+
!fbBatchId ||
|
|
1302
|
+
!writeEngineIdentity(fbStateRoot, {
|
|
1303
|
+
batchId: fbBatchId,
|
|
1304
|
+
pid: process.pid,
|
|
1305
|
+
supervisorPid: process.pid,
|
|
1306
|
+
startedAt: Date.now(),
|
|
1307
|
+
})
|
|
1308
|
+
) {
|
|
1309
|
+
batchState.phase = "failed";
|
|
1310
|
+
batchState.endedAt = Date.now();
|
|
1311
|
+
batchState.errors.push(
|
|
1312
|
+
"Engine start refused: fallback engine could not publish its identity (#631)",
|
|
1313
|
+
);
|
|
1314
|
+
updateWidget();
|
|
1315
|
+
onTerminal?.();
|
|
1316
|
+
return null;
|
|
1317
|
+
}
|
|
1318
|
+
batchState.batchId = fbBatchId; // engine adopts it (fresh) / resume verifies the target (resume)
|
|
1319
|
+
fallbackEngineActive = true;
|
|
1320
|
+
startBatchAsync(fallbackFn, batchState, ctx, updateWidget, () => {
|
|
1321
|
+
fallbackEngineActive = false;
|
|
1322
|
+
markEngineExited(fbStateRoot, fbBatchId, { pid: process.pid, exitReason: "fallback-settled" });
|
|
1323
|
+
onTerminal?.();
|
|
1324
|
+
});
|
|
1201
1325
|
return null;
|
|
1202
1326
|
}
|
|
1203
1327
|
|
|
@@ -1274,6 +1398,76 @@ export function startBatchInWorker(
|
|
|
1274
1398
|
});
|
|
1275
1399
|
|
|
1276
1400
|
// Send workerData as first IPC message
|
|
1401
|
+
// #631: publish the engine's identity (pid) BEFORE the engine is told to
|
|
1402
|
+
// start. A replacement supervisor uses it to VERIFY an orphaned engine is
|
|
1403
|
+
// dead before touching an inherited batch (a dead supervisor pid does not
|
|
1404
|
+
// imply a dead engine). The batchId is preallocated by the caller for BOTH
|
|
1405
|
+
// modes (fresh: the engine adopts it; resume: the gated target), so there is
|
|
1406
|
+
// no window in which this engine runs without a current identity record.
|
|
1407
|
+
// Publication is REQUIRED: if it cannot be written, the batch cannot be
|
|
1408
|
+
// owned verifiably — kill the child and fail closed (no fallback).
|
|
1409
|
+
// Exit marking is ATTEMPT-SCOPED (pid-matched): a delayed exit callback from
|
|
1410
|
+
// an old parent cannot mark a newer live engine as exited.
|
|
1411
|
+
const engineStateRoot = wkData.workspaceRoot ?? wkData.cwd;
|
|
1412
|
+
const enginePid = typeof child.pid === "number" ? child.pid : null;
|
|
1413
|
+
const engineIdentityBatchId = wkData.authorizedBatchId ?? null;
|
|
1414
|
+
if (!engineIdentityBatchId || enginePid === null) {
|
|
1415
|
+
try {
|
|
1416
|
+
child.kill();
|
|
1417
|
+
} catch {
|
|
1418
|
+
/* best effort */
|
|
1419
|
+
}
|
|
1420
|
+
const why = !engineIdentityBatchId
|
|
1421
|
+
? "no authorized batchId was preallocated"
|
|
1422
|
+
: "child has no pid";
|
|
1423
|
+
batchState.phase = "failed";
|
|
1424
|
+
batchState.endedAt = Date.now();
|
|
1425
|
+
batchState.errors.push(`Engine start refused: ${why} (#631 ownership invariant)`);
|
|
1426
|
+
safeCtxCallFromCallback(
|
|
1427
|
+
() => ctx.ui.notify(`❌ Engine start refused: ${why}.`, "error"),
|
|
1428
|
+
"spawn.identity.notify",
|
|
1429
|
+
);
|
|
1430
|
+
updateWidget();
|
|
1431
|
+
onTerminal?.();
|
|
1432
|
+
return null;
|
|
1433
|
+
}
|
|
1434
|
+
const published = writeEngineIdentity(engineStateRoot, {
|
|
1435
|
+
batchId: engineIdentityBatchId,
|
|
1436
|
+
pid: enginePid,
|
|
1437
|
+
supervisorPid: process.pid,
|
|
1438
|
+
startedAt: Date.now(),
|
|
1439
|
+
});
|
|
1440
|
+
if (!published) {
|
|
1441
|
+
try {
|
|
1442
|
+
child.kill();
|
|
1443
|
+
} catch {
|
|
1444
|
+
/* best effort */
|
|
1445
|
+
}
|
|
1446
|
+
batchState.phase = "failed";
|
|
1447
|
+
batchState.endedAt = Date.now();
|
|
1448
|
+
batchState.errors.push(
|
|
1449
|
+
`Engine start refused: could not publish engine identity to ${engineStateRoot}/.pi/runtime/${engineIdentityBatchId}/engine.json (#631)`,
|
|
1450
|
+
);
|
|
1451
|
+
safeCtxCallFromCallback(
|
|
1452
|
+
() =>
|
|
1453
|
+
ctx.ui.notify(
|
|
1454
|
+
`❌ Engine start refused: could not write engine identity under .pi/runtime/ — fix the filesystem and retry.`,
|
|
1455
|
+
"error",
|
|
1456
|
+
),
|
|
1457
|
+
"spawn.identity.notify",
|
|
1458
|
+
);
|
|
1459
|
+
updateWidget();
|
|
1460
|
+
onTerminal?.();
|
|
1461
|
+
return null;
|
|
1462
|
+
}
|
|
1463
|
+
child.on("exit", (code: number | null) => {
|
|
1464
|
+
markEngineExited(engineStateRoot, engineIdentityBatchId, {
|
|
1465
|
+
pid: enginePid,
|
|
1466
|
+
exitCode: code,
|
|
1467
|
+
exitReason: "child-exit",
|
|
1468
|
+
});
|
|
1469
|
+
});
|
|
1470
|
+
|
|
1277
1471
|
child.send({ type: "init", data: wkData });
|
|
1278
1472
|
|
|
1279
1473
|
// Terminal settlement guard (R001 §3): ensures onTerminal fires at most once.
|
|
@@ -1288,7 +1482,10 @@ export function startBatchInWorker(
|
|
|
1288
1482
|
child.on("message", (msg: WorkerToMainMessage) => {
|
|
1289
1483
|
switch (msg.type) {
|
|
1290
1484
|
case "notify":
|
|
1291
|
-
ctx
|
|
1485
|
+
// #620: ctx may be stale (session replaced/reload, or headless -p run
|
|
1486
|
+
// finalized while this forked worker still emits IPC). Accessing
|
|
1487
|
+
// ctx.ui throws assertActive; unguarded that crashes Pi. No-op on stale.
|
|
1488
|
+
safeCtxCallFromCallback(() => ctx.ui.notify(msg.msg, msg.level), "ipc.notify");
|
|
1292
1489
|
updateWidget();
|
|
1293
1490
|
break;
|
|
1294
1491
|
|
|
@@ -1338,11 +1535,26 @@ export function startBatchInWorker(
|
|
|
1338
1535
|
batchState.errors.push(`Unhandled engine error${sourceLabel}: ${msg.message}`);
|
|
1339
1536
|
if (stackLine) batchState.errors.push(`Engine stack: ${stackLine}`);
|
|
1340
1537
|
}
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1538
|
+
// #620: persist the failed state FIRST so a dead UI sink can never
|
|
1539
|
+
// prevent the dashboard/resume from seeing the failure. The engine-
|
|
1540
|
+
// worker is dead and can't persist — we must. (Reordered ahead of the
|
|
1541
|
+
// UI notify + supervisor alert, both of which are now stale-safe.)
|
|
1542
|
+
try {
|
|
1543
|
+
saveBatchState(JSON.stringify(batchState, null, 2), wkData.cwd);
|
|
1544
|
+
} catch {
|
|
1545
|
+
/* best effort */
|
|
1546
|
+
}
|
|
1547
|
+
// #620: stale-safe — a stale ctx here must not crash Pi nor block the
|
|
1548
|
+
// supervisor alert below.
|
|
1549
|
+
safeCtxCallFromCallback(
|
|
1550
|
+
() =>
|
|
1551
|
+
ctx.ui.notify(
|
|
1552
|
+
`❌ Engine crashed with unhandled error${sourceLabel}: ${msg.message}\n` +
|
|
1553
|
+
(stackLine ? ` ${stackLine}\n` : "") +
|
|
1554
|
+
` Batch ${batchState.batchId} marked as failed.`,
|
|
1555
|
+
"error",
|
|
1556
|
+
),
|
|
1557
|
+
"ipc.error.notify",
|
|
1346
1558
|
);
|
|
1347
1559
|
// Alert supervisor — this is the PRIMARY notification path for engine
|
|
1348
1560
|
// crashes caught by uncaughtException/unhandledRejection handlers.
|
|
@@ -1374,13 +1586,6 @@ export function startBatchInWorker(
|
|
|
1374
1586
|
: undefined,
|
|
1375
1587
|
},
|
|
1376
1588
|
});
|
|
1377
|
-
// Persist failed state to disk so dashboard/resume see it.
|
|
1378
|
-
// The engine-worker is dead and can't persist — we must do it here.
|
|
1379
|
-
try {
|
|
1380
|
-
saveBatchState(JSON.stringify(batchState, null, 2), wkData.cwd);
|
|
1381
|
-
} catch {
|
|
1382
|
-
/* best effort */
|
|
1383
|
-
}
|
|
1384
1589
|
updateWidget();
|
|
1385
1590
|
break;
|
|
1386
1591
|
}
|
|
@@ -1395,9 +1600,14 @@ export function startBatchInWorker(
|
|
|
1395
1600
|
batchState.endedAt = Date.now();
|
|
1396
1601
|
batchState.errors.push(`Engine process error: ${err.message}`);
|
|
1397
1602
|
}
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1603
|
+
safeCtxCallFromCallback(
|
|
1604
|
+
() =>
|
|
1605
|
+
ctx.ui.notify(
|
|
1606
|
+
`❌ Engine process error: ${err.message}\n` +
|
|
1607
|
+
` Batch ${batchState.batchId} marked as failed.`,
|
|
1608
|
+
"error",
|
|
1609
|
+
),
|
|
1610
|
+
"child.error.notify",
|
|
1401
1611
|
);
|
|
1402
1612
|
updateWidget();
|
|
1403
1613
|
// ── TP-076: Alert supervisor about engine process error ──
|
|
@@ -1444,7 +1654,10 @@ export function startBatchInWorker(
|
|
|
1444
1654
|
batchState.endedAt = Date.now();
|
|
1445
1655
|
batchState.errors.push(`Engine process exited with code ${code}`);
|
|
1446
1656
|
}
|
|
1447
|
-
|
|
1657
|
+
safeCtxCallFromCallback(
|
|
1658
|
+
() => ctx.ui.notify(`❌ Engine process exited unexpectedly (code ${code}).`, "error"),
|
|
1659
|
+
"child.exit.notify",
|
|
1660
|
+
);
|
|
1448
1661
|
updateWidget();
|
|
1449
1662
|
// ── TP-076: Alert supervisor about unexpected engine exit ──
|
|
1450
1663
|
onSupervisorAlert?.({
|
|
@@ -1545,8 +1758,10 @@ export function buildIntegrationExecutor(
|
|
|
1545
1758
|
}
|
|
1546
1759
|
},
|
|
1547
1760
|
deleteBatchState: () => {
|
|
1761
|
+
// #631: only delete the checkpoint of the batch actually integrated — never
|
|
1762
|
+
// an unrelated persisted batch (whose engine may still be alive).
|
|
1548
1763
|
try {
|
|
1549
|
-
|
|
1764
|
+
deleteBatchStateIfOwned(stateRoot ?? repoRoot, context.batchId, context.orchBranch);
|
|
1550
1765
|
} catch {
|
|
1551
1766
|
/* best effort */
|
|
1552
1767
|
}
|
|
@@ -1570,7 +1785,7 @@ export function buildIntegrationExecutor(
|
|
|
1570
1785
|
// as the manual /orch-integrate handler.
|
|
1571
1786
|
if (result.success && result.integratedLocally && context.batchId && opId) {
|
|
1572
1787
|
try {
|
|
1573
|
-
deleteStaleBranches(repoRoot, opId, context.batchId);
|
|
1788
|
+
deleteStaleBranches(repoRoot, opId, context.batchId, stateRoot ?? repoRoot);
|
|
1574
1789
|
dropBatchAutostash(repoRoot, context.batchId);
|
|
1575
1790
|
} catch {
|
|
1576
1791
|
/* best effort — don't fail integration for cleanup errors */
|
|
@@ -1607,7 +1822,21 @@ export function buildIntegrationExecutor(
|
|
|
1607
1822
|
*
|
|
1608
1823
|
* @since TP-043
|
|
1609
1824
|
*/
|
|
1610
|
-
export function buildCiDeps(
|
|
1825
|
+
export function buildCiDeps(
|
|
1826
|
+
repoRoot: string,
|
|
1827
|
+
stateRoot?: string,
|
|
1828
|
+
/**
|
|
1829
|
+
* #631: the IMMUTABLE identity of the batch whose PR this CI lifecycle belongs
|
|
1830
|
+
* to. Post-PR cleanup runs asynchronously (after CI passes and the PR merges)
|
|
1831
|
+
* — by then a NEWER batch may have persisted its checkpoint; deletion is
|
|
1832
|
+
* bound to this identity so it can never erase that one.
|
|
1833
|
+
*/
|
|
1834
|
+
owned?: { batchId: string; orchBranch: string },
|
|
1835
|
+
): CiDeps {
|
|
1836
|
+
// Capture primitives now: a caller mutating its object later cannot change
|
|
1837
|
+
// which batch this lifecycle is allowed to clean up.
|
|
1838
|
+
const ownedBatchId = owned?.batchId;
|
|
1839
|
+
const ownedOrchBranch = owned?.orchBranch;
|
|
1611
1840
|
return {
|
|
1612
1841
|
runCommand: (cmd: string, cmdArgs: string[]) => {
|
|
1613
1842
|
try {
|
|
@@ -1630,7 +1859,13 @@ export function buildCiDeps(repoRoot: string, stateRoot?: string): CiDeps {
|
|
|
1630
1859
|
runGit: (gitArgs: string[]) => runGit(gitArgs, repoRoot),
|
|
1631
1860
|
deleteBatchState: () => {
|
|
1632
1861
|
try {
|
|
1633
|
-
|
|
1862
|
+
if (ownedBatchId !== undefined && ownedOrchBranch !== undefined) {
|
|
1863
|
+
deleteBatchStateIfOwned(stateRoot ?? repoRoot, ownedBatchId, ownedOrchBranch);
|
|
1864
|
+
return;
|
|
1865
|
+
}
|
|
1866
|
+
// No identity supplied (legacy caller): refuse rather than delete blindly.
|
|
1867
|
+
execLog("supervisor", "none", "CI cleanup skipped: no owned batch identity supplied (#631)");
|
|
1868
|
+
return;
|
|
1634
1869
|
} catch {
|
|
1635
1870
|
/* best effort */
|
|
1636
1871
|
}
|
|
@@ -1833,6 +2068,130 @@ export default function (pi: ExtensionAPI) {
|
|
|
1833
2068
|
// Tracked so pause/abort can send control messages to the engine.
|
|
1834
2069
|
let activeWorker: ChildProcess | null = null;
|
|
1835
2070
|
|
|
2071
|
+
// ── #631: inherited-batch ownership evidence ──
|
|
2072
|
+
// Set by supervisor lock takeover when this session imports a batch it did
|
|
2073
|
+
// not start. `activeWorker` answers "is an engine attached to THIS process";
|
|
2074
|
+
// this answers "what do we know about the previous owner" so the active-
|
|
2075
|
+
// phase guards can decide whether an inherited "executing" batch is a
|
|
2076
|
+
// disconnected orphan (resumable) or still being driven elsewhere (refuse).
|
|
2077
|
+
let priorSupervisor: { pid: number; alive: boolean } | null = null;
|
|
2078
|
+
|
|
2079
|
+
/**
|
|
2080
|
+
* Is an engine running IN THIS PROCESS? Covers the forked child (until it has
|
|
2081
|
+
* actually terminated — exitCode/signalCode are set by Node on termination;
|
|
2082
|
+
* `killed` only means a signal was dispatched) and the main-thread fallback.
|
|
2083
|
+
*/
|
|
2084
|
+
function engineAttachedHere(): boolean {
|
|
2085
|
+
if (isFallbackEngineActive()) return true;
|
|
2086
|
+
return (
|
|
2087
|
+
activeWorker !== null && activeWorker.exitCode === null && activeWorker.signalCode === null
|
|
2088
|
+
);
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
/**
|
|
2092
|
+
* #631: THE ownership gate for every recovery mutation (resume / retry /
|
|
2093
|
+
* skip / force-merge / administrative pause). One rule, applied against the
|
|
2094
|
+
* ACTUAL persisted (or reconstructed) target — never against a phase this
|
|
2095
|
+
* session happens to have cached:
|
|
2096
|
+
*
|
|
2097
|
+
* 1. an engine is attached to THIS process (running or still exiting)
|
|
2098
|
+
* → refuse: wait for it to exit (or pause it) — even when the cached
|
|
2099
|
+
* phase already reads paused/failed: teardown is still in flight.
|
|
2100
|
+
* 2. the target's recorded engine is ALIVE elsewhere → refuse (double-drive).
|
|
2101
|
+
* 3. no identity recorded (or unreadable) → refuse: unknown ownership is not
|
|
2102
|
+
* confirmed shutdown; the audited orch_confirm_engine_shutdown path exists.
|
|
2103
|
+
* 4. recorded engine dead/exited (incl. operator-confirmed) → proceed; the
|
|
2104
|
+
* persisted-state eligibility rules take over.
|
|
2105
|
+
*
|
|
2106
|
+
* `force` never enters this decision. Returns a refusal message or null.
|
|
2107
|
+
*/
|
|
2108
|
+
function recoveryOwnershipGate(
|
|
2109
|
+
operation: string,
|
|
2110
|
+
stateRoot: string,
|
|
2111
|
+
target: { batchId: string; phase: string },
|
|
2112
|
+
): string | null {
|
|
2113
|
+
const decision = decideRecoveryOwnership({
|
|
2114
|
+
operation,
|
|
2115
|
+
local: {
|
|
2116
|
+
engineAttached: engineAttachedHere(),
|
|
2117
|
+
phase: orchBatchState.phase,
|
|
2118
|
+
batchId: orchBatchState.batchId,
|
|
2119
|
+
pid: activeWorker?.pid ?? (isFallbackEngineActive() ? process.pid : null),
|
|
2120
|
+
},
|
|
2121
|
+
target,
|
|
2122
|
+
liveness: assessEngineLiveness(stateRoot, target.batchId),
|
|
2123
|
+
priorSupervisor,
|
|
2124
|
+
});
|
|
2125
|
+
execLog("supervisor", target.batchId, `ownership gate: ${operation}`, {
|
|
2126
|
+
proceed: decision.proceed,
|
|
2127
|
+
reason: decision.reason.slice(0, 200),
|
|
2128
|
+
});
|
|
2129
|
+
return decision.proceed ? null : decision.reason;
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
/**
|
|
2133
|
+
* #631: the ONE root every ownership decision AND every state mutation must
|
|
2134
|
+
* use. The engine persists at `workspaceRoot ?? cwd`; a gate that authorizes
|
|
2135
|
+
* the workspace-root batch while the mutation loads the repo-root batch (or
|
|
2136
|
+
* vice versa) protects the wrong target. When the two roots differ and BOTH
|
|
2137
|
+
* hold a persisted batch with different ids, that is a conflict — refuse
|
|
2138
|
+
* rather than pick one.
|
|
2139
|
+
*/
|
|
2140
|
+
function canonicalStateRoot(fallbackCwd: string): string {
|
|
2141
|
+
return execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? fallbackCwd;
|
|
2142
|
+
}
|
|
2143
|
+
function conflictingRootsRefusal(operation: string, fallbackCwd: string): string | null {
|
|
2144
|
+
const canonical = canonicalStateRoot(fallbackCwd);
|
|
2145
|
+
const repo = execCtx?.repoRoot ?? fallbackCwd;
|
|
2146
|
+
if (canonical === repo) return null;
|
|
2147
|
+
let a: PersistedBatchState | null = null;
|
|
2148
|
+
let b: PersistedBatchState | null = null;
|
|
2149
|
+
try {
|
|
2150
|
+
a = loadBatchState(canonical);
|
|
2151
|
+
} catch {
|
|
2152
|
+
/* reported elsewhere */
|
|
2153
|
+
}
|
|
2154
|
+
try {
|
|
2155
|
+
b = loadBatchState(repo);
|
|
2156
|
+
} catch {
|
|
2157
|
+
/* reported elsewhere */
|
|
2158
|
+
}
|
|
2159
|
+
if (a && b && a.batchId !== b.batchId) {
|
|
2160
|
+
return (
|
|
2161
|
+
`❌ Cannot ${operation}: conflicting batch state — workspace root holds ${a.batchId} (${a.phase}) ` +
|
|
2162
|
+
`while the repo root holds ${b.batchId} (${b.phase}). Ownership can only be verified for one target; ` +
|
|
2163
|
+
`remove or archive the stale one (${repo}/.pi/batch-state.json is not the canonical location in workspace mode).`
|
|
2164
|
+
);
|
|
2165
|
+
}
|
|
2166
|
+
return null;
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
/**
|
|
2170
|
+
* #631: resolve the batch a recovery operation would act on, BEFORE
|
|
2171
|
+
* authorizing it. Persisted state first; on force-resume with no state file,
|
|
2172
|
+
* the same deterministic reconstruction the engine performs (so the parent
|
|
2173
|
+
* gates the very batch the child will resume — the child verifies the match).
|
|
2174
|
+
*/
|
|
2175
|
+
function resolveRecoveryTarget(
|
|
2176
|
+
stateRoot: string,
|
|
2177
|
+
allowReconstruction: boolean,
|
|
2178
|
+
): { batchId: string; phase: string } | null {
|
|
2179
|
+
try {
|
|
2180
|
+
const persisted = loadBatchState(stateRoot);
|
|
2181
|
+
if (persisted) return { batchId: persisted.batchId, phase: persisted.phase };
|
|
2182
|
+
} catch {
|
|
2183
|
+
/* the engine reports load errors with full context */
|
|
2184
|
+
}
|
|
2185
|
+
if (!allowReconstruction) return null;
|
|
2186
|
+
try {
|
|
2187
|
+
const r = reconstructBatchStateFromRuntime(stateRoot);
|
|
2188
|
+
if (r.ok) return { batchId: r.batchId, phase: r.state.phase };
|
|
2189
|
+
} catch {
|
|
2190
|
+
/* nothing to reconstruct */
|
|
2191
|
+
}
|
|
2192
|
+
return null;
|
|
2193
|
+
}
|
|
2194
|
+
|
|
1836
2195
|
// ── Supervisor State (TP-041) ────────────────────────────────────
|
|
1837
2196
|
let supervisorState = freshSupervisorState();
|
|
1838
2197
|
let supervisorConfig: SupervisorConfig = { ...DEFAULT_SUPERVISOR_CONFIG };
|
|
@@ -1869,6 +2228,23 @@ export default function (pi: ExtensionAPI) {
|
|
|
1869
2228
|
// individual sends) also protects the completed->triggerSupervisorIntegration
|
|
1870
2229
|
// branch, whose progress/result messages are the same splice hazard.
|
|
1871
2230
|
function runSupervisorBatchEndEpilogue(): void {
|
|
2231
|
+
// #610: re-resolve at DISPATCH time. If this batch was already integrated
|
|
2232
|
+
// (manually, or by auto-integration) — or its orch branch is simply gone —
|
|
2233
|
+
// the "ready for integration" banners are stale and must not be shown.
|
|
2234
|
+
if (orchBatchState.integratedAt) return;
|
|
2235
|
+
if (orchBatchState.phase === "completed" && orchBatchState.orchBranch && execCtx) {
|
|
2236
|
+
const branchExists = runGit(
|
|
2237
|
+
["rev-parse", "--verify", `refs/heads/${orchBatchState.orchBranch}`],
|
|
2238
|
+
execCtx.repoRoot,
|
|
2239
|
+
).ok;
|
|
2240
|
+
if (!branchExists) {
|
|
2241
|
+
process.stderr.write(
|
|
2242
|
+
`[taskplane] batch-end epilogue skipped: orch branch ${orchBatchState.orchBranch} no longer exists (already integrated) (#610)
|
|
2243
|
+
`,
|
|
2244
|
+
);
|
|
2245
|
+
return;
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
1872
2248
|
const mode = orchConfig.orchestrator.integration;
|
|
1873
2249
|
const opId = resolveOperatorId(orchConfig);
|
|
1874
2250
|
const sDeps: SummaryDeps = {
|
|
@@ -1888,8 +2264,23 @@ export default function (pi: ExtensionAPI) {
|
|
|
1888
2264
|
orchBatchState,
|
|
1889
2265
|
mode,
|
|
1890
2266
|
execCtx!.repoRoot,
|
|
1891
|
-
|
|
1892
|
-
|
|
2267
|
+
// #610: record integration in memory on success so any later epilogue
|
|
2268
|
+
// dispatch for this batch short-circuits instead of re-prompting.
|
|
2269
|
+
((mode, context) => {
|
|
2270
|
+
const r = buildIntegrationExecutor(
|
|
2271
|
+
execCtx!.repoRoot,
|
|
2272
|
+
opId,
|
|
2273
|
+
execCtx!.workspaceRoot,
|
|
2274
|
+
)(mode, context);
|
|
2275
|
+
if (r.success && r.integratedLocally && orchBatchState.batchId === context.batchId) {
|
|
2276
|
+
orchBatchState.integratedAt = Date.now();
|
|
2277
|
+
}
|
|
2278
|
+
return r;
|
|
2279
|
+
}) satisfies IntegrationExecutor,
|
|
2280
|
+
buildCiDeps(execCtx!.repoRoot, execCtx!.workspaceRoot, {
|
|
2281
|
+
batchId: orchBatchState.batchId,
|
|
2282
|
+
orchBranch: orchBatchState.orchBranch,
|
|
2283
|
+
}),
|
|
1893
2284
|
sDeps,
|
|
1894
2285
|
);
|
|
1895
2286
|
return;
|
|
@@ -1949,7 +2340,28 @@ export default function (pi: ExtensionAPI) {
|
|
|
1949
2340
|
// has a tool call in flight, eagerly stop batch monitoring (so a heartbeat
|
|
1950
2341
|
// timer send can't splice either) and defer the epilogue to the next settle.
|
|
1951
2342
|
function dispatchBatchEndEpilogue(ctx: ExtensionContext): void {
|
|
1952
|
-
|
|
2343
|
+
// #620: this runs from the engine-worker terminal (onTerminal) callback,
|
|
2344
|
+
// where ctx may be stale. ctx.isIdle() is assertActive-guarded, so an
|
|
2345
|
+
// unguarded call crashes Pi. If the ctx is stale the session is gone — there
|
|
2346
|
+
// is no live UI to render the epilogue — so skip dispatch entirely, but still
|
|
2347
|
+
// stop monitoring and drop any deferred epilogue so no later timer send
|
|
2348
|
+
// fires against the dead session. Non-stale errors are logged, not rethrown
|
|
2349
|
+
// (a throw here is an uncaught IPC-callback exception — the #620 crash class).
|
|
2350
|
+
let idle: boolean;
|
|
2351
|
+
try {
|
|
2352
|
+
idle = ctx.isIdle();
|
|
2353
|
+
} catch (err) {
|
|
2354
|
+
if (!isStaleExtensionCtx(err)) {
|
|
2355
|
+
console.error(
|
|
2356
|
+
`[taskplane] dispatchBatchEndEpilogue ctx.isIdle() threw (non-stale): ${
|
|
2357
|
+
err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
2358
|
+
}`,
|
|
2359
|
+
);
|
|
2360
|
+
}
|
|
2361
|
+
stopBatchMonitoring(supervisorState);
|
|
2362
|
+
noticeGate.invalidate();
|
|
2363
|
+
return;
|
|
2364
|
+
}
|
|
1953
2365
|
if (!idle) stopBatchMonitoring(supervisorState);
|
|
1954
2366
|
noticeGate.runOrDefer(idle, batchGeneration, runSupervisorBatchEndEpilogue);
|
|
1955
2367
|
}
|
|
@@ -2051,13 +2463,24 @@ export default function (pi: ExtensionAPI) {
|
|
|
2051
2463
|
const ctx = orchWidgetCtx;
|
|
2052
2464
|
const prefix = orchConfig.orchestrator.sessionPrefix;
|
|
2053
2465
|
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2466
|
+
// #620: updateOrchWidget is the single choke point for widget refresh and is
|
|
2467
|
+
// called from BOTH synchronous command handlers (ctx fresh) AND long-lived
|
|
2468
|
+
// async engine-worker IPC callbacks (ctx may be stale after session
|
|
2469
|
+
// replacement/reload or a finalized headless -p run). ctx.ui.setWidget
|
|
2470
|
+
// accesses the assertActive-guarded ctx.ui getter, so an unguarded call from
|
|
2471
|
+
// the async path crashes Pi. Guarding here covers every caller at once; for
|
|
2472
|
+
// the sync callers the stale branch simply never triggers.
|
|
2473
|
+
safeCtxCallFromCallback(
|
|
2474
|
+
() =>
|
|
2475
|
+
ctx.ui.setWidget(
|
|
2476
|
+
"task-orchestrator",
|
|
2477
|
+
createOrchWidget(
|
|
2478
|
+
() => orchBatchState,
|
|
2479
|
+
() => latestMonitorState,
|
|
2480
|
+
prefix,
|
|
2481
|
+
),
|
|
2482
|
+
),
|
|
2483
|
+
"widget.setWidget",
|
|
2061
2484
|
);
|
|
2062
2485
|
}
|
|
2063
2486
|
|
|
@@ -2395,8 +2818,55 @@ export default function (pi: ExtensionAPI) {
|
|
|
2395
2818
|
|
|
2396
2819
|
const { repoRoot } = execCtx;
|
|
2397
2820
|
|
|
2821
|
+
// #631: a fresh start deletes stale state and launches a NEW engine. That
|
|
2822
|
+
// must never happen under an engine that is still alive — ours (cached
|
|
2823
|
+
// completed/failed but the child has not terminated yet) or another
|
|
2824
|
+
// process's (inherited terminal phase whose engine is still running).
|
|
2825
|
+
// Ownership is checked against the persisted target, not the cached phase.
|
|
2826
|
+
{
|
|
2827
|
+
const startStateRoot = canonicalStateRoot(repoRoot);
|
|
2828
|
+
const conflict = conflictingRootsRefusal("orch_start", repoRoot);
|
|
2829
|
+
if (conflict) return { message: conflict, error: true };
|
|
2830
|
+
if (engineAttachedHere()) {
|
|
2831
|
+
return {
|
|
2832
|
+
message: `⏳ This session's engine (PID ${activeWorker?.pid ?? process.pid}) for batch ${orchBatchState.batchId} is still shutting down — starting a new batch now would race its teardown. Retry in a moment.`,
|
|
2833
|
+
error: true,
|
|
2834
|
+
};
|
|
2835
|
+
}
|
|
2836
|
+
const existing = resolveRecoveryTarget(startStateRoot, false);
|
|
2837
|
+
if (existing) {
|
|
2838
|
+
const liveness = assessEngineLiveness(startStateRoot, existing.batchId);
|
|
2839
|
+
// Same rule as every recovery mutation: alive or UNKNOWN ownership refuses.
|
|
2840
|
+
// (.DONE files and terminal phases are not shutdown evidence — the engine
|
|
2841
|
+
// persists again after cleanup.) Pre-#631 batches: one-time
|
|
2842
|
+
// orch_confirm_engine_shutdown before the first fresh start.
|
|
2843
|
+
{
|
|
2844
|
+
const decision = decideRecoveryOwnership({
|
|
2845
|
+
operation: "orch_start",
|
|
2846
|
+
local: {
|
|
2847
|
+
engineAttached: false,
|
|
2848
|
+
phase: orchBatchState.phase,
|
|
2849
|
+
batchId: orchBatchState.batchId,
|
|
2850
|
+
pid: null,
|
|
2851
|
+
},
|
|
2852
|
+
target: existing,
|
|
2853
|
+
liveness,
|
|
2854
|
+
priorSupervisor,
|
|
2855
|
+
});
|
|
2856
|
+
execLog("supervisor", existing.batchId, "ownership gate: orch_start", {
|
|
2857
|
+
proceed: decision.proceed,
|
|
2858
|
+
status: liveness.status,
|
|
2859
|
+
pid: liveness.identity?.pid,
|
|
2860
|
+
});
|
|
2861
|
+
if (!decision.proceed) return { message: decision.reason, error: true };
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
|
|
2398
2866
|
// Orphan detection
|
|
2399
|
-
|
|
2867
|
+
// #631: orphan/stale-state handling at the SAME root the gate authorized.
|
|
2868
|
+
const orphanStateRoot = canonicalStateRoot(repoRoot);
|
|
2869
|
+
const orphanResult = detectOrphanSessions(orchConfig.orchestrator.sessionPrefix, orphanStateRoot);
|
|
2400
2870
|
|
|
2401
2871
|
switch (orphanResult.recommendedAction) {
|
|
2402
2872
|
case "resume": {
|
|
@@ -2405,7 +2875,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2405
2875
|
const hasOrphans = orphanResult.orphanSessions.length > 0;
|
|
2406
2876
|
if (!hasOrphans && !resumablePhases.includes(phase)) {
|
|
2407
2877
|
try {
|
|
2408
|
-
deleteBatchState(
|
|
2878
|
+
deleteBatchState(orphanStateRoot);
|
|
2409
2879
|
} catch {
|
|
2410
2880
|
/* best effort */
|
|
2411
2881
|
}
|
|
@@ -2421,7 +2891,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2421
2891
|
return { message: orphanResult.userMessage, error: true };
|
|
2422
2892
|
case "cleanup-stale":
|
|
2423
2893
|
try {
|
|
2424
|
-
deleteBatchState(
|
|
2894
|
+
deleteBatchState(orphanStateRoot);
|
|
2425
2895
|
} catch {
|
|
2426
2896
|
/* best effort */
|
|
2427
2897
|
}
|
|
@@ -2495,6 +2965,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2495
2965
|
|
|
2496
2966
|
// Reset batch state for new execution
|
|
2497
2967
|
orchBatchState = freshOrchBatchState();
|
|
2968
|
+
priorSupervisor = null; // #631: this session now owns the engine it is about to fork
|
|
2498
2969
|
latestMonitorState = null;
|
|
2499
2970
|
|
|
2500
2971
|
// #621: a new batch supersedes any epilogue still deferred from the
|
|
@@ -2508,12 +2979,18 @@ export default function (pi: ExtensionAPI) {
|
|
|
2508
2979
|
orchBatchState.startedAt = Date.now();
|
|
2509
2980
|
updateOrchWidget();
|
|
2510
2981
|
|
|
2982
|
+
// #631: preallocate the batchId so the engine identity is published BEFORE
|
|
2983
|
+
// the engine starts (the engine adopts it instead of generating its own).
|
|
2984
|
+
const authorizedBatchId = generateBatchId();
|
|
2985
|
+
orchBatchState.batchId = authorizedBatchId;
|
|
2986
|
+
|
|
2511
2987
|
// Non-blocking engine launch in worker thread (TP-071)
|
|
2512
2988
|
activeWorker = startBatchInWorker(
|
|
2513
2989
|
{
|
|
2514
2990
|
engineWorker: true,
|
|
2515
2991
|
mode: "execute",
|
|
2516
2992
|
args: trimmedTarget,
|
|
2993
|
+
authorizedBatchId,
|
|
2517
2994
|
orchConfig,
|
|
2518
2995
|
runnerConfig,
|
|
2519
2996
|
cwd: repoRoot,
|
|
@@ -2561,7 +3038,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
2561
3038
|
);
|
|
2562
3039
|
return;
|
|
2563
3040
|
}
|
|
2564
|
-
pi.sendUserMessage
|
|
3041
|
+
// #620: stale-safe — pi.sendUserMessage is assertActive-guarded on
|
|
3042
|
+
// ExtensionAPI; a stale ctx from this async worker-IPC alert callback
|
|
3043
|
+
// would otherwise crash Pi. #597 safeSendMessageFromTimer covers
|
|
3044
|
+
// sendMessage only, so use the general callback guard here.
|
|
3045
|
+
safeCtxCallFromCallback(
|
|
3046
|
+
() =>
|
|
3047
|
+
pi.sendUserMessage(alert.summary, {
|
|
3048
|
+
// #review-boundary: spiral/order escalations are urgent — steer
|
|
3049
|
+
// (interrupt current turn) so the supervisor adjudicates now;
|
|
3050
|
+
// routine alerts stay followUp (queue to next turn boundary).
|
|
3051
|
+
deliverAs: alert.category === "review-intervention-needed" ? "steer" : "followUp",
|
|
3052
|
+
}),
|
|
3053
|
+
"alert.sendUserMessage",
|
|
3054
|
+
);
|
|
2565
3055
|
},
|
|
2566
3056
|
// TP-187 (#538): Lane-terminated handler.
|
|
2567
3057
|
(info) => {
|
|
@@ -2823,7 +3313,59 @@ export default function (pi: ExtensionAPI) {
|
|
|
2823
3313
|
if (orchBatchState.phase === "paused" || orchBatchState.pauseSignal.paused) {
|
|
2824
3314
|
return ORCH_MESSAGES.pauseAlreadyPaused(orchBatchState.batchId);
|
|
2825
3315
|
}
|
|
3316
|
+
|
|
3317
|
+
// #631: no engine attached to THIS process (phase inherited at takeover).
|
|
3318
|
+
// A pauseSignal here is inert — nothing runs in-process to honor it. Decide
|
|
3319
|
+
// from verified engine liveness:
|
|
3320
|
+
// - orphan engine confirmed gone → ADMINISTRATIVE pause: persist
|
|
3321
|
+
// phase=paused on disk (the non-destructive stop the operator otherwise
|
|
3322
|
+
// had to hand-edit). Surviving worker processes, if any, are reconciled
|
|
3323
|
+
// by orch_resume (registry pid liveness), not by this call.
|
|
3324
|
+
// - engine alive elsewhere → we cannot signal it; report, do not lie.
|
|
3325
|
+
if (!engineAttachedHere()) {
|
|
3326
|
+
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? process.cwd();
|
|
3327
|
+
const refusal = recoveryOwnershipGate("orch_pause (administrative)", stateRoot, {
|
|
3328
|
+
batchId: orchBatchState.batchId,
|
|
3329
|
+
phase: orchBatchState.phase,
|
|
3330
|
+
});
|
|
3331
|
+
if (refusal) {
|
|
3332
|
+
return (
|
|
3333
|
+
`⚠️ Batch ${orchBatchState.batchId} is "${orchBatchState.phase}" but no engine is attached to this session — ` +
|
|
3334
|
+
`a pause signal here would be inert.\n${refusal}`
|
|
3335
|
+
);
|
|
3336
|
+
}
|
|
3337
|
+
const decision = { reason: "recorded engine verified gone" };
|
|
3338
|
+
try {
|
|
3339
|
+
const persisted = loadBatchState(stateRoot);
|
|
3340
|
+
if (!persisted) return "❌ No persisted batch state found to pause.";
|
|
3341
|
+
if (persisted.batchId !== orchBatchState.batchId) {
|
|
3342
|
+
return `❌ Persisted batch (${persisted.batchId}) does not match the inherited batch (${orchBatchState.batchId}); refusing to pause.`;
|
|
3343
|
+
}
|
|
3344
|
+
const prevPhase = persisted.phase;
|
|
3345
|
+
persisted.phase = "paused";
|
|
3346
|
+
persisted.updatedAt = Date.now();
|
|
3347
|
+
persisted.errors.push(
|
|
3348
|
+
`Administrative pause by replacement supervisor (PID ${process.pid}) — inherited phase "${prevPhase}" with no live engine (${decision.reason.slice(0, 160)})`,
|
|
3349
|
+
);
|
|
3350
|
+
saveBatchState(JSON.stringify(persisted, null, 2), stateRoot);
|
|
3351
|
+
orchBatchState.phase = "paused";
|
|
3352
|
+
orchBatchState.pauseSignal.paused = true;
|
|
3353
|
+
orchBatchState.pauseSignal.cause = "operator";
|
|
3354
|
+
updateOrchWidget();
|
|
3355
|
+
return (
|
|
3356
|
+
`⏸️ Batch ${orchBatchState.batchId} administratively paused (was "${prevPhase}"; ${decision.reason}).
|
|
3357
|
+
` +
|
|
3358
|
+
` No engine was running to honor a live pause; state is now resumable on disk. ` +
|
|
3359
|
+
`Any worker processes that outlived the engine are reconciled by orch_resume(force=true) ` +
|
|
3360
|
+
`(registry pid liveness → re-execute in the existing worktree).`
|
|
3361
|
+
);
|
|
3362
|
+
} catch (err) {
|
|
3363
|
+
return `❌ Administrative pause failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
3364
|
+
}
|
|
3365
|
+
}
|
|
3366
|
+
|
|
2826
3367
|
orchBatchState.pauseSignal.paused = true;
|
|
3368
|
+
orchBatchState.pauseSignal.cause = "operator"; // in-process (fallback) engine reads this directly
|
|
2827
3369
|
// TP-071: Forward pause to engine process (its pauseSignal is separate)
|
|
2828
3370
|
activeWorker?.send({ type: "pause" });
|
|
2829
3371
|
updateOrchWidget();
|
|
@@ -2846,21 +3388,29 @@ export default function (pi: ExtensionAPI) {
|
|
|
2846
3388
|
};
|
|
2847
3389
|
}
|
|
2848
3390
|
|
|
2849
|
-
//
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
) {
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
3391
|
+
// #631: resolve the ACTUAL target (persisted, or reconstructed on force with
|
|
3392
|
+
// no state file) and run the single ownership gate against it — never
|
|
3393
|
+
// against a cached phase. The resolved batchId is also the id the new
|
|
3394
|
+
// engine is authorized for (identity published before it starts).
|
|
3395
|
+
const resumeStateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
3396
|
+
const resumeTarget = resolveRecoveryTarget(resumeStateRoot, force);
|
|
3397
|
+
if (!resumeTarget) {
|
|
3398
|
+
if (engineAttachedHere()) {
|
|
3399
|
+
return {
|
|
3400
|
+
message: `❌ Cannot orch_resume: this session's engine for batch ${orchBatchState.batchId} is still running.`,
|
|
3401
|
+
error: true,
|
|
3402
|
+
};
|
|
3403
|
+
}
|
|
3404
|
+
// No target to gate: let the engine produce the canonical "no state" error.
|
|
3405
|
+
} else {
|
|
3406
|
+
const refusal = recoveryOwnershipGate("orch_resume", resumeStateRoot, resumeTarget);
|
|
3407
|
+
if (refusal) return { message: refusal, error: true };
|
|
2860
3408
|
}
|
|
3409
|
+
const resumeTargetBatchId = resumeTarget?.batchId ?? null;
|
|
2861
3410
|
|
|
2862
3411
|
// Reset batch state for resume
|
|
2863
3412
|
orchBatchState = freshOrchBatchState();
|
|
3413
|
+
priorSupervisor = null; // #631: this session now owns the engine it is about to fork
|
|
2864
3414
|
latestMonitorState = null;
|
|
2865
3415
|
|
|
2866
3416
|
// #621: a resume supersedes any epilogue still deferred from the previous
|
|
@@ -2884,6 +3434,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2884
3434
|
engineWorker: true,
|
|
2885
3435
|
mode: "resume",
|
|
2886
3436
|
args: "",
|
|
3437
|
+
authorizedBatchId: resumeTargetBatchId ?? undefined,
|
|
2887
3438
|
orchConfig,
|
|
2888
3439
|
runnerConfig,
|
|
2889
3440
|
cwd: execCtx!.repoRoot,
|
|
@@ -2916,7 +3467,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
2916
3467
|
);
|
|
2917
3468
|
return;
|
|
2918
3469
|
}
|
|
2919
|
-
pi.sendUserMessage
|
|
3470
|
+
// #620: stale-safe — pi.sendUserMessage is assertActive-guarded on
|
|
3471
|
+
// ExtensionAPI; a stale ctx from this async worker-IPC alert callback
|
|
3472
|
+
// would otherwise crash Pi. #597 safeSendMessageFromTimer covers
|
|
3473
|
+
// sendMessage only, so use the general callback guard here.
|
|
3474
|
+
safeCtxCallFromCallback(
|
|
3475
|
+
() =>
|
|
3476
|
+
pi.sendUserMessage(alert.summary, {
|
|
3477
|
+
// #review-boundary: spiral/order escalations are urgent — steer
|
|
3478
|
+
// (interrupt current turn) so the supervisor adjudicates now;
|
|
3479
|
+
// routine alerts stay followUp (queue to next turn boundary).
|
|
3480
|
+
deliverAs: alert.category === "review-intervention-needed" ? "steer" : "followUp",
|
|
3481
|
+
}),
|
|
3482
|
+
"alert.sendUserMessage",
|
|
3483
|
+
);
|
|
2920
3484
|
},
|
|
2921
3485
|
// TP-187 (#538): Lane-terminated handler.
|
|
2922
3486
|
(info) => {
|
|
@@ -2970,7 +3534,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
2970
3534
|
const mode: AbortMode = hard ? "hard" : "graceful";
|
|
2971
3535
|
const prefix = orchConfig.orchestrator.sessionPrefix;
|
|
2972
3536
|
|
|
2973
|
-
|
|
3537
|
+
// #631: state is read, persisted and deleted at the CANONICAL root — the same
|
|
3538
|
+
// root the ownership gate below authorizes against.
|
|
3539
|
+
const stateRoot = canonicalStateRoot(ctx.cwd);
|
|
3540
|
+
{
|
|
3541
|
+
const conflict = conflictingRootsRefusal("orch_abort", ctx.cwd);
|
|
3542
|
+
if (conflict) return conflict;
|
|
3543
|
+
}
|
|
2974
3544
|
const messages: string[] = [`🛑 Abort requested (${mode} mode, prefix: ${prefix})...`];
|
|
2975
3545
|
|
|
2976
3546
|
// Step 1: Write abort signal file
|
|
@@ -2992,18 +3562,87 @@ export default function (pi: ExtensionAPI) {
|
|
|
2992
3562
|
// Step 2: Set pause signal and forward to worker
|
|
2993
3563
|
if (orchBatchState.pauseSignal) {
|
|
2994
3564
|
orchBatchState.pauseSignal.paused = true;
|
|
3565
|
+
orchBatchState.pauseSignal.cause = "abort";
|
|
2995
3566
|
messages.push(" ✓ Pause signal set on in-memory batch state");
|
|
2996
3567
|
}
|
|
2997
|
-
//
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3568
|
+
// ── #631: ownership + verified shutdown BEFORE any destructive step ──
|
|
3569
|
+
// Abort persists `stopped` and deletes batch state. That may only happen
|
|
3570
|
+
// once no engine can still be writing that state:
|
|
3571
|
+
// - engine attached HERE → cooperatively pause, then VERIFY it has exited
|
|
3572
|
+
// (graceful: within the grace period, then escalate; hard: kill now);
|
|
3573
|
+
// the main-thread fallback must have settled. Unverified → refuse cleanup.
|
|
3574
|
+
// - engine elsewhere → same rule as every recovery mutation: alive or
|
|
3575
|
+
// unknown (no identity) → refuse; verified dead/exited → proceed.
|
|
3576
|
+
const ownershipRoot = stateRoot;
|
|
3577
|
+
if (!engineAttachedHere()) {
|
|
3578
|
+
const target = resolveRecoveryTarget(ownershipRoot, false);
|
|
3579
|
+
if (target) {
|
|
3580
|
+
const decision = decideRecoveryOwnership({
|
|
3581
|
+
operation: "orch_abort",
|
|
3582
|
+
local: {
|
|
3583
|
+
engineAttached: false,
|
|
3584
|
+
phase: orchBatchState.phase,
|
|
3585
|
+
batchId: orchBatchState.batchId,
|
|
3586
|
+
pid: null,
|
|
3587
|
+
},
|
|
3588
|
+
target,
|
|
3589
|
+
liveness: assessEngineLiveness(ownershipRoot, target.batchId),
|
|
3590
|
+
priorSupervisor,
|
|
3591
|
+
});
|
|
3592
|
+
if (!decision.proceed) {
|
|
3593
|
+
try {
|
|
3594
|
+
unlinkSync(abortSignalFile);
|
|
3595
|
+
} catch {}
|
|
3596
|
+
return `${decision.reason}\n (abort here cannot stop that engine; it would only delete state underneath it)`;
|
|
3597
|
+
}
|
|
3006
3598
|
}
|
|
3599
|
+
} else if (activeWorker) {
|
|
3600
|
+
// Forked engine in this session.
|
|
3601
|
+
const child = activeWorker;
|
|
3602
|
+
child.send({ type: "pause" });
|
|
3603
|
+
const graceMs = Math.max(0, orchConfig.failure.abort_grace_period * 1000);
|
|
3604
|
+
let exited = false;
|
|
3605
|
+
if (!hard) {
|
|
3606
|
+
messages.push(
|
|
3607
|
+
` ✓ Pause signal forwarded to engine process — waiting up to ${Math.round(graceMs / 1000)}s for it to checkpoint and exit`,
|
|
3608
|
+
);
|
|
3609
|
+
exited = await waitForChildExit(child, graceMs);
|
|
3610
|
+
}
|
|
3611
|
+
if (!exited) {
|
|
3612
|
+
child.kill();
|
|
3613
|
+
exited = await waitForChildExit(child, 5_000);
|
|
3614
|
+
}
|
|
3615
|
+
if (!exited) {
|
|
3616
|
+
try {
|
|
3617
|
+
child.kill("SIGKILL");
|
|
3618
|
+
} catch {}
|
|
3619
|
+
exited = await waitForChildExit(child, 3_000);
|
|
3620
|
+
}
|
|
3621
|
+
if (!exited) {
|
|
3622
|
+
return (
|
|
3623
|
+
`❌ Abort: engine process (PID ${child.pid}) is still alive after pause${hard ? "" : ` (${Math.round(graceMs / 1000)}s grace)`}, SIGTERM and SIGKILL. ` +
|
|
3624
|
+
`Refusing to persist/delete batch state underneath a live engine. Terminate it manually and re-run orch_abort.`
|
|
3625
|
+
);
|
|
3626
|
+
}
|
|
3627
|
+
activeWorker = null;
|
|
3628
|
+
messages.push(
|
|
3629
|
+
` ✓ Engine process exit verified (PID ${child.pid}${hard ? ", hard abort" : ""})`,
|
|
3630
|
+
);
|
|
3631
|
+
} else {
|
|
3632
|
+
// Main-thread fallback engine: it shares orchBatchState, so the pause
|
|
3633
|
+
// signal above reaches it directly; wait for it to settle.
|
|
3634
|
+
const graceMs = Math.max(2_000, orchConfig.failure.abort_grace_period * 1000);
|
|
3635
|
+
const deadline = Date.now() + graceMs;
|
|
3636
|
+
while (isFallbackEngineActive() && Date.now() < deadline) {
|
|
3637
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
3638
|
+
}
|
|
3639
|
+
if (isFallbackEngineActive()) {
|
|
3640
|
+
return (
|
|
3641
|
+
`❌ Abort: the in-process (fallback) engine has not settled within ${Math.round(graceMs / 1000)}s of the pause signal. ` +
|
|
3642
|
+
`Refusing to persist/delete batch state underneath it. Wait for it to pause, then re-run orch_abort.`
|
|
3643
|
+
);
|
|
3644
|
+
}
|
|
3645
|
+
messages.push(" ✓ In-process engine settled");
|
|
3007
3646
|
}
|
|
3008
3647
|
|
|
3009
3648
|
const hasActiveBatch =
|
|
@@ -3126,6 +3765,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3126
3765
|
const pausablePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
3127
3766
|
if (pausablePhases.has(orchBatchState.phase)) {
|
|
3128
3767
|
orchBatchState.pauseSignal.paused = true;
|
|
3768
|
+
orchBatchState.pauseSignal.cause = "operator";
|
|
3129
3769
|
activeWorker?.send({ type: "pause" });
|
|
3130
3770
|
messages.push(` ✓ Wave paused (batch ${orchBatchState.batchId})`);
|
|
3131
3771
|
} else {
|
|
@@ -3198,12 +3838,6 @@ export default function (pi: ExtensionAPI) {
|
|
|
3198
3838
|
* The engine picks up the state change on its next poll cycle.
|
|
3199
3839
|
*/
|
|
3200
3840
|
function doOrchRetryTask(taskId: string, ctx: ExtensionContext): string {
|
|
3201
|
-
// TP-077 R001-1: Reject while engine is actively running (no IPC retry path)
|
|
3202
|
-
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
3203
|
-
if (activePhases.has(orchBatchState.phase)) {
|
|
3204
|
-
return `❌ Cannot retry task while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
3205
|
-
}
|
|
3206
|
-
|
|
3207
3841
|
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
3208
3842
|
|
|
3209
3843
|
// Load persisted state
|
|
@@ -3218,6 +3852,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
3218
3852
|
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
3219
3853
|
}
|
|
3220
3854
|
|
|
3855
|
+
// #631: single ownership gate against the PERSISTED target (TP-077 R001-1's
|
|
3856
|
+
// "reject while the engine is running" is case 1 of the gate).
|
|
3857
|
+
{
|
|
3858
|
+
const refusal = recoveryOwnershipGate("orch_retry_task", stateRoot, {
|
|
3859
|
+
batchId: state.batchId,
|
|
3860
|
+
phase: state.phase,
|
|
3861
|
+
});
|
|
3862
|
+
if (refusal) return refusal;
|
|
3863
|
+
}
|
|
3864
|
+
|
|
3221
3865
|
// Find the task
|
|
3222
3866
|
const taskRecord = state.tasks.find((t) => t.taskId === taskId);
|
|
3223
3867
|
if (!taskRecord) {
|
|
@@ -3225,12 +3869,24 @@ export default function (pi: ExtensionAPI) {
|
|
|
3225
3869
|
return `❌ Task "${taskId}" not found in batch ${state.batchId}.\nKnown tasks: ${knownIds || "(none)"}`;
|
|
3226
3870
|
}
|
|
3227
3871
|
|
|
3228
|
-
// Validate:
|
|
3229
|
-
|
|
3230
|
-
|
|
3872
|
+
// Validate: failed, stalled or SKIPPED tasks can be retried. Skipped is
|
|
3873
|
+
// included because a runtime defect (pause → skipped, penster
|
|
3874
|
+
// 20260906T194514) left tasks skipped that never ran; the documented
|
|
3875
|
+
// recovery (retry → resume) must be able to start from that state.
|
|
3876
|
+
if (
|
|
3877
|
+
taskRecord.status !== "failed" &&
|
|
3878
|
+
taskRecord.status !== "stalled" &&
|
|
3879
|
+
taskRecord.status !== "skipped"
|
|
3880
|
+
) {
|
|
3881
|
+
return `❌ Cannot retry task "${taskId}" — current status is "${taskRecord.status}". Only failed, stalled or skipped tasks can be retried.`;
|
|
3231
3882
|
}
|
|
3232
3883
|
|
|
3233
3884
|
const prevStatus = taskRecord.status;
|
|
3885
|
+
// Preserve recovery provenance: a saved partial-progress branch is the only
|
|
3886
|
+
// record of work whose worktree may already be gone. Report it instead of
|
|
3887
|
+
// silently clearing it.
|
|
3888
|
+
const preservedBranch = taskRecord.partialProgressBranch;
|
|
3889
|
+
const preservedCommits = taskRecord.partialProgressCommits;
|
|
3234
3890
|
|
|
3235
3891
|
// Reset task to pending
|
|
3236
3892
|
taskRecord.status = "pending";
|
|
@@ -3239,12 +3895,21 @@ export default function (pi: ExtensionAPI) {
|
|
|
3239
3895
|
taskRecord.startedAt = null;
|
|
3240
3896
|
taskRecord.endedAt = null;
|
|
3241
3897
|
taskRecord.exitDiagnostic = undefined;
|
|
3242
|
-
|
|
3243
|
-
|
|
3898
|
+
// Keep partialProgressBranch/Commits: they are recovery PROVENANCE (a saved
|
|
3899
|
+
// ref may be the only copy of the work); the engine overwrites them when the
|
|
3900
|
+
// re-executed task produces new partial progress.
|
|
3901
|
+
|
|
3902
|
+
// #629: on the v2 runtime the SEGMENT record is authoritative — resume's
|
|
3903
|
+
// reconstructSegmentFrontier() re-derives task status from segments, so a
|
|
3904
|
+
// task-only reset is silently undone and the wave is counted as done.
|
|
3905
|
+
// Reset the failed/stalled segments too (worktree identity preserved).
|
|
3906
|
+
const segmentReset = resetTaskSegmentsForRetry(state, taskId);
|
|
3244
3907
|
|
|
3245
3908
|
// Adjust counters: only decrement failedTasks if the task was in a failure state
|
|
3246
3909
|
if (prevStatus === "failed" || prevStatus === "stalled") {
|
|
3247
3910
|
state.failedTasks = Math.max(0, state.failedTasks - 1);
|
|
3911
|
+
} else if (prevStatus === "skipped") {
|
|
3912
|
+
state.skippedTasks = Math.max(0, (state.skippedTasks ?? 0) - 1);
|
|
3248
3913
|
}
|
|
3249
3914
|
|
|
3250
3915
|
// Recompute blocked dependents — the retried task is no longer a failure,
|
|
@@ -3259,6 +3924,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3259
3924
|
const newBlocked = computeTransitiveDependents(
|
|
3260
3925
|
remainingFailures,
|
|
3261
3926
|
orchBatchState.dependencyGraph,
|
|
3927
|
+
batchTaskScope(state.wavePlan),
|
|
3262
3928
|
);
|
|
3263
3929
|
state.blockedTaskIds = [...newBlocked].sort();
|
|
3264
3930
|
state.blockedTasks = newBlocked.size;
|
|
@@ -3272,6 +3938,19 @@ export default function (pi: ExtensionAPI) {
|
|
|
3272
3938
|
if (state.phase === "failed") {
|
|
3273
3939
|
state.phase = "stopped";
|
|
3274
3940
|
}
|
|
3941
|
+
// A "completed" batch that wrongly skipped a task (the pause→skipped defect)
|
|
3942
|
+
// must be re-openable: retrying a task in a completed batch moves the batch
|
|
3943
|
+
// to "stopped" (resumable with force). The ownership gate above already
|
|
3944
|
+
// verified no engine is driving it; refuse if the batch was integrated.
|
|
3945
|
+
let reopenedCompleted = false;
|
|
3946
|
+
if (state.phase === "completed") {
|
|
3947
|
+
if (orchBatchState.batchId === state.batchId && orchBatchState.integratedAt) {
|
|
3948
|
+
return `❌ Cannot retry "${taskId}": batch ${state.batchId} has already been integrated. Start a new batch for follow-up work.`;
|
|
3949
|
+
}
|
|
3950
|
+
state.phase = "stopped";
|
|
3951
|
+
state.endedAt = null;
|
|
3952
|
+
reopenedCompleted = true;
|
|
3953
|
+
}
|
|
3275
3954
|
|
|
3276
3955
|
// Update timestamp
|
|
3277
3956
|
state.updatedAt = Date.now();
|
|
@@ -3299,9 +3978,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
3299
3978
|
state.phase === "stopped"
|
|
3300
3979
|
? "Use orch_resume(force=true) to re-execute the batch."
|
|
3301
3980
|
: "Use orch_resume() to re-execute the batch.";
|
|
3981
|
+
const provenanceNote = preservedBranch
|
|
3982
|
+
? ` Preserved progress: branch ${preservedBranch}${preservedCommits ? ` (${preservedCommits} commit(s))` : ""} — the worktree may be recreated from the base; inspect/cherry-pick that branch if the work must carry forward.\n`
|
|
3983
|
+
: "";
|
|
3984
|
+
const reopenNote = reopenedCompleted
|
|
3985
|
+
? ` Batch was "completed" — reopened as "stopped" (task ${taskId} had been ${prevStatus}).\n`
|
|
3986
|
+
: "";
|
|
3302
3987
|
return (
|
|
3303
3988
|
`✅ Task "${taskId}" reset to pending for re-execution.\n` +
|
|
3304
3989
|
` Previous status: ${prevStatus}\n` +
|
|
3990
|
+
(segmentReset.resetSegmentIds.length > 0
|
|
3991
|
+
? ` Segments reset: ${segmentReset.resetSegmentIds.join(", ")}${segmentReset.preservedSegmentIds.length > 0 ? ` (preserved: ${segmentReset.preservedSegmentIds.join(", ")})` : ""}\n`
|
|
3992
|
+
: "") +
|
|
3993
|
+
reopenNote +
|
|
3994
|
+
provenanceNote +
|
|
3305
3995
|
` Batch phase: ${state.phase} | Failed: ${state.failedTasks}/${state.totalTasks}\n` +
|
|
3306
3996
|
` ${resumeHint}`
|
|
3307
3997
|
);
|
|
@@ -3314,12 +4004,6 @@ export default function (pi: ExtensionAPI) {
|
|
|
3314
4004
|
* The engine picks up the state change on its next poll cycle.
|
|
3315
4005
|
*/
|
|
3316
4006
|
function doOrchSkipTask(taskId: string, ctx: ExtensionContext): string {
|
|
3317
|
-
// TP-077 R001-1: Reject while engine is actively running (no IPC skip path)
|
|
3318
|
-
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
3319
|
-
if (activePhases.has(orchBatchState.phase)) {
|
|
3320
|
-
return `❌ Cannot skip task while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
3321
|
-
}
|
|
3322
|
-
|
|
3323
4007
|
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
3324
4008
|
|
|
3325
4009
|
// Load persisted state
|
|
@@ -3334,6 +4018,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
3334
4018
|
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
3335
4019
|
}
|
|
3336
4020
|
|
|
4021
|
+
// #631: single ownership gate against the PERSISTED target (TP-077 R001-1's
|
|
4022
|
+
// "reject while the engine is running" is case 1 of the gate).
|
|
4023
|
+
{
|
|
4024
|
+
const refusal = recoveryOwnershipGate("orch_skip_task", stateRoot, {
|
|
4025
|
+
batchId: state.batchId,
|
|
4026
|
+
phase: state.phase,
|
|
4027
|
+
});
|
|
4028
|
+
if (refusal) return refusal;
|
|
4029
|
+
}
|
|
4030
|
+
|
|
3337
4031
|
// Find the task
|
|
3338
4032
|
const taskRecord = state.tasks.find((t) => t.taskId === taskId);
|
|
3339
4033
|
if (!taskRecord) {
|
|
@@ -3357,6 +4051,8 @@ export default function (pi: ExtensionAPI) {
|
|
|
3357
4051
|
taskRecord.status = "skipped";
|
|
3358
4052
|
taskRecord.exitReason = "Skipped by supervisor";
|
|
3359
4053
|
taskRecord.endedAt = Date.now();
|
|
4054
|
+
// #629: keep segment records in agreement (segment authority on v2).
|
|
4055
|
+
markTaskSegmentsSkipped(state, taskId, taskRecord.endedAt);
|
|
3360
4056
|
|
|
3361
4057
|
// Adjust counters
|
|
3362
4058
|
state.skippedTasks = (state.skippedTasks ?? 0) + 1;
|
|
@@ -3382,6 +4078,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3382
4078
|
const newBlocked = computeTransitiveDependents(
|
|
3383
4079
|
remainingFailures,
|
|
3384
4080
|
orchBatchState.dependencyGraph,
|
|
4081
|
+
batchTaskScope(state.wavePlan),
|
|
3385
4082
|
);
|
|
3386
4083
|
|
|
3387
4084
|
// Find tasks that were blocked but are now unblocked
|
|
@@ -3464,12 +4161,6 @@ export default function (pi: ExtensionAPI) {
|
|
|
3464
4161
|
skipFailed: boolean,
|
|
3465
4162
|
ctx: ExtensionContext,
|
|
3466
4163
|
): string {
|
|
3467
|
-
// Reject while engine is actively running
|
|
3468
|
-
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
3469
|
-
if (activePhases.has(orchBatchState.phase)) {
|
|
3470
|
-
return `❌ Cannot force merge while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
3471
|
-
}
|
|
3472
|
-
|
|
3473
4164
|
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
3474
4165
|
|
|
3475
4166
|
// Load persisted state
|
|
@@ -3484,6 +4175,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
3484
4175
|
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
3485
4176
|
}
|
|
3486
4177
|
|
|
4178
|
+
// #631: single ownership gate against the PERSISTED target (TP-077 R001-1's
|
|
4179
|
+
// "reject while the engine is running" is case 1 of the gate).
|
|
4180
|
+
{
|
|
4181
|
+
const refusal = recoveryOwnershipGate("orch_force_merge", stateRoot, {
|
|
4182
|
+
batchId: state.batchId,
|
|
4183
|
+
phase: state.phase,
|
|
4184
|
+
});
|
|
4185
|
+
if (refusal) return refusal;
|
|
4186
|
+
}
|
|
4187
|
+
|
|
3487
4188
|
// Force-merge is a recovery action for non-running failed/paused batches.
|
|
3488
4189
|
const resumablePhases = new Set(["paused", "stopped", "failed"]);
|
|
3489
4190
|
if (!resumablePhases.has(state.phase)) {
|
|
@@ -3594,6 +4295,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3594
4295
|
const newBlocked = computeTransitiveDependents(
|
|
3595
4296
|
remainingFailures,
|
|
3596
4297
|
orchBatchState.dependencyGraph,
|
|
4298
|
+
batchTaskScope(state.wavePlan),
|
|
3597
4299
|
);
|
|
3598
4300
|
state.blockedTaskIds = [...newBlocked].sort();
|
|
3599
4301
|
state.blockedTasks = newBlocked.size;
|
|
@@ -3688,8 +4390,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
3688
4390
|
// Resolve integration context
|
|
3689
4391
|
const { repoRoot } = execCtx!;
|
|
3690
4392
|
const stateRoot = execCtx!.workspaceRoot;
|
|
4393
|
+
// #631: batch state is resolved at the CANONICAL root (workspace root in
|
|
4394
|
+
// workspace mode) — the same root the ownership gate authorizes against.
|
|
4395
|
+
{
|
|
4396
|
+
const conflict = conflictingRootsRefusal("orch_integrate", repoRoot);
|
|
4397
|
+
if (conflict) return { message: conflict, error: true };
|
|
4398
|
+
}
|
|
3691
4399
|
const resolution = resolveIntegrationContext(parsed, {
|
|
3692
|
-
loadBatchState: () => loadBatchState(repoRoot),
|
|
4400
|
+
loadBatchState: () => loadBatchState(stateRoot ?? repoRoot),
|
|
3693
4401
|
getCurrentBranch: () => getCurrentBranch(repoRoot),
|
|
3694
4402
|
listOrchBranches: () => {
|
|
3695
4403
|
const result = runGit(["branch", "--list", "orch/*"], repoRoot);
|
|
@@ -3710,11 +4418,83 @@ export default function (pi: ExtensionAPI) {
|
|
|
3710
4418
|
return { message: resolution.error, error: severity !== "info" };
|
|
3711
4419
|
}
|
|
3712
4420
|
|
|
3713
|
-
const { orchBranch, baseBranch,
|
|
3714
|
-
|
|
4421
|
+
const { orchBranch, baseBranch, currentBranch, notices } = resolution as IntegrationContext;
|
|
4422
|
+
let batchId = (resolution as IntegrationContext).batchId;
|
|
3715
4423
|
const outputLines: string[] = [];
|
|
3716
4424
|
let hasWarning = false;
|
|
3717
4425
|
|
|
4426
|
+
// #631: integration merges and cleans up the orch branch. A "completed"
|
|
4427
|
+
// phase on disk is not proof the engine has finished tearing down (or that
|
|
4428
|
+
// an inherited batch's engine is gone). Refuse while it is verifiably alive.
|
|
4429
|
+
// (dead/exited/none proceed — integration does not drive the engine.)
|
|
4430
|
+
{
|
|
4431
|
+
if (engineAttachedHere()) {
|
|
4432
|
+
return {
|
|
4433
|
+
message: `⏳ This session's engine (PID ${activeWorker?.pid ?? process.pid}) is still running/shutting down — integrate after it exits.`,
|
|
4434
|
+
error: true,
|
|
4435
|
+
};
|
|
4436
|
+
}
|
|
4437
|
+
// Ownership is looked up BY THE SELECTED BRANCH at the canonical state root —
|
|
4438
|
+
// not by whichever batch happens to be persisted or reconstructable. The
|
|
4439
|
+
// batch behind `orchBranch` may have no batch-state.json (aborted) and may
|
|
4440
|
+
// not be reconstructable (worker manifests gone) while its engine identity
|
|
4441
|
+
// still records a live pid. Every associated batch must pass the full rule
|
|
4442
|
+
// (alive/none → refuse; dead/exited → proceed). Persisted state for the same
|
|
4443
|
+
// branch is included; a persisted batch for a DIFFERENT branch is irrelevant.
|
|
4444
|
+
const integRoot = stateRoot ?? repoRoot;
|
|
4445
|
+
const associated = new Map<
|
|
4446
|
+
string,
|
|
4447
|
+
{ batchId: string; phase: string; liveness: ReturnType<typeof assessEngineLiveness> }
|
|
4448
|
+
>();
|
|
4449
|
+
for (const b of findBatchesForOrchBranch(integRoot, orchBranch)) {
|
|
4450
|
+
associated.set(b.batchId, { batchId: b.batchId, phase: "unknown", liveness: b.liveness });
|
|
4451
|
+
}
|
|
4452
|
+
try {
|
|
4453
|
+
const persisted = loadBatchState(integRoot);
|
|
4454
|
+
if (persisted && (persisted.orchBranch === orchBranch || persisted.batchId === batchId)) {
|
|
4455
|
+
associated.set(persisted.batchId, {
|
|
4456
|
+
batchId: persisted.batchId,
|
|
4457
|
+
phase: persisted.phase,
|
|
4458
|
+
liveness: assessEngineLiveness(integRoot, persisted.batchId),
|
|
4459
|
+
});
|
|
4460
|
+
}
|
|
4461
|
+
} catch {
|
|
4462
|
+
/* reported elsewhere */
|
|
4463
|
+
}
|
|
4464
|
+
if (batchId && !associated.has(batchId)) {
|
|
4465
|
+
associated.set(batchId, {
|
|
4466
|
+
batchId,
|
|
4467
|
+
phase: "completed",
|
|
4468
|
+
liveness: assessEngineLiveness(integRoot, batchId),
|
|
4469
|
+
});
|
|
4470
|
+
}
|
|
4471
|
+
for (const target of associated.values()) {
|
|
4472
|
+
const decision = decideRecoveryOwnership({
|
|
4473
|
+
operation: "orch_integrate",
|
|
4474
|
+
local: {
|
|
4475
|
+
engineAttached: false,
|
|
4476
|
+
phase: orchBatchState.phase,
|
|
4477
|
+
batchId: orchBatchState.batchId,
|
|
4478
|
+
pid: null,
|
|
4479
|
+
},
|
|
4480
|
+
target: { batchId: target.batchId, phase: target.phase },
|
|
4481
|
+
liveness: target.liveness,
|
|
4482
|
+
priorSupervisor,
|
|
4483
|
+
});
|
|
4484
|
+
if (!decision.proceed) return { message: decision.reason, error: true };
|
|
4485
|
+
}
|
|
4486
|
+
// No batch is associated with this branch (pure branch integration):
|
|
4487
|
+
// nothing an engine could be driving — proceed.
|
|
4488
|
+
|
|
4489
|
+
// Bind cleanup/history to the batch behind THIS branch. The resolver
|
|
4490
|
+
// leaves batchId empty when an explicit branch differs from persisted
|
|
4491
|
+
// state; a unique associated runtime batch fills it in. Ambiguous (>1)
|
|
4492
|
+
// → leave empty: batch-scoped cleanup is skipped rather than guessed.
|
|
4493
|
+
if (!batchId && associated.size === 1) {
|
|
4494
|
+
batchId = [...associated.values()][0].batchId;
|
|
4495
|
+
}
|
|
4496
|
+
}
|
|
4497
|
+
|
|
3718
4498
|
for (const notice of notices) {
|
|
3719
4499
|
outputLines.push(notice);
|
|
3720
4500
|
}
|
|
@@ -3849,7 +4629,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3849
4629
|
|
|
3850
4630
|
const branchCleanupLines: string[] = [];
|
|
3851
4631
|
for (const repo of allRepos) {
|
|
3852
|
-
const branchCleanup = deleteStaleBranches(repo.root, opId, batchId);
|
|
4632
|
+
const branchCleanup = deleteStaleBranches(repo.root, opId, batchId, stateRoot ?? repoRoot);
|
|
3853
4633
|
const totalDeleted =
|
|
3854
4634
|
branchCleanup.deletedTaskBranches.length + branchCleanup.deletedSavedBranches.length;
|
|
3855
4635
|
if (totalDeleted > 0 || branchCleanup.failedDeletes.length > 0) {
|
|
@@ -3896,6 +4676,17 @@ export default function (pi: ExtensionAPI) {
|
|
|
3896
4676
|
hasWarning = true;
|
|
3897
4677
|
}
|
|
3898
4678
|
|
|
4679
|
+
// #610: the batch is integrated. Any batch-end epilogue still DEFERRED behind
|
|
4680
|
+
// this turn (the engine finished while the supervisor was mid-turn, #621)
|
|
4681
|
+
// would fire at agent_settled with stale content — "Ready for integration /
|
|
4682
|
+
// run orch_integrate()" or the supervised "Integration Plan … merge commit"
|
|
4683
|
+
// prompt for an orch branch that no longer exists. Supersede it now, and mark
|
|
4684
|
+
// the in-memory batch integrated so a not-yet-deferred dispatch also skips.
|
|
4685
|
+
if (orchBatchState.batchId === batchId || !batchId) {
|
|
4686
|
+
orchBatchState.integratedAt = Date.now();
|
|
4687
|
+
}
|
|
4688
|
+
supersedeDeferredEpilogue();
|
|
4689
|
+
|
|
3899
4690
|
// TP-179: Write integratedAt to batch history before deleting state
|
|
3900
4691
|
if (batchId) {
|
|
3901
4692
|
try {
|
|
@@ -3905,8 +4696,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
3905
4696
|
}
|
|
3906
4697
|
}
|
|
3907
4698
|
|
|
4699
|
+
// #631: delete persisted state ONLY if it belongs to the integrated branch/batch.
|
|
4700
|
+
// An explicit branch integration must not erase an unrelated batch's recovery
|
|
4701
|
+
// checkpoint (its engine may even still be alive).
|
|
3908
4702
|
try {
|
|
3909
|
-
|
|
4703
|
+
const deleted = deleteBatchStateIfOwned(stateRoot ?? repoRoot, batchId, orchBranch);
|
|
4704
|
+
if (deleted === false) {
|
|
4705
|
+
outputLines.push(
|
|
4706
|
+
`ℹ️ Persisted batch state belongs to a different batch/branch — left in place (not part of this integration).`,
|
|
4707
|
+
);
|
|
4708
|
+
}
|
|
3910
4709
|
} catch {
|
|
3911
4710
|
/* best effort */
|
|
3912
4711
|
}
|
|
@@ -4147,6 +4946,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
4147
4946
|
|
|
4148
4947
|
ctx.ui.notify(`🔄 **${reason}** Activating supervisor.\n\n` + summary, "info");
|
|
4149
4948
|
|
|
4949
|
+
// #631: record what we know about the previous owner. The engine of a
|
|
4950
|
+
// batch we did not start is NOT attached to this process; the active-
|
|
4951
|
+
// phase guards use this + engine.json to decide whether an inherited
|
|
4952
|
+
// "executing" phase is a resumable orphan.
|
|
4953
|
+
priorSupervisor =
|
|
4954
|
+
"lock" in lockResult && lockResult.lock
|
|
4955
|
+
? { pid: lockResult.lock.pid, alive: isProcessAlive(lockResult.lock.pid) }
|
|
4956
|
+
: null;
|
|
4150
4957
|
// Populate orchBatchState from persisted state
|
|
4151
4958
|
orchBatchState.batchId = batchState.batchId;
|
|
4152
4959
|
orchBatchState.phase = batchState.phase as typeof orchBatchState.phase;
|
|
@@ -4193,6 +5000,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
4193
5000
|
"warning",
|
|
4194
5001
|
);
|
|
4195
5002
|
|
|
5003
|
+
// #631: record what we know about the previous owner. The engine of a
|
|
5004
|
+
// batch we did not start is NOT attached to this process; the active-
|
|
5005
|
+
// phase guards use this + engine.json to decide whether an inherited
|
|
5006
|
+
// "executing" phase is a resumable orphan.
|
|
5007
|
+
priorSupervisor =
|
|
5008
|
+
"lock" in lockResult && lockResult.lock
|
|
5009
|
+
? { pid: lockResult.lock.pid, alive: isProcessAlive(lockResult.lock.pid) }
|
|
5010
|
+
: null;
|
|
4196
5011
|
// Populate orchBatchState from persisted state
|
|
4197
5012
|
orchBatchState.batchId = batchState.batchId;
|
|
4198
5013
|
orchBatchState.phase = batchState.phase as typeof orchBatchState.phase;
|
|
@@ -4695,6 +5510,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
4695
5510
|
"The 'to' parameter must be a valid agent session name from the current batch.",
|
|
4696
5511
|
"Use orch_status() to see active session names.",
|
|
4697
5512
|
"Default type is 'steer' (course correction). Other types: 'query', 'abort', 'info'.",
|
|
5513
|
+
"HOLD CONTRACT (#630): when a worker is holding for a ruling you escalated for, send type='info' to ACKNOWLEDGE (\"received, ruling pending\") — it keeps the worker on hold and resets its relaunch budget. Send type='steer' only for the actual ruling/instruction — it releases the hold.",
|
|
4698
5514
|
"Messages are limited to 4KB. For larger context, write to a file and reference by path.",
|
|
4699
5515
|
],
|
|
4700
5516
|
parameters: Type.Object({
|
|
@@ -4796,6 +5612,29 @@ export default function (pi: ExtensionAPI) {
|
|
|
4796
5612
|
return `❌ Batch ${state.batchId} is in terminal phase (${state.phase}). Start or resume a batch before sending messages.`;
|
|
4797
5613
|
}
|
|
4798
5614
|
|
|
5615
|
+
// #630: a registry agent still marked running whose PROCESS is gone gets a
|
|
5616
|
+
// distinct, actionable error (pid, last-seen, what to do) instead of the
|
|
5617
|
+
// generic "unknown session" that collectKnownAgentIds would otherwise
|
|
5618
|
+
// produce after filtering dead pids. This is the #630 incident: the worker
|
|
5619
|
+
// died after an unanswered escalation; registry stayed `running` frozen.
|
|
5620
|
+
try {
|
|
5621
|
+
const registry = readRegistrySnapshot(stateRoot, state.batchId);
|
|
5622
|
+
const manifest = registry?.agents[to];
|
|
5623
|
+
if (manifest && !isTerminalStatus(manifest.status) && !registryIsProcessAlive(manifest.pid)) {
|
|
5624
|
+
const lastSeen = new Date(registry!.updatedAt).toISOString();
|
|
5625
|
+
return (
|
|
5626
|
+
`❌ Agent "${to}" is DEAD: its process (PID ${manifest.pid}) no longer exists, but the registry still ` +
|
|
5627
|
+
`marks it "${manifest.status}" (last registry update ${lastSeen}${manifest.taskId ? `, task ${manifest.taskId}` : ""}). ` +
|
|
5628
|
+
`No live consumer can receive this message.\n` +
|
|
5629
|
+
` Do NOT hand-edit registry.json. orch_resume(force=true) reconciles the dead worker ` +
|
|
5630
|
+
`(re-execute in its existing worktree; committed work survives) — if the batch is inherited, ` +
|
|
5631
|
+
`the ownership gate must pass first (see the takeover summary's Engine line).`
|
|
5632
|
+
);
|
|
5633
|
+
}
|
|
5634
|
+
} catch {
|
|
5635
|
+
/* registry unreadable — fall through to the normal validation */
|
|
5636
|
+
}
|
|
5637
|
+
|
|
4799
5638
|
// Build valid runtime agent IDs (registry-first, legacy fallback).
|
|
4800
5639
|
const validSessions = new Set<string>(collectKnownAgentIds(stateRoot, state));
|
|
4801
5640
|
|
|
@@ -5241,6 +6080,188 @@ export default function (pi: ExtensionAPI) {
|
|
|
5241
6080
|
return lines.join("\n");
|
|
5242
6081
|
}
|
|
5243
6082
|
|
|
6083
|
+
/**
|
|
6084
|
+
* #631: record operator-verified engine shutdown for the current/persisted
|
|
6085
|
+
* batch. Shared by the supervisor tool and the /orch-confirm-engine-shutdown
|
|
6086
|
+
* command. Always audited.
|
|
6087
|
+
*/
|
|
6088
|
+
function doOrchConfirmEngineShutdown(
|
|
6089
|
+
note: string,
|
|
6090
|
+
stateRoot: string,
|
|
6091
|
+
explicitBatchId?: string,
|
|
6092
|
+
): string {
|
|
6093
|
+
// Target: an EXPLICIT batchId (exactly the batch a refusal named — works for
|
|
6094
|
+
// meta-only legacy batches that are not reconstructable), else the same
|
|
6095
|
+
// target the recovery gates use: persisted → reconstructed → cached.
|
|
6096
|
+
const explicit = (explicitBatchId ?? "").trim();
|
|
6097
|
+
if (explicit && !/^[A-Za-z0-9._-]+$/.test(explicit)) {
|
|
6098
|
+
return `❌ Invalid batchId "${explicit}".`;
|
|
6099
|
+
}
|
|
6100
|
+
const target = explicit ? null : resolveRecoveryTarget(stateRoot, true);
|
|
6101
|
+
const batchId =
|
|
6102
|
+
explicit || target?.batchId || orchBatchState.batchId || supervisorState.batchId || "";
|
|
6103
|
+
if (!batchId) {
|
|
6104
|
+
return (
|
|
6105
|
+
"❌ No batch to confirm shutdown for (no batch on disk, nothing reconstructable, nothing in memory). " +
|
|
6106
|
+
"Pass the batchId named by the refusal explicitly."
|
|
6107
|
+
);
|
|
6108
|
+
}
|
|
6109
|
+
if (!note || note.trim().length < 8) {
|
|
6110
|
+
return "❌ A verification note is required (what you checked and how) — it is written to engine.json and the audit trail.";
|
|
6111
|
+
}
|
|
6112
|
+
const result = recordOperatorConfirmedShutdown(stateRoot, batchId, {
|
|
6113
|
+
supervisorPid: process.pid,
|
|
6114
|
+
note,
|
|
6115
|
+
});
|
|
6116
|
+
logRecoveryAction(stateRoot, batchId, {
|
|
6117
|
+
action: "confirm_engine_shutdown",
|
|
6118
|
+
classification: "destructive",
|
|
6119
|
+
context: `operator-verified engine shutdown for inherited batch: ${note.slice(0, 300)}`,
|
|
6120
|
+
command: "orch_confirm_engine_shutdown",
|
|
6121
|
+
result: result.ok ? "success" : "failure",
|
|
6122
|
+
detail: result.reason,
|
|
6123
|
+
});
|
|
6124
|
+
return result.ok
|
|
6125
|
+
? `✅ ${result.reason}. Recovery tools (orch_resume(force=true), retry/skip/force_merge) will now proceed via the verified path. Audit entry written.`
|
|
6126
|
+
: `❌ Not recorded: ${result.reason}`;
|
|
6127
|
+
}
|
|
6128
|
+
|
|
6129
|
+
pi.registerCommand("orch-confirm-engine-shutdown", {
|
|
6130
|
+
description:
|
|
6131
|
+
"Record operator-verified engine shutdown for a batch with no engine identity (#631): /orch-confirm-engine-shutdown [--batch <batchId>] <what you verified>",
|
|
6132
|
+
handler: async (args, ctx) => {
|
|
6133
|
+
// Syntax: /orch-confirm-engine-shutdown [--batch <batchId>] <note>
|
|
6134
|
+
let raw = (args ?? "").trim();
|
|
6135
|
+
let explicitBatchId: string | undefined;
|
|
6136
|
+
const m = /(?:^|\s)--batch\s+(\S+)/.exec(raw);
|
|
6137
|
+
if (m) {
|
|
6138
|
+
explicitBatchId = m[1];
|
|
6139
|
+
raw = raw.replace(m[0], " ").trim();
|
|
6140
|
+
}
|
|
6141
|
+
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
6142
|
+
const result = doOrchConfirmEngineShutdown(raw, stateRoot, explicitBatchId);
|
|
6143
|
+
ctx.ui.notify(result, result.startsWith("✅") ? "info" : "warning");
|
|
6144
|
+
},
|
|
6145
|
+
});
|
|
6146
|
+
|
|
6147
|
+
// ── #631: explicit, audited legacy shutdown confirmation ──
|
|
6148
|
+
// For a batch with NO engine identity (pre-#631 engine, or one that never
|
|
6149
|
+
// published), recovery tools fail closed: unknown ownership is not confirmed
|
|
6150
|
+
// shutdown. The operator verifies out-of-band that no engine process exists,
|
|
6151
|
+
// then records it here. The record is an `exited` identity (so every gate
|
|
6152
|
+
// proceeds through the normal verified path) AND an audit-trail entry.
|
|
6153
|
+
pi.registerTool({
|
|
6154
|
+
name: "orch_confirm_engine_shutdown",
|
|
6155
|
+
label: "Confirm Engine Shutdown",
|
|
6156
|
+
description:
|
|
6157
|
+
"Record that the operator verified NO engine process is running for the inherited batch (used only when " +
|
|
6158
|
+
"no engine identity is recorded, so the runtime cannot verify it). Unblocks orch_resume/retry/skip/force_merge. " +
|
|
6159
|
+
"Refuses when a real engine identity exists — that path is pid-verified and cannot be overridden.",
|
|
6160
|
+
promptSnippet:
|
|
6161
|
+
"orch_confirm_engine_shutdown(note) — record operator-verified engine shutdown for a batch with no engine identity",
|
|
6162
|
+
promptGuidelines: [
|
|
6163
|
+
"Only after verifying out-of-band that no engine process exists for this repo (e.g. Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -match 'engine-worker' } on Windows; pgrep -af engine-worker on POSIX).",
|
|
6164
|
+
"Never use this to override a refusal that names a live engine PID — wait for it or terminate it.",
|
|
6165
|
+
"The note should say what you checked; it is written to engine.json and the audit trail.",
|
|
6166
|
+
],
|
|
6167
|
+
parameters: Type.Object({
|
|
6168
|
+
note: Type.String({
|
|
6169
|
+
description:
|
|
6170
|
+
"What was verified and how (e.g. 'no engine-worker processes in Get-CimInstance output at 21:32')",
|
|
6171
|
+
}),
|
|
6172
|
+
batchId: Type.Optional(
|
|
6173
|
+
Type.String({
|
|
6174
|
+
description:
|
|
6175
|
+
"Exact batchId to confirm (the one named by the refusal). Omit to use the persisted/reconstructed batch.",
|
|
6176
|
+
}),
|
|
6177
|
+
),
|
|
6178
|
+
}),
|
|
6179
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
6180
|
+
const text = doOrchConfirmEngineShutdown(params.note, resolveToolStateRoot(ctx), params.batchId);
|
|
6181
|
+
return { content: [{ type: "text" as const, text }], details: undefined };
|
|
6182
|
+
},
|
|
6183
|
+
});
|
|
6184
|
+
|
|
6185
|
+
// ── #625: Audit-trail tool ───────────────────────────────────
|
|
6186
|
+
// The supervisor's audit trail (.pi/supervisor/actions.jsonl) was previously
|
|
6187
|
+
// hand-appended via bash per the system-prompt instructions — the LLM invented
|
|
6188
|
+
// the `ts` values (5+ hours off, non-monotonic, crossing calendar days).
|
|
6189
|
+
// logRecoveryAction() existed but nothing exposed it. This tool code-stamps
|
|
6190
|
+
// ts and batchId so the audit trail is trustworthy.
|
|
6191
|
+
pi.registerTool({
|
|
6192
|
+
name: "log_recovery_action",
|
|
6193
|
+
label: "Log Recovery Action",
|
|
6194
|
+
description:
|
|
6195
|
+
"Append an entry to the supervisor audit trail (.pi/supervisor/actions.jsonl). " +
|
|
6196
|
+
"Timestamps and batchId are stamped by code — never hand-write the file.",
|
|
6197
|
+
promptSnippet:
|
|
6198
|
+
"log_recovery_action(action, classification, context, command, result, detail, …) — append audit-trail entry",
|
|
6199
|
+
promptGuidelines: [
|
|
6200
|
+
"Use log_recovery_action for EVERY audit-trail entry — never append to actions.jsonl with bash.",
|
|
6201
|
+
"Timestamps and batchId are stamped automatically; do not supply them.",
|
|
6202
|
+
'For destructive actions: log result="pending" BEFORE executing, then log the real result after.',
|
|
6203
|
+
"classification: diagnostic | tier0_known | destructive.",
|
|
6204
|
+
],
|
|
6205
|
+
parameters: Type.Object({
|
|
6206
|
+
action: Type.String({ description: 'Action identifier, e.g. "merge_retry", "kill_session"' }),
|
|
6207
|
+
classification: Type.Union(
|
|
6208
|
+
[Type.Literal("diagnostic"), Type.Literal("tier0_known"), Type.Literal("destructive")],
|
|
6209
|
+
{ description: "Recovery action classification" },
|
|
6210
|
+
),
|
|
6211
|
+
context: Type.String({ description: "Why this action was taken" }),
|
|
6212
|
+
command: Type.String({ description: "Command or operation executed" }),
|
|
6213
|
+
result: Type.Union(
|
|
6214
|
+
[
|
|
6215
|
+
Type.Literal("pending"),
|
|
6216
|
+
Type.Literal("success"),
|
|
6217
|
+
Type.Literal("failure"),
|
|
6218
|
+
Type.Literal("skipped"),
|
|
6219
|
+
],
|
|
6220
|
+
{ description: "Outcome (pending = before a destructive action)" },
|
|
6221
|
+
),
|
|
6222
|
+
detail: Type.String({ description: "Result detail — error on failure, summary on success" }),
|
|
6223
|
+
waveIndex: Type.Optional(Type.Number({ description: "Wave index if wave-scoped" })),
|
|
6224
|
+
laneNumber: Type.Optional(Type.Number({ description: "Lane number if lane-scoped" })),
|
|
6225
|
+
taskId: Type.Optional(Type.String({ description: "Task ID if task-scoped" })),
|
|
6226
|
+
}),
|
|
6227
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
6228
|
+
try {
|
|
6229
|
+
const stateRoot = resolveToolStateRoot(ctx);
|
|
6230
|
+
const batchId = orchBatchState.batchId || supervisorState.batchId || "unknown";
|
|
6231
|
+
logRecoveryAction(stateRoot, batchId, {
|
|
6232
|
+
action: params.action,
|
|
6233
|
+
classification: params.classification,
|
|
6234
|
+
context: params.context,
|
|
6235
|
+
command: params.command,
|
|
6236
|
+
result: params.result,
|
|
6237
|
+
detail: params.detail,
|
|
6238
|
+
...(params.waveIndex !== undefined ? { waveIndex: params.waveIndex } : {}),
|
|
6239
|
+
...(params.laneNumber !== undefined ? { laneNumber: params.laneNumber } : {}),
|
|
6240
|
+
...(params.taskId !== undefined ? { taskId: params.taskId } : {}),
|
|
6241
|
+
});
|
|
6242
|
+
return {
|
|
6243
|
+
content: [
|
|
6244
|
+
{
|
|
6245
|
+
type: "text" as const,
|
|
6246
|
+
text: `Audit entry logged: ${params.action} (${params.classification}, ${params.result})`,
|
|
6247
|
+
},
|
|
6248
|
+
],
|
|
6249
|
+
details: undefined,
|
|
6250
|
+
};
|
|
6251
|
+
} catch (err) {
|
|
6252
|
+
return {
|
|
6253
|
+
content: [
|
|
6254
|
+
{
|
|
6255
|
+
type: "text" as const,
|
|
6256
|
+
text: `Error logging audit entry: ${err instanceof Error ? err.message : String(err)}`,
|
|
6257
|
+
},
|
|
6258
|
+
],
|
|
6259
|
+
details: undefined,
|
|
6260
|
+
};
|
|
6261
|
+
}
|
|
6262
|
+
},
|
|
6263
|
+
});
|
|
6264
|
+
|
|
5244
6265
|
pi.registerTool({
|
|
5245
6266
|
name: "trigger_wrap_up",
|
|
5246
6267
|
label: "Trigger Wrap Up",
|
|
@@ -5760,6 +6781,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
5760
6781
|
"info",
|
|
5761
6782
|
);
|
|
5762
6783
|
|
|
6784
|
+
// #631: record what we know about the previous owner. The engine of a
|
|
6785
|
+
// batch we did not start is NOT attached to this process; the active-
|
|
6786
|
+
// phase guards use this + engine.json to decide whether an inherited
|
|
6787
|
+
// "executing" phase is a resumable orphan.
|
|
6788
|
+
priorSupervisor =
|
|
6789
|
+
"lock" in lockResult && lockResult.lock
|
|
6790
|
+
? { pid: lockResult.lock.pid, alive: isProcessAlive(lockResult.lock.pid) }
|
|
6791
|
+
: null;
|
|
5763
6792
|
// Populate orchBatchState from persisted state for the supervisor
|
|
5764
6793
|
// prompt rebuild. We copy the key fields used by the system prompt.
|
|
5765
6794
|
orchBatchState.batchId = batchState.batchId;
|
|
@@ -5811,6 +6840,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
5811
6840
|
|
|
5812
6841
|
// Store the live lock info so the /orch handler can detect it
|
|
5813
6842
|
// (preventing a second /orch from starting a concurrent batch).
|
|
6843
|
+
// #631: the other session's engine is live and NOT ours — record it.
|
|
6844
|
+
priorSupervisor =
|
|
6845
|
+
"lock" in lockResult && lockResult.lock
|
|
6846
|
+
? { pid: lockResult.lock.pid, alive: isProcessAlive(lockResult.lock.pid) }
|
|
6847
|
+
: null;
|
|
5814
6848
|
orchBatchState.batchId = batchState.batchId;
|
|
5815
6849
|
orchBatchState.phase = batchState.phase as typeof orchBatchState.phase;
|
|
5816
6850
|
orchBatchState.baseBranch = batchState.baseBranch;
|