taskplane 0.30.4 → 0.30.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/taskplane/agent-bridge-extension.ts +66 -8
- package/extensions/taskplane/agent-host.ts +170 -9
- package/extensions/taskplane/config-loader.ts +9 -0
- package/extensions/taskplane/config-schema.ts +47 -1
- package/extensions/taskplane/context-repair.ts +158 -0
- package/extensions/taskplane/diagnostic-reports.ts +109 -6
- package/extensions/taskplane/diagnostics.ts +3 -0
- package/extensions/taskplane/engine-identity.ts +401 -0
- package/extensions/taskplane/engine-worker.ts +59 -3
- package/extensions/taskplane/engine.ts +137 -17
- package/extensions/taskplane/execution.ts +89 -8
- package/extensions/taskplane/extension.ts +1288 -241
- package/extensions/taskplane/git.ts +74 -0
- package/extensions/taskplane/lane-runner.ts +971 -95
- package/extensions/taskplane/process-registry.ts +7 -2
- package/extensions/taskplane/resume.ts +559 -23
- package/extensions/taskplane/review-analysis.ts +450 -0
- package/extensions/taskplane/segment-recovery.ts +192 -0
- package/extensions/taskplane/supervisor-dispatch.ts +103 -0
- package/extensions/taskplane/supervisor-primer.md +182 -2
- package/extensions/taskplane/supervisor.ts +247 -24
- package/extensions/taskplane/types.ts +142 -4
- package/extensions/taskplane/worktree.ts +143 -2
- package/package.json +1 -1
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
StateFileError,
|
|
27
27
|
WorkspaceConfigError,
|
|
28
28
|
freshOrchBatchState,
|
|
29
|
+
generateBatchId,
|
|
29
30
|
} from "./types.ts";
|
|
30
31
|
import type {
|
|
31
32
|
AbortMode,
|
|
@@ -44,6 +45,7 @@ import {
|
|
|
44
45
|
loadBatchState,
|
|
45
46
|
saveBatchState,
|
|
46
47
|
detectOrphanSessions,
|
|
48
|
+
reconstructBatchStateFromRuntime,
|
|
47
49
|
updateBatchHistoryIntegration,
|
|
48
50
|
} from "./persistence.ts";
|
|
49
51
|
import {
|
|
@@ -53,7 +55,12 @@ import {
|
|
|
53
55
|
formatPreflightResults,
|
|
54
56
|
runPreflight,
|
|
55
57
|
} from "./worktree.ts";
|
|
56
|
-
import {
|
|
58
|
+
import {
|
|
59
|
+
batchTaskScope,
|
|
60
|
+
computeTransitiveDependents,
|
|
61
|
+
execLog,
|
|
62
|
+
resolveCanonicalTaskPaths,
|
|
63
|
+
} from "./execution.ts";
|
|
57
64
|
import { executeOrchBatch } from "./engine.ts";
|
|
58
65
|
import { formatDiscoveryResults, runDiscovery } from "./discovery.ts";
|
|
59
66
|
import { formatOrchSessions, listOrchSessions } from "./sessions.ts";
|
|
@@ -67,6 +74,7 @@ import {
|
|
|
67
74
|
} from "./config.ts";
|
|
68
75
|
import { resolveOperatorId } from "./naming.ts";
|
|
69
76
|
import { reconstructAllocatedLanes, resumeOrchBatch } from "./resume.ts";
|
|
77
|
+
import { markTaskSegmentsSkipped, resetTaskSegmentsForRetry } from "./segment-recovery.ts";
|
|
70
78
|
import { buildExecutionContext } from "./workspace.ts";
|
|
71
79
|
import { openSettingsTui } from "./settings-tui.ts";
|
|
72
80
|
import { loadProjectConfig } from "./config-loader.ts";
|
|
@@ -102,11 +110,20 @@ import {
|
|
|
102
110
|
isProcessAlive as registryIsProcessAlive,
|
|
103
111
|
isTerminalStatus,
|
|
104
112
|
} from "./process-registry.ts";
|
|
113
|
+
import {
|
|
114
|
+
assessEngineLiveness,
|
|
115
|
+
decideRecoveryOwnership,
|
|
116
|
+
findBatchesForOrchBranch,
|
|
117
|
+
markEngineExited,
|
|
118
|
+
recordOperatorConfirmedShutdown,
|
|
119
|
+
writeEngineIdentity,
|
|
120
|
+
} from "./engine-identity.ts";
|
|
105
121
|
import type { MailboxMessageType } from "./types.ts";
|
|
106
122
|
import {
|
|
107
123
|
activateSupervisor,
|
|
108
124
|
deactivateSupervisor,
|
|
109
125
|
transitionToRoutingMode,
|
|
126
|
+
stopBatchMonitoring,
|
|
110
127
|
freshSupervisorState,
|
|
111
128
|
registerSupervisorPromptHook,
|
|
112
129
|
checkSupervisorLockOnStartup,
|
|
@@ -117,7 +134,12 @@ import {
|
|
|
117
134
|
triggerSupervisorIntegration,
|
|
118
135
|
presentBatchSummary,
|
|
119
136
|
resolveModelFromString,
|
|
137
|
+
isStaleExtensionCtx,
|
|
138
|
+
safeCtxCallFromCallback,
|
|
139
|
+
logRecoveryAction,
|
|
120
140
|
} from "./supervisor.ts";
|
|
141
|
+
import { SupervisorNoticeGate } from "./supervisor-dispatch.ts";
|
|
142
|
+
import { repairToolResultOrdering } from "./context-repair.ts";
|
|
121
143
|
import type {
|
|
122
144
|
SupervisorConfig,
|
|
123
145
|
SupervisorRoutingContext,
|
|
@@ -333,6 +355,17 @@ export function resolveIntegrationContext(
|
|
|
333
355
|
|
|
334
356
|
// Source 2: CLI positional branch arg overrides or fills in
|
|
335
357
|
if (parsed.orchBranchArg) {
|
|
358
|
+
// #631: an explicit branch that differs from the persisted batch's branch
|
|
359
|
+
// must NOT inherit that batch's id — cleanup/history/ownership would then
|
|
360
|
+
// target an unrelated batch. The batch behind the selected branch (if any)
|
|
361
|
+
// is looked up from runtime artifacts by the caller.
|
|
362
|
+
if (orchBranch && batchId && orchBranch !== parsed.orchBranchArg) {
|
|
363
|
+
notices.push(
|
|
364
|
+
`ℹ️ Persisted batch ${batchId} belongs to ${orchBranch}; integrating ${parsed.orchBranchArg} instead — ` +
|
|
365
|
+
`batch-scoped cleanup/history will use the batch associated with that branch, if any.`,
|
|
366
|
+
);
|
|
367
|
+
batchId = "";
|
|
368
|
+
}
|
|
336
369
|
orchBranch = parsed.orchBranchArg;
|
|
337
370
|
}
|
|
338
371
|
|
|
@@ -1049,10 +1082,17 @@ export function startBatchAsync(
|
|
|
1049
1082
|
batchState.endedAt = Date.now();
|
|
1050
1083
|
batchState.errors.push(`Unhandled engine error: ${errMsg}`);
|
|
1051
1084
|
}
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1085
|
+
// #620: this .catch runs when the (main-thread fallback) engine promise
|
|
1086
|
+
// rejects later — an async window where ctx may be stale. Guard the UI
|
|
1087
|
+
// sink so a stale-ctx throw can't crash Pi; onTerminal still runs.
|
|
1088
|
+
safeCtxCallFromCallback(
|
|
1089
|
+
() =>
|
|
1090
|
+
ctx.ui.notify(
|
|
1091
|
+
`❌ Engine crashed with unhandled error: ${errMsg}\n` +
|
|
1092
|
+
` Batch ${batchState.batchId} marked as failed.`,
|
|
1093
|
+
"error",
|
|
1094
|
+
),
|
|
1095
|
+
"startBatchAsync.catch.notify",
|
|
1056
1096
|
);
|
|
1057
1097
|
updateWidget();
|
|
1058
1098
|
// TP-041 R002-3: Deactivate supervisor on all terminal paths
|
|
@@ -1109,6 +1149,55 @@ function resolveEngineWorkerPath(): string {
|
|
|
1109
1149
|
*
|
|
1110
1150
|
* @since TP-071
|
|
1111
1151
|
*/
|
|
1152
|
+
/** #631: true while a main-thread fallback engine is running in this process. */
|
|
1153
|
+
let fallbackEngineActive = false;
|
|
1154
|
+
export function isFallbackEngineActive(): boolean {
|
|
1155
|
+
return fallbackEngineActive;
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
/**
|
|
1159
|
+
* #631: delete `batch-state.json` at `stateRoot` ONLY when it belongs to the
|
|
1160
|
+
* integrated batch (same batchId) or branch (same orchBranch). Returns true when
|
|
1161
|
+
* deleted, false when an unrelated batch's checkpoint was preserved, null when
|
|
1162
|
+
* nothing was persisted.
|
|
1163
|
+
*/
|
|
1164
|
+
export function deleteBatchStateIfOwned(
|
|
1165
|
+
stateRoot: string,
|
|
1166
|
+
batchId: string,
|
|
1167
|
+
orchBranch: string,
|
|
1168
|
+
): boolean | null {
|
|
1169
|
+
let persisted: PersistedBatchState | null = null;
|
|
1170
|
+
try {
|
|
1171
|
+
persisted = loadBatchState(stateRoot);
|
|
1172
|
+
} catch {
|
|
1173
|
+
// Unreadable state: ownership cannot be established — preserve it (a corrupt
|
|
1174
|
+
// checkpoint is still evidence an operator may need).
|
|
1175
|
+
return false;
|
|
1176
|
+
}
|
|
1177
|
+
if (!persisted) return null;
|
|
1178
|
+
const owned =
|
|
1179
|
+
(batchId && persisted.batchId === batchId) || (orchBranch && persisted.orchBranch === orchBranch);
|
|
1180
|
+
if (!owned) return false;
|
|
1181
|
+
deleteBatchState(stateRoot);
|
|
1182
|
+
return true;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
/** #631: resolve true once the child has actually terminated, false on timeout. */
|
|
1186
|
+
export function waitForChildExit(child: ChildProcess, timeoutMs: number): Promise<boolean> {
|
|
1187
|
+
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(true);
|
|
1188
|
+
return new Promise((resolve) => {
|
|
1189
|
+
const timer = setTimeout(() => {
|
|
1190
|
+
child.off("exit", onExit);
|
|
1191
|
+
resolve(false);
|
|
1192
|
+
}, timeoutMs);
|
|
1193
|
+
const onExit = () => {
|
|
1194
|
+
clearTimeout(timer);
|
|
1195
|
+
resolve(true);
|
|
1196
|
+
};
|
|
1197
|
+
child.once("exit", onExit);
|
|
1198
|
+
});
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1112
1201
|
export function startBatchInWorker(
|
|
1113
1202
|
wkData: EngineWorkerData,
|
|
1114
1203
|
batchState: import("./types.ts").OrchBatchRuntimeState,
|
|
@@ -1139,9 +1228,16 @@ export function startBatchInWorker(
|
|
|
1139
1228
|
});
|
|
1140
1229
|
} catch (spawnErr: unknown) {
|
|
1141
1230
|
const errMsg = spawnErr instanceof Error ? spawnErr.message : String(spawnErr);
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1231
|
+
// #620: stale-safe (uniform with the async callbacks below). This runs
|
|
1232
|
+
// synchronously during the command so ctx is normally fresh, but guarding
|
|
1233
|
+
// keeps the "no bare ctx.ui.notify in startBatchInWorker" invariant simple.
|
|
1234
|
+
safeCtxCallFromCallback(
|
|
1235
|
+
() =>
|
|
1236
|
+
ctx.ui.notify(
|
|
1237
|
+
`⚠️ Engine process spawn failed: ${errMsg}\n Falling back to main-thread execution.`,
|
|
1238
|
+
"warning",
|
|
1239
|
+
),
|
|
1240
|
+
"spawn.fallback.notify",
|
|
1145
1241
|
);
|
|
1146
1242
|
// Construct fallback engine function from workerData and run on main thread
|
|
1147
1243
|
const wsConfig = wkData.workspaceConfig
|
|
@@ -1156,7 +1252,7 @@ export function startBatchInWorker(
|
|
|
1156
1252
|
wkData.cwd,
|
|
1157
1253
|
batchState,
|
|
1158
1254
|
(msg: string, lvl: "info" | "warning" | "error") => {
|
|
1159
|
-
ctx.ui.notify(msg, lvl);
|
|
1255
|
+
safeCtxCallFromCallback(() => ctx.ui.notify(msg, lvl), "fallback.notify");
|
|
1160
1256
|
updateWidget();
|
|
1161
1257
|
},
|
|
1162
1258
|
(monState: import("./types.ts").MonitorState) => {
|
|
@@ -1179,7 +1275,7 @@ export function startBatchInWorker(
|
|
|
1179
1275
|
wkData.cwd,
|
|
1180
1276
|
batchState,
|
|
1181
1277
|
(msg: string, lvl: "info" | "warning" | "error") => {
|
|
1182
|
-
ctx.ui.notify(msg, lvl);
|
|
1278
|
+
safeCtxCallFromCallback(() => ctx.ui.notify(msg, lvl), "fallback.notify");
|
|
1183
1279
|
updateWidget();
|
|
1184
1280
|
},
|
|
1185
1281
|
(monState: import("./types.ts").MonitorState) => {
|
|
@@ -1194,7 +1290,38 @@ export function startBatchInWorker(
|
|
|
1194
1290
|
null, // onLaneTerminated — main-thread fallback path
|
|
1195
1291
|
null, // onLaneRespawned — main-thread fallback path
|
|
1196
1292
|
);
|
|
1197
|
-
|
|
1293
|
+
// #631: the fallback runs the engine IN THIS PROCESS. It must be just as
|
|
1294
|
+
// visible/ownable as a forked engine: publish an identity with our own pid
|
|
1295
|
+
// and mark it exited when the run settles. If identity cannot be
|
|
1296
|
+
// published, refuse to start (ownership invariant) instead of running an
|
|
1297
|
+
// invisible engine.
|
|
1298
|
+
const fbStateRoot = wkData.workspaceRoot ?? wkData.cwd;
|
|
1299
|
+
const fbBatchId = wkData.authorizedBatchId ?? null;
|
|
1300
|
+
if (
|
|
1301
|
+
!fbBatchId ||
|
|
1302
|
+
!writeEngineIdentity(fbStateRoot, {
|
|
1303
|
+
batchId: fbBatchId,
|
|
1304
|
+
pid: process.pid,
|
|
1305
|
+
supervisorPid: process.pid,
|
|
1306
|
+
startedAt: Date.now(),
|
|
1307
|
+
})
|
|
1308
|
+
) {
|
|
1309
|
+
batchState.phase = "failed";
|
|
1310
|
+
batchState.endedAt = Date.now();
|
|
1311
|
+
batchState.errors.push(
|
|
1312
|
+
"Engine start refused: fallback engine could not publish its identity (#631)",
|
|
1313
|
+
);
|
|
1314
|
+
updateWidget();
|
|
1315
|
+
onTerminal?.();
|
|
1316
|
+
return null;
|
|
1317
|
+
}
|
|
1318
|
+
batchState.batchId = fbBatchId; // engine adopts it (fresh) / resume verifies the target (resume)
|
|
1319
|
+
fallbackEngineActive = true;
|
|
1320
|
+
startBatchAsync(fallbackFn, batchState, ctx, updateWidget, () => {
|
|
1321
|
+
fallbackEngineActive = false;
|
|
1322
|
+
markEngineExited(fbStateRoot, fbBatchId, { pid: process.pid, exitReason: "fallback-settled" });
|
|
1323
|
+
onTerminal?.();
|
|
1324
|
+
});
|
|
1198
1325
|
return null;
|
|
1199
1326
|
}
|
|
1200
1327
|
|
|
@@ -1271,6 +1398,76 @@ export function startBatchInWorker(
|
|
|
1271
1398
|
});
|
|
1272
1399
|
|
|
1273
1400
|
// Send workerData as first IPC message
|
|
1401
|
+
// #631: publish the engine's identity (pid) BEFORE the engine is told to
|
|
1402
|
+
// start. A replacement supervisor uses it to VERIFY an orphaned engine is
|
|
1403
|
+
// dead before touching an inherited batch (a dead supervisor pid does not
|
|
1404
|
+
// imply a dead engine). The batchId is preallocated by the caller for BOTH
|
|
1405
|
+
// modes (fresh: the engine adopts it; resume: the gated target), so there is
|
|
1406
|
+
// no window in which this engine runs without a current identity record.
|
|
1407
|
+
// Publication is REQUIRED: if it cannot be written, the batch cannot be
|
|
1408
|
+
// owned verifiably — kill the child and fail closed (no fallback).
|
|
1409
|
+
// Exit marking is ATTEMPT-SCOPED (pid-matched): a delayed exit callback from
|
|
1410
|
+
// an old parent cannot mark a newer live engine as exited.
|
|
1411
|
+
const engineStateRoot = wkData.workspaceRoot ?? wkData.cwd;
|
|
1412
|
+
const enginePid = typeof child.pid === "number" ? child.pid : null;
|
|
1413
|
+
const engineIdentityBatchId = wkData.authorizedBatchId ?? null;
|
|
1414
|
+
if (!engineIdentityBatchId || enginePid === null) {
|
|
1415
|
+
try {
|
|
1416
|
+
child.kill();
|
|
1417
|
+
} catch {
|
|
1418
|
+
/* best effort */
|
|
1419
|
+
}
|
|
1420
|
+
const why = !engineIdentityBatchId
|
|
1421
|
+
? "no authorized batchId was preallocated"
|
|
1422
|
+
: "child has no pid";
|
|
1423
|
+
batchState.phase = "failed";
|
|
1424
|
+
batchState.endedAt = Date.now();
|
|
1425
|
+
batchState.errors.push(`Engine start refused: ${why} (#631 ownership invariant)`);
|
|
1426
|
+
safeCtxCallFromCallback(
|
|
1427
|
+
() => ctx.ui.notify(`❌ Engine start refused: ${why}.`, "error"),
|
|
1428
|
+
"spawn.identity.notify",
|
|
1429
|
+
);
|
|
1430
|
+
updateWidget();
|
|
1431
|
+
onTerminal?.();
|
|
1432
|
+
return null;
|
|
1433
|
+
}
|
|
1434
|
+
const published = writeEngineIdentity(engineStateRoot, {
|
|
1435
|
+
batchId: engineIdentityBatchId,
|
|
1436
|
+
pid: enginePid,
|
|
1437
|
+
supervisorPid: process.pid,
|
|
1438
|
+
startedAt: Date.now(),
|
|
1439
|
+
});
|
|
1440
|
+
if (!published) {
|
|
1441
|
+
try {
|
|
1442
|
+
child.kill();
|
|
1443
|
+
} catch {
|
|
1444
|
+
/* best effort */
|
|
1445
|
+
}
|
|
1446
|
+
batchState.phase = "failed";
|
|
1447
|
+
batchState.endedAt = Date.now();
|
|
1448
|
+
batchState.errors.push(
|
|
1449
|
+
`Engine start refused: could not publish engine identity to ${engineStateRoot}/.pi/runtime/${engineIdentityBatchId}/engine.json (#631)`,
|
|
1450
|
+
);
|
|
1451
|
+
safeCtxCallFromCallback(
|
|
1452
|
+
() =>
|
|
1453
|
+
ctx.ui.notify(
|
|
1454
|
+
`❌ Engine start refused: could not write engine identity under .pi/runtime/ — fix the filesystem and retry.`,
|
|
1455
|
+
"error",
|
|
1456
|
+
),
|
|
1457
|
+
"spawn.identity.notify",
|
|
1458
|
+
);
|
|
1459
|
+
updateWidget();
|
|
1460
|
+
onTerminal?.();
|
|
1461
|
+
return null;
|
|
1462
|
+
}
|
|
1463
|
+
child.on("exit", (code: number | null) => {
|
|
1464
|
+
markEngineExited(engineStateRoot, engineIdentityBatchId, {
|
|
1465
|
+
pid: enginePid,
|
|
1466
|
+
exitCode: code,
|
|
1467
|
+
exitReason: "child-exit",
|
|
1468
|
+
});
|
|
1469
|
+
});
|
|
1470
|
+
|
|
1274
1471
|
child.send({ type: "init", data: wkData });
|
|
1275
1472
|
|
|
1276
1473
|
// Terminal settlement guard (R001 §3): ensures onTerminal fires at most once.
|
|
@@ -1285,7 +1482,10 @@ export function startBatchInWorker(
|
|
|
1285
1482
|
child.on("message", (msg: WorkerToMainMessage) => {
|
|
1286
1483
|
switch (msg.type) {
|
|
1287
1484
|
case "notify":
|
|
1288
|
-
ctx
|
|
1485
|
+
// #620: ctx may be stale (session replaced/reload, or headless -p run
|
|
1486
|
+
// finalized while this forked worker still emits IPC). Accessing
|
|
1487
|
+
// ctx.ui throws assertActive; unguarded that crashes Pi. No-op on stale.
|
|
1488
|
+
safeCtxCallFromCallback(() => ctx.ui.notify(msg.msg, msg.level), "ipc.notify");
|
|
1289
1489
|
updateWidget();
|
|
1290
1490
|
break;
|
|
1291
1491
|
|
|
@@ -1335,11 +1535,26 @@ export function startBatchInWorker(
|
|
|
1335
1535
|
batchState.errors.push(`Unhandled engine error${sourceLabel}: ${msg.message}`);
|
|
1336
1536
|
if (stackLine) batchState.errors.push(`Engine stack: ${stackLine}`);
|
|
1337
1537
|
}
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1538
|
+
// #620: persist the failed state FIRST so a dead UI sink can never
|
|
1539
|
+
// prevent the dashboard/resume from seeing the failure. The engine-
|
|
1540
|
+
// worker is dead and can't persist — we must. (Reordered ahead of the
|
|
1541
|
+
// UI notify + supervisor alert, both of which are now stale-safe.)
|
|
1542
|
+
try {
|
|
1543
|
+
saveBatchState(JSON.stringify(batchState, null, 2), wkData.cwd);
|
|
1544
|
+
} catch {
|
|
1545
|
+
/* best effort */
|
|
1546
|
+
}
|
|
1547
|
+
// #620: stale-safe — a stale ctx here must not crash Pi nor block the
|
|
1548
|
+
// supervisor alert below.
|
|
1549
|
+
safeCtxCallFromCallback(
|
|
1550
|
+
() =>
|
|
1551
|
+
ctx.ui.notify(
|
|
1552
|
+
`❌ Engine crashed with unhandled error${sourceLabel}: ${msg.message}\n` +
|
|
1553
|
+
(stackLine ? ` ${stackLine}\n` : "") +
|
|
1554
|
+
` Batch ${batchState.batchId} marked as failed.`,
|
|
1555
|
+
"error",
|
|
1556
|
+
),
|
|
1557
|
+
"ipc.error.notify",
|
|
1343
1558
|
);
|
|
1344
1559
|
// Alert supervisor — this is the PRIMARY notification path for engine
|
|
1345
1560
|
// crashes caught by uncaughtException/unhandledRejection handlers.
|
|
@@ -1371,13 +1586,6 @@ export function startBatchInWorker(
|
|
|
1371
1586
|
: undefined,
|
|
1372
1587
|
},
|
|
1373
1588
|
});
|
|
1374
|
-
// Persist failed state to disk so dashboard/resume see it.
|
|
1375
|
-
// The engine-worker is dead and can't persist — we must do it here.
|
|
1376
|
-
try {
|
|
1377
|
-
saveBatchState(JSON.stringify(batchState, null, 2), wkData.cwd);
|
|
1378
|
-
} catch {
|
|
1379
|
-
/* best effort */
|
|
1380
|
-
}
|
|
1381
1589
|
updateWidget();
|
|
1382
1590
|
break;
|
|
1383
1591
|
}
|
|
@@ -1392,9 +1600,14 @@ export function startBatchInWorker(
|
|
|
1392
1600
|
batchState.endedAt = Date.now();
|
|
1393
1601
|
batchState.errors.push(`Engine process error: ${err.message}`);
|
|
1394
1602
|
}
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1603
|
+
safeCtxCallFromCallback(
|
|
1604
|
+
() =>
|
|
1605
|
+
ctx.ui.notify(
|
|
1606
|
+
`❌ Engine process error: ${err.message}\n` +
|
|
1607
|
+
` Batch ${batchState.batchId} marked as failed.`,
|
|
1608
|
+
"error",
|
|
1609
|
+
),
|
|
1610
|
+
"child.error.notify",
|
|
1398
1611
|
);
|
|
1399
1612
|
updateWidget();
|
|
1400
1613
|
// ── TP-076: Alert supervisor about engine process error ──
|
|
@@ -1441,7 +1654,10 @@ export function startBatchInWorker(
|
|
|
1441
1654
|
batchState.endedAt = Date.now();
|
|
1442
1655
|
batchState.errors.push(`Engine process exited with code ${code}`);
|
|
1443
1656
|
}
|
|
1444
|
-
|
|
1657
|
+
safeCtxCallFromCallback(
|
|
1658
|
+
() => ctx.ui.notify(`❌ Engine process exited unexpectedly (code ${code}).`, "error"),
|
|
1659
|
+
"child.exit.notify",
|
|
1660
|
+
);
|
|
1445
1661
|
updateWidget();
|
|
1446
1662
|
// ── TP-076: Alert supervisor about unexpected engine exit ──
|
|
1447
1663
|
onSupervisorAlert?.({
|
|
@@ -1542,8 +1758,10 @@ export function buildIntegrationExecutor(
|
|
|
1542
1758
|
}
|
|
1543
1759
|
},
|
|
1544
1760
|
deleteBatchState: () => {
|
|
1761
|
+
// #631: only delete the checkpoint of the batch actually integrated — never
|
|
1762
|
+
// an unrelated persisted batch (whose engine may still be alive).
|
|
1545
1763
|
try {
|
|
1546
|
-
|
|
1764
|
+
deleteBatchStateIfOwned(stateRoot ?? repoRoot, context.batchId, context.orchBranch);
|
|
1547
1765
|
} catch {
|
|
1548
1766
|
/* best effort */
|
|
1549
1767
|
}
|
|
@@ -1567,7 +1785,7 @@ export function buildIntegrationExecutor(
|
|
|
1567
1785
|
// as the manual /orch-integrate handler.
|
|
1568
1786
|
if (result.success && result.integratedLocally && context.batchId && opId) {
|
|
1569
1787
|
try {
|
|
1570
|
-
deleteStaleBranches(repoRoot, opId, context.batchId);
|
|
1788
|
+
deleteStaleBranches(repoRoot, opId, context.batchId, stateRoot ?? repoRoot);
|
|
1571
1789
|
dropBatchAutostash(repoRoot, context.batchId);
|
|
1572
1790
|
} catch {
|
|
1573
1791
|
/* best effort — don't fail integration for cleanup errors */
|
|
@@ -1604,7 +1822,21 @@ export function buildIntegrationExecutor(
|
|
|
1604
1822
|
*
|
|
1605
1823
|
* @since TP-043
|
|
1606
1824
|
*/
|
|
1607
|
-
export function buildCiDeps(
|
|
1825
|
+
export function buildCiDeps(
|
|
1826
|
+
repoRoot: string,
|
|
1827
|
+
stateRoot?: string,
|
|
1828
|
+
/**
|
|
1829
|
+
* #631: the IMMUTABLE identity of the batch whose PR this CI lifecycle belongs
|
|
1830
|
+
* to. Post-PR cleanup runs asynchronously (after CI passes and the PR merges)
|
|
1831
|
+
* — by then a NEWER batch may have persisted its checkpoint; deletion is
|
|
1832
|
+
* bound to this identity so it can never erase that one.
|
|
1833
|
+
*/
|
|
1834
|
+
owned?: { batchId: string; orchBranch: string },
|
|
1835
|
+
): CiDeps {
|
|
1836
|
+
// Capture primitives now: a caller mutating its object later cannot change
|
|
1837
|
+
// which batch this lifecycle is allowed to clean up.
|
|
1838
|
+
const ownedBatchId = owned?.batchId;
|
|
1839
|
+
const ownedOrchBranch = owned?.orchBranch;
|
|
1608
1840
|
return {
|
|
1609
1841
|
runCommand: (cmd: string, cmdArgs: string[]) => {
|
|
1610
1842
|
try {
|
|
@@ -1627,7 +1859,13 @@ export function buildCiDeps(repoRoot: string, stateRoot?: string): CiDeps {
|
|
|
1627
1859
|
runGit: (gitArgs: string[]) => runGit(gitArgs, repoRoot),
|
|
1628
1860
|
deleteBatchState: () => {
|
|
1629
1861
|
try {
|
|
1630
|
-
|
|
1862
|
+
if (ownedBatchId !== undefined && ownedOrchBranch !== undefined) {
|
|
1863
|
+
deleteBatchStateIfOwned(stateRoot ?? repoRoot, ownedBatchId, ownedOrchBranch);
|
|
1864
|
+
return;
|
|
1865
|
+
}
|
|
1866
|
+
// No identity supplied (legacy caller): refuse rather than delete blindly.
|
|
1867
|
+
execLog("supervisor", "none", "CI cleanup skipped: no owned batch identity supplied (#631)");
|
|
1868
|
+
return;
|
|
1631
1869
|
} catch {
|
|
1632
1870
|
/* best effort */
|
|
1633
1871
|
}
|
|
@@ -1830,10 +2068,304 @@ export default function (pi: ExtensionAPI) {
|
|
|
1830
2068
|
// Tracked so pause/abort can send control messages to the engine.
|
|
1831
2069
|
let activeWorker: ChildProcess | null = null;
|
|
1832
2070
|
|
|
2071
|
+
// ── #631: inherited-batch ownership evidence ──
|
|
2072
|
+
// Set by supervisor lock takeover when this session imports a batch it did
|
|
2073
|
+
// not start. `activeWorker` answers "is an engine attached to THIS process";
|
|
2074
|
+
// this answers "what do we know about the previous owner" so the active-
|
|
2075
|
+
// phase guards can decide whether an inherited "executing" batch is a
|
|
2076
|
+
// disconnected orphan (resumable) or still being driven elsewhere (refuse).
|
|
2077
|
+
let priorSupervisor: { pid: number; alive: boolean } | null = null;
|
|
2078
|
+
|
|
2079
|
+
/**
|
|
2080
|
+
* Is an engine running IN THIS PROCESS? Covers the forked child (until it has
|
|
2081
|
+
* actually terminated — exitCode/signalCode are set by Node on termination;
|
|
2082
|
+
* `killed` only means a signal was dispatched) and the main-thread fallback.
|
|
2083
|
+
*/
|
|
2084
|
+
function engineAttachedHere(): boolean {
|
|
2085
|
+
if (isFallbackEngineActive()) return true;
|
|
2086
|
+
return (
|
|
2087
|
+
activeWorker !== null && activeWorker.exitCode === null && activeWorker.signalCode === null
|
|
2088
|
+
);
|
|
2089
|
+
}
|
|
2090
|
+
|
|
2091
|
+
/**
|
|
2092
|
+
* #631: THE ownership gate for every recovery mutation (resume / retry /
|
|
2093
|
+
* skip / force-merge / administrative pause). One rule, applied against the
|
|
2094
|
+
* ACTUAL persisted (or reconstructed) target — never against a phase this
|
|
2095
|
+
* session happens to have cached:
|
|
2096
|
+
*
|
|
2097
|
+
* 1. an engine is attached to THIS process (running or still exiting)
|
|
2098
|
+
* → refuse: wait for it to exit (or pause it) — even when the cached
|
|
2099
|
+
* phase already reads paused/failed: teardown is still in flight.
|
|
2100
|
+
* 2. the target's recorded engine is ALIVE elsewhere → refuse (double-drive).
|
|
2101
|
+
* 3. no identity recorded (or unreadable) → refuse: unknown ownership is not
|
|
2102
|
+
* confirmed shutdown; the audited orch_confirm_engine_shutdown path exists.
|
|
2103
|
+
* 4. recorded engine dead/exited (incl. operator-confirmed) → proceed; the
|
|
2104
|
+
* persisted-state eligibility rules take over.
|
|
2105
|
+
*
|
|
2106
|
+
* `force` never enters this decision. Returns a refusal message or null.
|
|
2107
|
+
*/
|
|
2108
|
+
function recoveryOwnershipGate(
|
|
2109
|
+
operation: string,
|
|
2110
|
+
stateRoot: string,
|
|
2111
|
+
target: { batchId: string; phase: string },
|
|
2112
|
+
): string | null {
|
|
2113
|
+
const decision = decideRecoveryOwnership({
|
|
2114
|
+
operation,
|
|
2115
|
+
local: {
|
|
2116
|
+
engineAttached: engineAttachedHere(),
|
|
2117
|
+
phase: orchBatchState.phase,
|
|
2118
|
+
batchId: orchBatchState.batchId,
|
|
2119
|
+
pid: activeWorker?.pid ?? (isFallbackEngineActive() ? process.pid : null),
|
|
2120
|
+
},
|
|
2121
|
+
target,
|
|
2122
|
+
liveness: assessEngineLiveness(stateRoot, target.batchId),
|
|
2123
|
+
priorSupervisor,
|
|
2124
|
+
});
|
|
2125
|
+
execLog("supervisor", target.batchId, `ownership gate: ${operation}`, {
|
|
2126
|
+
proceed: decision.proceed,
|
|
2127
|
+
reason: decision.reason.slice(0, 200),
|
|
2128
|
+
});
|
|
2129
|
+
return decision.proceed ? null : decision.reason;
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
/**
|
|
2133
|
+
* #631: the ONE root every ownership decision AND every state mutation must
|
|
2134
|
+
* use. The engine persists at `workspaceRoot ?? cwd`; a gate that authorizes
|
|
2135
|
+
* the workspace-root batch while the mutation loads the repo-root batch (or
|
|
2136
|
+
* vice versa) protects the wrong target. When the two roots differ and BOTH
|
|
2137
|
+
* hold a persisted batch with different ids, that is a conflict — refuse
|
|
2138
|
+
* rather than pick one.
|
|
2139
|
+
*/
|
|
2140
|
+
function canonicalStateRoot(fallbackCwd: string): string {
|
|
2141
|
+
return execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? fallbackCwd;
|
|
2142
|
+
}
|
|
2143
|
+
function conflictingRootsRefusal(operation: string, fallbackCwd: string): string | null {
|
|
2144
|
+
const canonical = canonicalStateRoot(fallbackCwd);
|
|
2145
|
+
const repo = execCtx?.repoRoot ?? fallbackCwd;
|
|
2146
|
+
if (canonical === repo) return null;
|
|
2147
|
+
let a: PersistedBatchState | null = null;
|
|
2148
|
+
let b: PersistedBatchState | null = null;
|
|
2149
|
+
try {
|
|
2150
|
+
a = loadBatchState(canonical);
|
|
2151
|
+
} catch {
|
|
2152
|
+
/* reported elsewhere */
|
|
2153
|
+
}
|
|
2154
|
+
try {
|
|
2155
|
+
b = loadBatchState(repo);
|
|
2156
|
+
} catch {
|
|
2157
|
+
/* reported elsewhere */
|
|
2158
|
+
}
|
|
2159
|
+
if (a && b && a.batchId !== b.batchId) {
|
|
2160
|
+
return (
|
|
2161
|
+
`❌ Cannot ${operation}: conflicting batch state — workspace root holds ${a.batchId} (${a.phase}) ` +
|
|
2162
|
+
`while the repo root holds ${b.batchId} (${b.phase}). Ownership can only be verified for one target; ` +
|
|
2163
|
+
`remove or archive the stale one (${repo}/.pi/batch-state.json is not the canonical location in workspace mode).`
|
|
2164
|
+
);
|
|
2165
|
+
}
|
|
2166
|
+
return null;
|
|
2167
|
+
}
|
|
2168
|
+
|
|
2169
|
+
/**
|
|
2170
|
+
* #631: resolve the batch a recovery operation would act on, BEFORE
|
|
2171
|
+
* authorizing it. Persisted state first; on force-resume with no state file,
|
|
2172
|
+
* the same deterministic reconstruction the engine performs (so the parent
|
|
2173
|
+
* gates the very batch the child will resume — the child verifies the match).
|
|
2174
|
+
*/
|
|
2175
|
+
function resolveRecoveryTarget(
|
|
2176
|
+
stateRoot: string,
|
|
2177
|
+
allowReconstruction: boolean,
|
|
2178
|
+
): { batchId: string; phase: string } | null {
|
|
2179
|
+
try {
|
|
2180
|
+
const persisted = loadBatchState(stateRoot);
|
|
2181
|
+
if (persisted) return { batchId: persisted.batchId, phase: persisted.phase };
|
|
2182
|
+
} catch {
|
|
2183
|
+
/* the engine reports load errors with full context */
|
|
2184
|
+
}
|
|
2185
|
+
if (!allowReconstruction) return null;
|
|
2186
|
+
try {
|
|
2187
|
+
const r = reconstructBatchStateFromRuntime(stateRoot);
|
|
2188
|
+
if (r.ok) return { batchId: r.batchId, phase: r.state.phase };
|
|
2189
|
+
} catch {
|
|
2190
|
+
/* nothing to reconstruct */
|
|
2191
|
+
}
|
|
2192
|
+
return null;
|
|
2193
|
+
}
|
|
2194
|
+
|
|
1833
2195
|
// ── Supervisor State (TP-041) ────────────────────────────────────
|
|
1834
2196
|
let supervisorState = freshSupervisorState();
|
|
1835
2197
|
let supervisorConfig: SupervisorConfig = { ...DEFAULT_SUPERVISOR_CONFIG };
|
|
1836
2198
|
|
|
2199
|
+
// ── #621: Batch-end epilogue gate ────────────────────────────────
|
|
2200
|
+
// The batch-end epilogue appends display banners via
|
|
2201
|
+
// pi.sendMessage(..., {triggerTurn:false}), which immediately splices a
|
|
2202
|
+
// custom entry into the session tree. If the interactive agent has a tool
|
|
2203
|
+
// call in flight, that splice lands between an assistant `tool_use` and its
|
|
2204
|
+
// `tool_result` and produces an Anthropic 400 that wedges the session. The
|
|
2205
|
+
// gate runs the epilogue immediately when idle, else defers it to the next
|
|
2206
|
+
// `agent_settled` boundary. `batchGeneration` tags deferred work so a newer
|
|
2207
|
+
// batch invalidates a stale pending epilogue.
|
|
2208
|
+
const noticeGate = new SupervisorNoticeGate();
|
|
2209
|
+
let batchGeneration = 0;
|
|
2210
|
+
|
|
2211
|
+
// #621: Both /orch (doOrchStart) and /orch-resume (doOrchResume) must, on
|
|
2212
|
+
// (re)start, supersede any batch-end epilogue still deferred from a previous
|
|
2213
|
+
// batch: bump the generation (so a later agent_settled no longer matches the
|
|
2214
|
+
// stale pending work) AND drop the pending closure. Extracted into one helper
|
|
2215
|
+
// so the two entry points cannot drift apart again (the original /orch-resume
|
|
2216
|
+
// gap was exactly this drift). Call immediately after freshOrchBatchState().
|
|
2217
|
+
function supersedeDeferredEpilogue(): void {
|
|
2218
|
+
batchGeneration++;
|
|
2219
|
+
noticeGate.invalidate();
|
|
2220
|
+
}
|
|
2221
|
+
|
|
2222
|
+
// #621: The batch-end epilogue, shared by /orch (doOrchStart) and
|
|
2223
|
+
// /orch-resume (doOrchResume). Appends the batch-summary / integration-skipped
|
|
2224
|
+
// banners and transitions the supervisor to routing mode. Both entry points
|
|
2225
|
+
// historically inlined identical logic differing only in `repoRoot` vs
|
|
2226
|
+
// `execCtx!.repoRoot` — the same value, since doOrchStart destructures
|
|
2227
|
+
// `const { repoRoot } = execCtx`. Deferring the WHOLE epilogue (rather than
|
|
2228
|
+
// individual sends) also protects the completed->triggerSupervisorIntegration
|
|
2229
|
+
// branch, whose progress/result messages are the same splice hazard.
|
|
2230
|
+
function runSupervisorBatchEndEpilogue(): void {
|
|
2231
|
+
// #610: re-resolve at DISPATCH time. If this batch was already integrated
|
|
2232
|
+
// (manually, or by auto-integration) — or its orch branch is simply gone —
|
|
2233
|
+
// the "ready for integration" banners are stale and must not be shown.
|
|
2234
|
+
if (orchBatchState.integratedAt) return;
|
|
2235
|
+
if (orchBatchState.phase === "completed" && orchBatchState.orchBranch && execCtx) {
|
|
2236
|
+
const branchExists = runGit(
|
|
2237
|
+
["rev-parse", "--verify", `refs/heads/${orchBatchState.orchBranch}`],
|
|
2238
|
+
execCtx.repoRoot,
|
|
2239
|
+
).ok;
|
|
2240
|
+
if (!branchExists) {
|
|
2241
|
+
process.stderr.write(
|
|
2242
|
+
`[taskplane] batch-end epilogue skipped: orch branch ${orchBatchState.orchBranch} no longer exists (already integrated) (#610)
|
|
2243
|
+
`,
|
|
2244
|
+
);
|
|
2245
|
+
return;
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
const mode = orchConfig.orchestrator.integration;
|
|
2249
|
+
const opId = resolveOperatorId(orchConfig);
|
|
2250
|
+
const sDeps: SummaryDeps = {
|
|
2251
|
+
opId,
|
|
2252
|
+
diagnostics: orchBatchState.diagnostics ?? null,
|
|
2253
|
+
mergeResults: (orchBatchState.mergeResults || []).map((mr) => ({
|
|
2254
|
+
waveIndex: mr.waveIndex,
|
|
2255
|
+
status: mr.status,
|
|
2256
|
+
failedLane: mr.failedLane,
|
|
2257
|
+
failureReason: mr.failureReason,
|
|
2258
|
+
})),
|
|
2259
|
+
};
|
|
2260
|
+
if (orchBatchState.phase === "completed" && (mode === "supervised" || mode === "auto")) {
|
|
2261
|
+
triggerSupervisorIntegration(
|
|
2262
|
+
pi,
|
|
2263
|
+
supervisorState,
|
|
2264
|
+
orchBatchState,
|
|
2265
|
+
mode,
|
|
2266
|
+
execCtx!.repoRoot,
|
|
2267
|
+
// #610: record integration in memory on success so any later epilogue
|
|
2268
|
+
// dispatch for this batch short-circuits instead of re-prompting.
|
|
2269
|
+
((mode, context) => {
|
|
2270
|
+
const r = buildIntegrationExecutor(
|
|
2271
|
+
execCtx!.repoRoot,
|
|
2272
|
+
opId,
|
|
2273
|
+
execCtx!.workspaceRoot,
|
|
2274
|
+
)(mode, context);
|
|
2275
|
+
if (r.success && r.integratedLocally && orchBatchState.batchId === context.batchId) {
|
|
2276
|
+
orchBatchState.integratedAt = Date.now();
|
|
2277
|
+
}
|
|
2278
|
+
return r;
|
|
2279
|
+
}) satisfies IntegrationExecutor,
|
|
2280
|
+
buildCiDeps(execCtx!.repoRoot, execCtx!.workspaceRoot, {
|
|
2281
|
+
batchId: orchBatchState.batchId,
|
|
2282
|
+
orchBranch: orchBatchState.orchBranch,
|
|
2283
|
+
}),
|
|
2284
|
+
sDeps,
|
|
2285
|
+
);
|
|
2286
|
+
return;
|
|
2287
|
+
}
|
|
2288
|
+
if ((mode === "supervised" || mode === "auto") && orchBatchState.phase !== "completed") {
|
|
2289
|
+
pi.sendMessage(
|
|
2290
|
+
{
|
|
2291
|
+
customType: "supervisor-integration-skipped",
|
|
2292
|
+
content: [
|
|
2293
|
+
{
|
|
2294
|
+
type: "text",
|
|
2295
|
+
text:
|
|
2296
|
+
`📋 **Batch ended** (phase: ${orchBatchState.phase}). ` +
|
|
2297
|
+
`Integration skipped — only completed batches are eligible.\n` +
|
|
2298
|
+
`Use \`/orch-resume\` to continue or \`/orch-integrate\` manually after resolving issues.`,
|
|
2299
|
+
},
|
|
2300
|
+
],
|
|
2301
|
+
display: `Integration skipped — batch ${orchBatchState.phase}`,
|
|
2302
|
+
},
|
|
2303
|
+
{ triggerTurn: false },
|
|
2304
|
+
);
|
|
2305
|
+
}
|
|
2306
|
+
presentBatchSummary(
|
|
2307
|
+
pi,
|
|
2308
|
+
orchBatchState,
|
|
2309
|
+
execCtx!.workspaceRoot,
|
|
2310
|
+
opId,
|
|
2311
|
+
orchBatchState.diagnostics,
|
|
2312
|
+
sDeps.mergeResults,
|
|
2313
|
+
);
|
|
2314
|
+
const postBatchContext: SupervisorRoutingContext =
|
|
2315
|
+
orchBatchState.phase === "completed"
|
|
2316
|
+
? {
|
|
2317
|
+
routingState: "completed-batch",
|
|
2318
|
+
contextMessage:
|
|
2319
|
+
`Batch **${orchBatchState.batchId}** completed — ` +
|
|
2320
|
+
`${orchBatchState.succeededTasks}/${orchBatchState.totalTasks} tasks succeeded.\n\n` +
|
|
2321
|
+
`The orch branch \`${orchBatchState.orchBranch}\` is ready to integrate.\n` +
|
|
2322
|
+
`Would you like me to integrate it, or would you prefer to review first?\n\n` +
|
|
2323
|
+
`You can also:\n` +
|
|
2324
|
+
`• Run \`/orch-integrate\` (or \`/orch-integrate --pr\`) to integrate\n` +
|
|
2325
|
+
`• Create new tasks for the next batch\n` +
|
|
2326
|
+
`• Run a health check`,
|
|
2327
|
+
}
|
|
2328
|
+
: {
|
|
2329
|
+
routingState: "no-tasks",
|
|
2330
|
+
contextMessage:
|
|
2331
|
+
`Batch **${orchBatchState.batchId}** ended (${orchBatchState.phase}).\n\n` +
|
|
2332
|
+
`${orchBatchState.succeededTasks} succeeded, ${orchBatchState.failedTasks} failed, ` +
|
|
2333
|
+
`${orchBatchState.skippedTasks} skipped.\n\n` +
|
|
2334
|
+
`What would you like to do next?`,
|
|
2335
|
+
};
|
|
2336
|
+
transitionToRoutingMode(pi, supervisorState, postBatchContext);
|
|
2337
|
+
}
|
|
2338
|
+
|
|
2339
|
+
// #621: Route the batch-end epilogue through the idle gate. When the agent
|
|
2340
|
+
// has a tool call in flight, eagerly stop batch monitoring (so a heartbeat
|
|
2341
|
+
// timer send can't splice either) and defer the epilogue to the next settle.
|
|
2342
|
+
function dispatchBatchEndEpilogue(ctx: ExtensionContext): void {
|
|
2343
|
+
// #620: this runs from the engine-worker terminal (onTerminal) callback,
|
|
2344
|
+
// where ctx may be stale. ctx.isIdle() is assertActive-guarded, so an
|
|
2345
|
+
// unguarded call crashes Pi. If the ctx is stale the session is gone — there
|
|
2346
|
+
// is no live UI to render the epilogue — so skip dispatch entirely, but still
|
|
2347
|
+
// stop monitoring and drop any deferred epilogue so no later timer send
|
|
2348
|
+
// fires against the dead session. Non-stale errors are logged, not rethrown
|
|
2349
|
+
// (a throw here is an uncaught IPC-callback exception — the #620 crash class).
|
|
2350
|
+
let idle: boolean;
|
|
2351
|
+
try {
|
|
2352
|
+
idle = ctx.isIdle();
|
|
2353
|
+
} catch (err) {
|
|
2354
|
+
if (!isStaleExtensionCtx(err)) {
|
|
2355
|
+
console.error(
|
|
2356
|
+
`[taskplane] dispatchBatchEndEpilogue ctx.isIdle() threw (non-stale): ${
|
|
2357
|
+
err instanceof Error ? (err.stack ?? err.message) : String(err)
|
|
2358
|
+
}`,
|
|
2359
|
+
);
|
|
2360
|
+
}
|
|
2361
|
+
stopBatchMonitoring(supervisorState);
|
|
2362
|
+
noticeGate.invalidate();
|
|
2363
|
+
return;
|
|
2364
|
+
}
|
|
2365
|
+
if (!idle) stopBatchMonitoring(supervisorState);
|
|
2366
|
+
noticeGate.runOrDefer(idle, batchGeneration, runSupervisorBatchEndEpilogue);
|
|
2367
|
+
}
|
|
2368
|
+
|
|
1837
2369
|
// TP-187 (#538): Zombie-alert filter state
|
|
1838
2370
|
// Lane numbers and agent IDs that have reached a terminal state (no-progress
|
|
1839
2371
|
// kill, hard-fail, or supervisor-takeover). Supervisor-alert IPC messages
|
|
@@ -1931,13 +2463,24 @@ export default function (pi: ExtensionAPI) {
|
|
|
1931
2463
|
const ctx = orchWidgetCtx;
|
|
1932
2464
|
const prefix = orchConfig.orchestrator.sessionPrefix;
|
|
1933
2465
|
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
2466
|
+
// #620: updateOrchWidget is the single choke point for widget refresh and is
|
|
2467
|
+
// called from BOTH synchronous command handlers (ctx fresh) AND long-lived
|
|
2468
|
+
// async engine-worker IPC callbacks (ctx may be stale after session
|
|
2469
|
+
// replacement/reload or a finalized headless -p run). ctx.ui.setWidget
|
|
2470
|
+
// accesses the assertActive-guarded ctx.ui getter, so an unguarded call from
|
|
2471
|
+
// the async path crashes Pi. Guarding here covers every caller at once; for
|
|
2472
|
+
// the sync callers the stale branch simply never triggers.
|
|
2473
|
+
safeCtxCallFromCallback(
|
|
2474
|
+
() =>
|
|
2475
|
+
ctx.ui.setWidget(
|
|
2476
|
+
"task-orchestrator",
|
|
2477
|
+
createOrchWidget(
|
|
2478
|
+
() => orchBatchState,
|
|
2479
|
+
() => latestMonitorState,
|
|
2480
|
+
prefix,
|
|
2481
|
+
),
|
|
2482
|
+
),
|
|
2483
|
+
"widget.setWidget",
|
|
1941
2484
|
);
|
|
1942
2485
|
}
|
|
1943
2486
|
|
|
@@ -2275,8 +2818,55 @@ export default function (pi: ExtensionAPI) {
|
|
|
2275
2818
|
|
|
2276
2819
|
const { repoRoot } = execCtx;
|
|
2277
2820
|
|
|
2821
|
+
// #631: a fresh start deletes stale state and launches a NEW engine. That
|
|
2822
|
+
// must never happen under an engine that is still alive — ours (cached
|
|
2823
|
+
// completed/failed but the child has not terminated yet) or another
|
|
2824
|
+
// process's (inherited terminal phase whose engine is still running).
|
|
2825
|
+
// Ownership is checked against the persisted target, not the cached phase.
|
|
2826
|
+
{
|
|
2827
|
+
const startStateRoot = canonicalStateRoot(repoRoot);
|
|
2828
|
+
const conflict = conflictingRootsRefusal("orch_start", repoRoot);
|
|
2829
|
+
if (conflict) return { message: conflict, error: true };
|
|
2830
|
+
if (engineAttachedHere()) {
|
|
2831
|
+
return {
|
|
2832
|
+
message: `⏳ This session's engine (PID ${activeWorker?.pid ?? process.pid}) for batch ${orchBatchState.batchId} is still shutting down — starting a new batch now would race its teardown. Retry in a moment.`,
|
|
2833
|
+
error: true,
|
|
2834
|
+
};
|
|
2835
|
+
}
|
|
2836
|
+
const existing = resolveRecoveryTarget(startStateRoot, false);
|
|
2837
|
+
if (existing) {
|
|
2838
|
+
const liveness = assessEngineLiveness(startStateRoot, existing.batchId);
|
|
2839
|
+
// Same rule as every recovery mutation: alive or UNKNOWN ownership refuses.
|
|
2840
|
+
// (.DONE files and terminal phases are not shutdown evidence — the engine
|
|
2841
|
+
// persists again after cleanup.) Pre-#631 batches: one-time
|
|
2842
|
+
// orch_confirm_engine_shutdown before the first fresh start.
|
|
2843
|
+
{
|
|
2844
|
+
const decision = decideRecoveryOwnership({
|
|
2845
|
+
operation: "orch_start",
|
|
2846
|
+
local: {
|
|
2847
|
+
engineAttached: false,
|
|
2848
|
+
phase: orchBatchState.phase,
|
|
2849
|
+
batchId: orchBatchState.batchId,
|
|
2850
|
+
pid: null,
|
|
2851
|
+
},
|
|
2852
|
+
target: existing,
|
|
2853
|
+
liveness,
|
|
2854
|
+
priorSupervisor,
|
|
2855
|
+
});
|
|
2856
|
+
execLog("supervisor", existing.batchId, "ownership gate: orch_start", {
|
|
2857
|
+
proceed: decision.proceed,
|
|
2858
|
+
status: liveness.status,
|
|
2859
|
+
pid: liveness.identity?.pid,
|
|
2860
|
+
});
|
|
2861
|
+
if (!decision.proceed) return { message: decision.reason, error: true };
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
}
|
|
2865
|
+
|
|
2278
2866
|
// Orphan detection
|
|
2279
|
-
|
|
2867
|
+
// #631: orphan/stale-state handling at the SAME root the gate authorized.
|
|
2868
|
+
const orphanStateRoot = canonicalStateRoot(repoRoot);
|
|
2869
|
+
const orphanResult = detectOrphanSessions(orchConfig.orchestrator.sessionPrefix, orphanStateRoot);
|
|
2280
2870
|
|
|
2281
2871
|
switch (orphanResult.recommendedAction) {
|
|
2282
2872
|
case "resume": {
|
|
@@ -2285,7 +2875,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2285
2875
|
const hasOrphans = orphanResult.orphanSessions.length > 0;
|
|
2286
2876
|
if (!hasOrphans && !resumablePhases.includes(phase)) {
|
|
2287
2877
|
try {
|
|
2288
|
-
deleteBatchState(
|
|
2878
|
+
deleteBatchState(orphanStateRoot);
|
|
2289
2879
|
} catch {
|
|
2290
2880
|
/* best effort */
|
|
2291
2881
|
}
|
|
@@ -2301,7 +2891,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2301
2891
|
return { message: orphanResult.userMessage, error: true };
|
|
2302
2892
|
case "cleanup-stale":
|
|
2303
2893
|
try {
|
|
2304
|
-
deleteBatchState(
|
|
2894
|
+
deleteBatchState(orphanStateRoot);
|
|
2305
2895
|
} catch {
|
|
2306
2896
|
/* best effort */
|
|
2307
2897
|
}
|
|
@@ -2375,8 +2965,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
2375
2965
|
|
|
2376
2966
|
// Reset batch state for new execution
|
|
2377
2967
|
orchBatchState = freshOrchBatchState();
|
|
2968
|
+
priorSupervisor = null; // #631: this session now owns the engine it is about to fork
|
|
2378
2969
|
latestMonitorState = null;
|
|
2379
2970
|
|
|
2971
|
+
// #621: a new batch supersedes any epilogue still deferred from the
|
|
2972
|
+
// previous batch. Bump the generation and drop the stale pending work.
|
|
2973
|
+
supersedeDeferredEpilogue();
|
|
2974
|
+
|
|
2380
2975
|
// TP-187 (#538): Clear zombie-alert filter for the new batch.
|
|
2381
2976
|
clearTerminationFilter("new_batch_started");
|
|
2382
2977
|
|
|
@@ -2384,12 +2979,18 @@ export default function (pi: ExtensionAPI) {
|
|
|
2384
2979
|
orchBatchState.startedAt = Date.now();
|
|
2385
2980
|
updateOrchWidget();
|
|
2386
2981
|
|
|
2982
|
+
// #631: preallocate the batchId so the engine identity is published BEFORE
|
|
2983
|
+
// the engine starts (the engine adopts it instead of generating its own).
|
|
2984
|
+
const authorizedBatchId = generateBatchId();
|
|
2985
|
+
orchBatchState.batchId = authorizedBatchId;
|
|
2986
|
+
|
|
2387
2987
|
// Non-blocking engine launch in worker thread (TP-071)
|
|
2388
2988
|
activeWorker = startBatchInWorker(
|
|
2389
2989
|
{
|
|
2390
2990
|
engineWorker: true,
|
|
2391
2991
|
mode: "execute",
|
|
2392
2992
|
args: trimmedTarget,
|
|
2993
|
+
authorizedBatchId,
|
|
2393
2994
|
orchConfig,
|
|
2394
2995
|
runnerConfig,
|
|
2395
2996
|
cwd: repoRoot,
|
|
@@ -2424,80 +3025,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2424
3025
|
if (changed) updateOrchWidget();
|
|
2425
3026
|
},
|
|
2426
3027
|
() => {
|
|
2427
|
-
|
|
2428
|
-
const opId = resolveOperatorId(orchConfig);
|
|
2429
|
-
const sDeps: SummaryDeps = {
|
|
2430
|
-
opId,
|
|
2431
|
-
diagnostics: orchBatchState.diagnostics ?? null,
|
|
2432
|
-
mergeResults: (orchBatchState.mergeResults || []).map((mr) => ({
|
|
2433
|
-
waveIndex: mr.waveIndex,
|
|
2434
|
-
status: mr.status,
|
|
2435
|
-
failedLane: mr.failedLane,
|
|
2436
|
-
failureReason: mr.failureReason,
|
|
2437
|
-
})),
|
|
2438
|
-
};
|
|
2439
|
-
if (orchBatchState.phase === "completed" && (mode === "supervised" || mode === "auto")) {
|
|
2440
|
-
triggerSupervisorIntegration(
|
|
2441
|
-
pi,
|
|
2442
|
-
supervisorState,
|
|
2443
|
-
orchBatchState,
|
|
2444
|
-
mode,
|
|
2445
|
-
repoRoot,
|
|
2446
|
-
buildIntegrationExecutor(repoRoot, opId, execCtx!.workspaceRoot),
|
|
2447
|
-
buildCiDeps(repoRoot, execCtx!.workspaceRoot),
|
|
2448
|
-
sDeps,
|
|
2449
|
-
);
|
|
2450
|
-
return;
|
|
2451
|
-
}
|
|
2452
|
-
if ((mode === "supervised" || mode === "auto") && orchBatchState.phase !== "completed") {
|
|
2453
|
-
pi.sendMessage(
|
|
2454
|
-
{
|
|
2455
|
-
customType: "supervisor-integration-skipped",
|
|
2456
|
-
content: [
|
|
2457
|
-
{
|
|
2458
|
-
type: "text",
|
|
2459
|
-
text:
|
|
2460
|
-
`📋 **Batch ended** (phase: ${orchBatchState.phase}). ` +
|
|
2461
|
-
`Integration skipped — only completed batches are eligible.\n` +
|
|
2462
|
-
`Use \`/orch-resume\` to continue or \`/orch-integrate\` manually after resolving issues.`,
|
|
2463
|
-
},
|
|
2464
|
-
],
|
|
2465
|
-
display: `Integration skipped — batch ${orchBatchState.phase}`,
|
|
2466
|
-
},
|
|
2467
|
-
{ triggerTurn: false },
|
|
2468
|
-
);
|
|
2469
|
-
}
|
|
2470
|
-
presentBatchSummary(
|
|
2471
|
-
pi,
|
|
2472
|
-
orchBatchState,
|
|
2473
|
-
execCtx!.workspaceRoot,
|
|
2474
|
-
opId,
|
|
2475
|
-
orchBatchState.diagnostics,
|
|
2476
|
-
sDeps.mergeResults,
|
|
2477
|
-
);
|
|
2478
|
-
const postBatchContext: SupervisorRoutingContext =
|
|
2479
|
-
orchBatchState.phase === "completed"
|
|
2480
|
-
? {
|
|
2481
|
-
routingState: "completed-batch",
|
|
2482
|
-
contextMessage:
|
|
2483
|
-
`Batch **${orchBatchState.batchId}** completed — ` +
|
|
2484
|
-
`${orchBatchState.succeededTasks}/${orchBatchState.totalTasks} tasks succeeded.\n\n` +
|
|
2485
|
-
`The orch branch \`${orchBatchState.orchBranch}\` is ready to integrate.\n` +
|
|
2486
|
-
`Would you like me to integrate it, or would you prefer to review first?\n\n` +
|
|
2487
|
-
`You can also:\n` +
|
|
2488
|
-
`• Run \`/orch-integrate\` (or \`/orch-integrate --pr\`) to integrate\n` +
|
|
2489
|
-
`• Create new tasks for the next batch\n` +
|
|
2490
|
-
`• Run a health check`,
|
|
2491
|
-
}
|
|
2492
|
-
: {
|
|
2493
|
-
routingState: "no-tasks",
|
|
2494
|
-
contextMessage:
|
|
2495
|
-
`Batch **${orchBatchState.batchId}** ended (${orchBatchState.phase}).\n\n` +
|
|
2496
|
-
`${orchBatchState.succeededTasks} succeeded, ${orchBatchState.failedTasks} failed, ` +
|
|
2497
|
-
`${orchBatchState.skippedTasks} skipped.\n\n` +
|
|
2498
|
-
`What would you like to do next?`,
|
|
2499
|
-
};
|
|
2500
|
-
transitionToRoutingMode(pi, supervisorState, postBatchContext);
|
|
3028
|
+
dispatchBatchEndEpilogue(ctx);
|
|
2501
3029
|
},
|
|
2502
3030
|
// ── TP-076: Supervisor alert handler — injects alerts as user messages ──
|
|
2503
3031
|
(alert) => {
|
|
@@ -2510,7 +3038,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
2510
3038
|
);
|
|
2511
3039
|
return;
|
|
2512
3040
|
}
|
|
2513
|
-
pi.sendUserMessage
|
|
3041
|
+
// #620: stale-safe — pi.sendUserMessage is assertActive-guarded on
|
|
3042
|
+
// ExtensionAPI; a stale ctx from this async worker-IPC alert callback
|
|
3043
|
+
// would otherwise crash Pi. #597 safeSendMessageFromTimer covers
|
|
3044
|
+
// sendMessage only, so use the general callback guard here.
|
|
3045
|
+
safeCtxCallFromCallback(
|
|
3046
|
+
() =>
|
|
3047
|
+
pi.sendUserMessage(alert.summary, {
|
|
3048
|
+
// #review-boundary: spiral/order escalations are urgent — steer
|
|
3049
|
+
// (interrupt current turn) so the supervisor adjudicates now;
|
|
3050
|
+
// routine alerts stay followUp (queue to next turn boundary).
|
|
3051
|
+
deliverAs: alert.category === "review-intervention-needed" ? "steer" : "followUp",
|
|
3052
|
+
}),
|
|
3053
|
+
"alert.sendUserMessage",
|
|
3054
|
+
);
|
|
2514
3055
|
},
|
|
2515
3056
|
// TP-187 (#538): Lane-terminated handler.
|
|
2516
3057
|
(info) => {
|
|
@@ -2772,7 +3313,59 @@ export default function (pi: ExtensionAPI) {
|
|
|
2772
3313
|
if (orchBatchState.phase === "paused" || orchBatchState.pauseSignal.paused) {
|
|
2773
3314
|
return ORCH_MESSAGES.pauseAlreadyPaused(orchBatchState.batchId);
|
|
2774
3315
|
}
|
|
3316
|
+
|
|
3317
|
+
// #631: no engine attached to THIS process (phase inherited at takeover).
|
|
3318
|
+
// A pauseSignal here is inert — nothing runs in-process to honor it. Decide
|
|
3319
|
+
// from verified engine liveness:
|
|
3320
|
+
// - orphan engine confirmed gone → ADMINISTRATIVE pause: persist
|
|
3321
|
+
// phase=paused on disk (the non-destructive stop the operator otherwise
|
|
3322
|
+
// had to hand-edit). Surviving worker processes, if any, are reconciled
|
|
3323
|
+
// by orch_resume (registry pid liveness), not by this call.
|
|
3324
|
+
// - engine alive elsewhere → we cannot signal it; report, do not lie.
|
|
3325
|
+
if (!engineAttachedHere()) {
|
|
3326
|
+
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? process.cwd();
|
|
3327
|
+
const refusal = recoveryOwnershipGate("orch_pause (administrative)", stateRoot, {
|
|
3328
|
+
batchId: orchBatchState.batchId,
|
|
3329
|
+
phase: orchBatchState.phase,
|
|
3330
|
+
});
|
|
3331
|
+
if (refusal) {
|
|
3332
|
+
return (
|
|
3333
|
+
`⚠️ Batch ${orchBatchState.batchId} is "${orchBatchState.phase}" but no engine is attached to this session — ` +
|
|
3334
|
+
`a pause signal here would be inert.\n${refusal}`
|
|
3335
|
+
);
|
|
3336
|
+
}
|
|
3337
|
+
const decision = { reason: "recorded engine verified gone" };
|
|
3338
|
+
try {
|
|
3339
|
+
const persisted = loadBatchState(stateRoot);
|
|
3340
|
+
if (!persisted) return "❌ No persisted batch state found to pause.";
|
|
3341
|
+
if (persisted.batchId !== orchBatchState.batchId) {
|
|
3342
|
+
return `❌ Persisted batch (${persisted.batchId}) does not match the inherited batch (${orchBatchState.batchId}); refusing to pause.`;
|
|
3343
|
+
}
|
|
3344
|
+
const prevPhase = persisted.phase;
|
|
3345
|
+
persisted.phase = "paused";
|
|
3346
|
+
persisted.updatedAt = Date.now();
|
|
3347
|
+
persisted.errors.push(
|
|
3348
|
+
`Administrative pause by replacement supervisor (PID ${process.pid}) — inherited phase "${prevPhase}" with no live engine (${decision.reason.slice(0, 160)})`,
|
|
3349
|
+
);
|
|
3350
|
+
saveBatchState(JSON.stringify(persisted, null, 2), stateRoot);
|
|
3351
|
+
orchBatchState.phase = "paused";
|
|
3352
|
+
orchBatchState.pauseSignal.paused = true;
|
|
3353
|
+
orchBatchState.pauseSignal.cause = "operator";
|
|
3354
|
+
updateOrchWidget();
|
|
3355
|
+
return (
|
|
3356
|
+
`⏸️ Batch ${orchBatchState.batchId} administratively paused (was "${prevPhase}"; ${decision.reason}).
|
|
3357
|
+
` +
|
|
3358
|
+
` No engine was running to honor a live pause; state is now resumable on disk. ` +
|
|
3359
|
+
`Any worker processes that outlived the engine are reconciled by orch_resume(force=true) ` +
|
|
3360
|
+
`(registry pid liveness → re-execute in the existing worktree).`
|
|
3361
|
+
);
|
|
3362
|
+
} catch (err) {
|
|
3363
|
+
return `❌ Administrative pause failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
3364
|
+
}
|
|
3365
|
+
}
|
|
3366
|
+
|
|
2775
3367
|
orchBatchState.pauseSignal.paused = true;
|
|
3368
|
+
orchBatchState.pauseSignal.cause = "operator"; // in-process (fallback) engine reads this directly
|
|
2776
3369
|
// TP-071: Forward pause to engine process (its pauseSignal is separate)
|
|
2777
3370
|
activeWorker?.send({ type: "pause" });
|
|
2778
3371
|
updateOrchWidget();
|
|
@@ -2795,23 +3388,39 @@ export default function (pi: ExtensionAPI) {
|
|
|
2795
3388
|
};
|
|
2796
3389
|
}
|
|
2797
3390
|
|
|
2798
|
-
//
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
) {
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
3391
|
+
// #631: resolve the ACTUAL target (persisted, or reconstructed on force with
|
|
3392
|
+
// no state file) and run the single ownership gate against it — never
|
|
3393
|
+
// against a cached phase. The resolved batchId is also the id the new
|
|
3394
|
+
// engine is authorized for (identity published before it starts).
|
|
3395
|
+
const resumeStateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
3396
|
+
const resumeTarget = resolveRecoveryTarget(resumeStateRoot, force);
|
|
3397
|
+
if (!resumeTarget) {
|
|
3398
|
+
if (engineAttachedHere()) {
|
|
3399
|
+
return {
|
|
3400
|
+
message: `❌ Cannot orch_resume: this session's engine for batch ${orchBatchState.batchId} is still running.`,
|
|
3401
|
+
error: true,
|
|
3402
|
+
};
|
|
3403
|
+
}
|
|
3404
|
+
// No target to gate: let the engine produce the canonical "no state" error.
|
|
3405
|
+
} else {
|
|
3406
|
+
const refusal = recoveryOwnershipGate("orch_resume", resumeStateRoot, resumeTarget);
|
|
3407
|
+
if (refusal) return { message: refusal, error: true };
|
|
2809
3408
|
}
|
|
3409
|
+
const resumeTargetBatchId = resumeTarget?.batchId ?? null;
|
|
2810
3410
|
|
|
2811
3411
|
// Reset batch state for resume
|
|
2812
3412
|
orchBatchState = freshOrchBatchState();
|
|
3413
|
+
priorSupervisor = null; // #631: this session now owns the engine it is about to fork
|
|
2813
3414
|
latestMonitorState = null;
|
|
2814
3415
|
|
|
3416
|
+
// #621: a resume supersedes any epilogue still deferred from the previous
|
|
3417
|
+
// batch, exactly as doOrchStart does. Without this, an epilogue deferred
|
|
3418
|
+
// mid-tool by the prior batch keeps the same batchGeneration; if the user
|
|
3419
|
+
// resumes before `agent_settled` flushes it, onSettled() sees a matching
|
|
3420
|
+
// generation and fires the stale epilogue against the resumed batch
|
|
3421
|
+
// (wrong/duplicate banner). Shared helper mirrors doOrchStart exactly.
|
|
3422
|
+
supersedeDeferredEpilogue();
|
|
3423
|
+
|
|
2815
3424
|
// TP-187 (#538): Clear zombie-alert filter so post-resume alerts pass through.
|
|
2816
3425
|
clearTerminationFilter("orch_resume_called");
|
|
2817
3426
|
|
|
@@ -2825,6 +3434,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2825
3434
|
engineWorker: true,
|
|
2826
3435
|
mode: "resume",
|
|
2827
3436
|
args: "",
|
|
3437
|
+
authorizedBatchId: resumeTargetBatchId ?? undefined,
|
|
2828
3438
|
orchConfig,
|
|
2829
3439
|
runnerConfig,
|
|
2830
3440
|
cwd: execCtx!.repoRoot,
|
|
@@ -2844,80 +3454,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
2844
3454
|
updateOrchWidget();
|
|
2845
3455
|
},
|
|
2846
3456
|
() => {
|
|
2847
|
-
|
|
2848
|
-
const opId = resolveOperatorId(orchConfig);
|
|
2849
|
-
const sDeps: SummaryDeps = {
|
|
2850
|
-
opId,
|
|
2851
|
-
diagnostics: orchBatchState.diagnostics ?? null,
|
|
2852
|
-
mergeResults: (orchBatchState.mergeResults || []).map((mr) => ({
|
|
2853
|
-
waveIndex: mr.waveIndex,
|
|
2854
|
-
status: mr.status,
|
|
2855
|
-
failedLane: mr.failedLane,
|
|
2856
|
-
failureReason: mr.failureReason,
|
|
2857
|
-
})),
|
|
2858
|
-
};
|
|
2859
|
-
if (orchBatchState.phase === "completed" && (mode === "supervised" || mode === "auto")) {
|
|
2860
|
-
triggerSupervisorIntegration(
|
|
2861
|
-
pi,
|
|
2862
|
-
supervisorState,
|
|
2863
|
-
orchBatchState,
|
|
2864
|
-
mode,
|
|
2865
|
-
execCtx!.repoRoot,
|
|
2866
|
-
buildIntegrationExecutor(execCtx!.repoRoot, opId, execCtx!.workspaceRoot),
|
|
2867
|
-
buildCiDeps(execCtx!.repoRoot, execCtx!.workspaceRoot),
|
|
2868
|
-
sDeps,
|
|
2869
|
-
);
|
|
2870
|
-
return;
|
|
2871
|
-
}
|
|
2872
|
-
if ((mode === "supervised" || mode === "auto") && orchBatchState.phase !== "completed") {
|
|
2873
|
-
pi.sendMessage(
|
|
2874
|
-
{
|
|
2875
|
-
customType: "supervisor-integration-skipped",
|
|
2876
|
-
content: [
|
|
2877
|
-
{
|
|
2878
|
-
type: "text",
|
|
2879
|
-
text:
|
|
2880
|
-
`📋 **Batch ended** (phase: ${orchBatchState.phase}). ` +
|
|
2881
|
-
`Integration skipped — only completed batches are eligible.\n` +
|
|
2882
|
-
`Use \`/orch-resume\` to continue or \`/orch-integrate\` manually after resolving issues.`,
|
|
2883
|
-
},
|
|
2884
|
-
],
|
|
2885
|
-
display: `Integration skipped — batch ${orchBatchState.phase}`,
|
|
2886
|
-
},
|
|
2887
|
-
{ triggerTurn: false },
|
|
2888
|
-
);
|
|
2889
|
-
}
|
|
2890
|
-
presentBatchSummary(
|
|
2891
|
-
pi,
|
|
2892
|
-
orchBatchState,
|
|
2893
|
-
execCtx!.workspaceRoot,
|
|
2894
|
-
opId,
|
|
2895
|
-
orchBatchState.diagnostics,
|
|
2896
|
-
sDeps.mergeResults,
|
|
2897
|
-
);
|
|
2898
|
-
const postBatchContext: SupervisorRoutingContext =
|
|
2899
|
-
orchBatchState.phase === "completed"
|
|
2900
|
-
? {
|
|
2901
|
-
routingState: "completed-batch",
|
|
2902
|
-
contextMessage:
|
|
2903
|
-
`Batch **${orchBatchState.batchId}** completed — ` +
|
|
2904
|
-
`${orchBatchState.succeededTasks}/${orchBatchState.totalTasks} tasks succeeded.\n\n` +
|
|
2905
|
-
`The orch branch \`${orchBatchState.orchBranch}\` is ready to integrate.\n` +
|
|
2906
|
-
`Would you like me to integrate it, or would you prefer to review first?\n\n` +
|
|
2907
|
-
`You can also:\n` +
|
|
2908
|
-
`• Run \`/orch-integrate\` (or \`/orch-integrate --pr\`) to integrate\n` +
|
|
2909
|
-
`• Create new tasks for the next batch\n` +
|
|
2910
|
-
`• Run a health check`,
|
|
2911
|
-
}
|
|
2912
|
-
: {
|
|
2913
|
-
routingState: "no-tasks",
|
|
2914
|
-
contextMessage:
|
|
2915
|
-
`Batch **${orchBatchState.batchId}** ended (${orchBatchState.phase}).\n\n` +
|
|
2916
|
-
`${orchBatchState.succeededTasks} succeeded, ${orchBatchState.failedTasks} failed, ` +
|
|
2917
|
-
`${orchBatchState.skippedTasks} skipped.\n\n` +
|
|
2918
|
-
`What would you like to do next?`,
|
|
2919
|
-
};
|
|
2920
|
-
transitionToRoutingMode(pi, supervisorState, postBatchContext);
|
|
3457
|
+
dispatchBatchEndEpilogue(ctx);
|
|
2921
3458
|
},
|
|
2922
3459
|
// ── TP-076: Supervisor alert handler — injects alerts as user messages ──
|
|
2923
3460
|
(alert) => {
|
|
@@ -2930,7 +3467,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
2930
3467
|
);
|
|
2931
3468
|
return;
|
|
2932
3469
|
}
|
|
2933
|
-
pi.sendUserMessage
|
|
3470
|
+
// #620: stale-safe — pi.sendUserMessage is assertActive-guarded on
|
|
3471
|
+
// ExtensionAPI; a stale ctx from this async worker-IPC alert callback
|
|
3472
|
+
// would otherwise crash Pi. #597 safeSendMessageFromTimer covers
|
|
3473
|
+
// sendMessage only, so use the general callback guard here.
|
|
3474
|
+
safeCtxCallFromCallback(
|
|
3475
|
+
() =>
|
|
3476
|
+
pi.sendUserMessage(alert.summary, {
|
|
3477
|
+
// #review-boundary: spiral/order escalations are urgent — steer
|
|
3478
|
+
// (interrupt current turn) so the supervisor adjudicates now;
|
|
3479
|
+
// routine alerts stay followUp (queue to next turn boundary).
|
|
3480
|
+
deliverAs: alert.category === "review-intervention-needed" ? "steer" : "followUp",
|
|
3481
|
+
}),
|
|
3482
|
+
"alert.sendUserMessage",
|
|
3483
|
+
);
|
|
2934
3484
|
},
|
|
2935
3485
|
// TP-187 (#538): Lane-terminated handler.
|
|
2936
3486
|
(info) => {
|
|
@@ -2984,7 +3534,13 @@ export default function (pi: ExtensionAPI) {
|
|
|
2984
3534
|
const mode: AbortMode = hard ? "hard" : "graceful";
|
|
2985
3535
|
const prefix = orchConfig.orchestrator.sessionPrefix;
|
|
2986
3536
|
|
|
2987
|
-
|
|
3537
|
+
// #631: state is read, persisted and deleted at the CANONICAL root — the same
|
|
3538
|
+
// root the ownership gate below authorizes against.
|
|
3539
|
+
const stateRoot = canonicalStateRoot(ctx.cwd);
|
|
3540
|
+
{
|
|
3541
|
+
const conflict = conflictingRootsRefusal("orch_abort", ctx.cwd);
|
|
3542
|
+
if (conflict) return conflict;
|
|
3543
|
+
}
|
|
2988
3544
|
const messages: string[] = [`🛑 Abort requested (${mode} mode, prefix: ${prefix})...`];
|
|
2989
3545
|
|
|
2990
3546
|
// Step 1: Write abort signal file
|
|
@@ -3006,18 +3562,87 @@ export default function (pi: ExtensionAPI) {
|
|
|
3006
3562
|
// Step 2: Set pause signal and forward to worker
|
|
3007
3563
|
if (orchBatchState.pauseSignal) {
|
|
3008
3564
|
orchBatchState.pauseSignal.paused = true;
|
|
3565
|
+
orchBatchState.pauseSignal.cause = "abort";
|
|
3009
3566
|
messages.push(" ✓ Pause signal set on in-memory batch state");
|
|
3010
3567
|
}
|
|
3011
|
-
//
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3568
|
+
// ── #631: ownership + verified shutdown BEFORE any destructive step ──
|
|
3569
|
+
// Abort persists `stopped` and deletes batch state. That may only happen
|
|
3570
|
+
// once no engine can still be writing that state:
|
|
3571
|
+
// - engine attached HERE → cooperatively pause, then VERIFY it has exited
|
|
3572
|
+
// (graceful: within the grace period, then escalate; hard: kill now);
|
|
3573
|
+
// the main-thread fallback must have settled. Unverified → refuse cleanup.
|
|
3574
|
+
// - engine elsewhere → same rule as every recovery mutation: alive or
|
|
3575
|
+
// unknown (no identity) → refuse; verified dead/exited → proceed.
|
|
3576
|
+
const ownershipRoot = stateRoot;
|
|
3577
|
+
if (!engineAttachedHere()) {
|
|
3578
|
+
const target = resolveRecoveryTarget(ownershipRoot, false);
|
|
3579
|
+
if (target) {
|
|
3580
|
+
const decision = decideRecoveryOwnership({
|
|
3581
|
+
operation: "orch_abort",
|
|
3582
|
+
local: {
|
|
3583
|
+
engineAttached: false,
|
|
3584
|
+
phase: orchBatchState.phase,
|
|
3585
|
+
batchId: orchBatchState.batchId,
|
|
3586
|
+
pid: null,
|
|
3587
|
+
},
|
|
3588
|
+
target,
|
|
3589
|
+
liveness: assessEngineLiveness(ownershipRoot, target.batchId),
|
|
3590
|
+
priorSupervisor,
|
|
3591
|
+
});
|
|
3592
|
+
if (!decision.proceed) {
|
|
3593
|
+
try {
|
|
3594
|
+
unlinkSync(abortSignalFile);
|
|
3595
|
+
} catch {}
|
|
3596
|
+
return `${decision.reason}\n (abort here cannot stop that engine; it would only delete state underneath it)`;
|
|
3597
|
+
}
|
|
3598
|
+
}
|
|
3599
|
+
} else if (activeWorker) {
|
|
3600
|
+
// Forked engine in this session.
|
|
3601
|
+
const child = activeWorker;
|
|
3602
|
+
child.send({ type: "pause" });
|
|
3603
|
+
const graceMs = Math.max(0, orchConfig.failure.abort_grace_period * 1000);
|
|
3604
|
+
let exited = false;
|
|
3605
|
+
if (!hard) {
|
|
3606
|
+
messages.push(
|
|
3607
|
+
` ✓ Pause signal forwarded to engine process — waiting up to ${Math.round(graceMs / 1000)}s for it to checkpoint and exit`,
|
|
3608
|
+
);
|
|
3609
|
+
exited = await waitForChildExit(child, graceMs);
|
|
3610
|
+
}
|
|
3611
|
+
if (!exited) {
|
|
3612
|
+
child.kill();
|
|
3613
|
+
exited = await waitForChildExit(child, 5_000);
|
|
3614
|
+
}
|
|
3615
|
+
if (!exited) {
|
|
3616
|
+
try {
|
|
3617
|
+
child.kill("SIGKILL");
|
|
3618
|
+
} catch {}
|
|
3619
|
+
exited = await waitForChildExit(child, 3_000);
|
|
3620
|
+
}
|
|
3621
|
+
if (!exited) {
|
|
3622
|
+
return (
|
|
3623
|
+
`❌ Abort: engine process (PID ${child.pid}) is still alive after pause${hard ? "" : ` (${Math.round(graceMs / 1000)}s grace)`}, SIGTERM and SIGKILL. ` +
|
|
3624
|
+
`Refusing to persist/delete batch state underneath a live engine. Terminate it manually and re-run orch_abort.`
|
|
3625
|
+
);
|
|
3626
|
+
}
|
|
3627
|
+
activeWorker = null;
|
|
3628
|
+
messages.push(
|
|
3629
|
+
` ✓ Engine process exit verified (PID ${child.pid}${hard ? ", hard abort" : ""})`,
|
|
3630
|
+
);
|
|
3631
|
+
} else {
|
|
3632
|
+
// Main-thread fallback engine: it shares orchBatchState, so the pause
|
|
3633
|
+
// signal above reaches it directly; wait for it to settle.
|
|
3634
|
+
const graceMs = Math.max(2_000, orchConfig.failure.abort_grace_period * 1000);
|
|
3635
|
+
const deadline = Date.now() + graceMs;
|
|
3636
|
+
while (isFallbackEngineActive() && Date.now() < deadline) {
|
|
3637
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
3638
|
+
}
|
|
3639
|
+
if (isFallbackEngineActive()) {
|
|
3640
|
+
return (
|
|
3641
|
+
`❌ Abort: the in-process (fallback) engine has not settled within ${Math.round(graceMs / 1000)}s of the pause signal. ` +
|
|
3642
|
+
`Refusing to persist/delete batch state underneath it. Wait for it to pause, then re-run orch_abort.`
|
|
3643
|
+
);
|
|
3020
3644
|
}
|
|
3645
|
+
messages.push(" ✓ In-process engine settled");
|
|
3021
3646
|
}
|
|
3022
3647
|
|
|
3023
3648
|
const hasActiveBatch =
|
|
@@ -3140,6 +3765,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3140
3765
|
const pausablePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
3141
3766
|
if (pausablePhases.has(orchBatchState.phase)) {
|
|
3142
3767
|
orchBatchState.pauseSignal.paused = true;
|
|
3768
|
+
orchBatchState.pauseSignal.cause = "operator";
|
|
3143
3769
|
activeWorker?.send({ type: "pause" });
|
|
3144
3770
|
messages.push(` ✓ Wave paused (batch ${orchBatchState.batchId})`);
|
|
3145
3771
|
} else {
|
|
@@ -3212,12 +3838,6 @@ export default function (pi: ExtensionAPI) {
|
|
|
3212
3838
|
* The engine picks up the state change on its next poll cycle.
|
|
3213
3839
|
*/
|
|
3214
3840
|
function doOrchRetryTask(taskId: string, ctx: ExtensionContext): string {
|
|
3215
|
-
// TP-077 R001-1: Reject while engine is actively running (no IPC retry path)
|
|
3216
|
-
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
3217
|
-
if (activePhases.has(orchBatchState.phase)) {
|
|
3218
|
-
return `❌ Cannot retry task while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
3219
|
-
}
|
|
3220
|
-
|
|
3221
3841
|
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
3222
3842
|
|
|
3223
3843
|
// Load persisted state
|
|
@@ -3232,6 +3852,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
3232
3852
|
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
3233
3853
|
}
|
|
3234
3854
|
|
|
3855
|
+
// #631: single ownership gate against the PERSISTED target (TP-077 R001-1's
|
|
3856
|
+
// "reject while the engine is running" is case 1 of the gate).
|
|
3857
|
+
{
|
|
3858
|
+
const refusal = recoveryOwnershipGate("orch_retry_task", stateRoot, {
|
|
3859
|
+
batchId: state.batchId,
|
|
3860
|
+
phase: state.phase,
|
|
3861
|
+
});
|
|
3862
|
+
if (refusal) return refusal;
|
|
3863
|
+
}
|
|
3864
|
+
|
|
3235
3865
|
// Find the task
|
|
3236
3866
|
const taskRecord = state.tasks.find((t) => t.taskId === taskId);
|
|
3237
3867
|
if (!taskRecord) {
|
|
@@ -3239,12 +3869,24 @@ export default function (pi: ExtensionAPI) {
|
|
|
3239
3869
|
return `❌ Task "${taskId}" not found in batch ${state.batchId}.\nKnown tasks: ${knownIds || "(none)"}`;
|
|
3240
3870
|
}
|
|
3241
3871
|
|
|
3242
|
-
// Validate:
|
|
3243
|
-
|
|
3244
|
-
|
|
3872
|
+
// Validate: failed, stalled or SKIPPED tasks can be retried. Skipped is
|
|
3873
|
+
// included because a runtime defect (pause → skipped, penster
|
|
3874
|
+
// 20260906T194514) left tasks skipped that never ran; the documented
|
|
3875
|
+
// recovery (retry → resume) must be able to start from that state.
|
|
3876
|
+
if (
|
|
3877
|
+
taskRecord.status !== "failed" &&
|
|
3878
|
+
taskRecord.status !== "stalled" &&
|
|
3879
|
+
taskRecord.status !== "skipped"
|
|
3880
|
+
) {
|
|
3881
|
+
return `❌ Cannot retry task "${taskId}" — current status is "${taskRecord.status}". Only failed, stalled or skipped tasks can be retried.`;
|
|
3245
3882
|
}
|
|
3246
3883
|
|
|
3247
3884
|
const prevStatus = taskRecord.status;
|
|
3885
|
+
// Preserve recovery provenance: a saved partial-progress branch is the only
|
|
3886
|
+
// record of work whose worktree may already be gone. Report it instead of
|
|
3887
|
+
// silently clearing it.
|
|
3888
|
+
const preservedBranch = taskRecord.partialProgressBranch;
|
|
3889
|
+
const preservedCommits = taskRecord.partialProgressCommits;
|
|
3248
3890
|
|
|
3249
3891
|
// Reset task to pending
|
|
3250
3892
|
taskRecord.status = "pending";
|
|
@@ -3253,12 +3895,21 @@ export default function (pi: ExtensionAPI) {
|
|
|
3253
3895
|
taskRecord.startedAt = null;
|
|
3254
3896
|
taskRecord.endedAt = null;
|
|
3255
3897
|
taskRecord.exitDiagnostic = undefined;
|
|
3256
|
-
|
|
3257
|
-
|
|
3898
|
+
// Keep partialProgressBranch/Commits: they are recovery PROVENANCE (a saved
|
|
3899
|
+
// ref may be the only copy of the work); the engine overwrites them when the
|
|
3900
|
+
// re-executed task produces new partial progress.
|
|
3901
|
+
|
|
3902
|
+
// #629: on the v2 runtime the SEGMENT record is authoritative — resume's
|
|
3903
|
+
// reconstructSegmentFrontier() re-derives task status from segments, so a
|
|
3904
|
+
// task-only reset is silently undone and the wave is counted as done.
|
|
3905
|
+
// Reset the failed/stalled segments too (worktree identity preserved).
|
|
3906
|
+
const segmentReset = resetTaskSegmentsForRetry(state, taskId);
|
|
3258
3907
|
|
|
3259
3908
|
// Adjust counters: only decrement failedTasks if the task was in a failure state
|
|
3260
3909
|
if (prevStatus === "failed" || prevStatus === "stalled") {
|
|
3261
3910
|
state.failedTasks = Math.max(0, state.failedTasks - 1);
|
|
3911
|
+
} else if (prevStatus === "skipped") {
|
|
3912
|
+
state.skippedTasks = Math.max(0, (state.skippedTasks ?? 0) - 1);
|
|
3262
3913
|
}
|
|
3263
3914
|
|
|
3264
3915
|
// Recompute blocked dependents — the retried task is no longer a failure,
|
|
@@ -3273,6 +3924,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3273
3924
|
const newBlocked = computeTransitiveDependents(
|
|
3274
3925
|
remainingFailures,
|
|
3275
3926
|
orchBatchState.dependencyGraph,
|
|
3927
|
+
batchTaskScope(state.wavePlan),
|
|
3276
3928
|
);
|
|
3277
3929
|
state.blockedTaskIds = [...newBlocked].sort();
|
|
3278
3930
|
state.blockedTasks = newBlocked.size;
|
|
@@ -3286,6 +3938,19 @@ export default function (pi: ExtensionAPI) {
|
|
|
3286
3938
|
if (state.phase === "failed") {
|
|
3287
3939
|
state.phase = "stopped";
|
|
3288
3940
|
}
|
|
3941
|
+
// A "completed" batch that wrongly skipped a task (the pause→skipped defect)
|
|
3942
|
+
// must be re-openable: retrying a task in a completed batch moves the batch
|
|
3943
|
+
// to "stopped" (resumable with force). The ownership gate above already
|
|
3944
|
+
// verified no engine is driving it; refuse if the batch was integrated.
|
|
3945
|
+
let reopenedCompleted = false;
|
|
3946
|
+
if (state.phase === "completed") {
|
|
3947
|
+
if (orchBatchState.batchId === state.batchId && orchBatchState.integratedAt) {
|
|
3948
|
+
return `❌ Cannot retry "${taskId}": batch ${state.batchId} has already been integrated. Start a new batch for follow-up work.`;
|
|
3949
|
+
}
|
|
3950
|
+
state.phase = "stopped";
|
|
3951
|
+
state.endedAt = null;
|
|
3952
|
+
reopenedCompleted = true;
|
|
3953
|
+
}
|
|
3289
3954
|
|
|
3290
3955
|
// Update timestamp
|
|
3291
3956
|
state.updatedAt = Date.now();
|
|
@@ -3313,9 +3978,20 @@ export default function (pi: ExtensionAPI) {
|
|
|
3313
3978
|
state.phase === "stopped"
|
|
3314
3979
|
? "Use orch_resume(force=true) to re-execute the batch."
|
|
3315
3980
|
: "Use orch_resume() to re-execute the batch.";
|
|
3981
|
+
const provenanceNote = preservedBranch
|
|
3982
|
+
? ` Preserved progress: branch ${preservedBranch}${preservedCommits ? ` (${preservedCommits} commit(s))` : ""} — the worktree may be recreated from the base; inspect/cherry-pick that branch if the work must carry forward.\n`
|
|
3983
|
+
: "";
|
|
3984
|
+
const reopenNote = reopenedCompleted
|
|
3985
|
+
? ` Batch was "completed" — reopened as "stopped" (task ${taskId} had been ${prevStatus}).\n`
|
|
3986
|
+
: "";
|
|
3316
3987
|
return (
|
|
3317
3988
|
`✅ Task "${taskId}" reset to pending for re-execution.\n` +
|
|
3318
3989
|
` Previous status: ${prevStatus}\n` +
|
|
3990
|
+
(segmentReset.resetSegmentIds.length > 0
|
|
3991
|
+
? ` Segments reset: ${segmentReset.resetSegmentIds.join(", ")}${segmentReset.preservedSegmentIds.length > 0 ? ` (preserved: ${segmentReset.preservedSegmentIds.join(", ")})` : ""}\n`
|
|
3992
|
+
: "") +
|
|
3993
|
+
reopenNote +
|
|
3994
|
+
provenanceNote +
|
|
3319
3995
|
` Batch phase: ${state.phase} | Failed: ${state.failedTasks}/${state.totalTasks}\n` +
|
|
3320
3996
|
` ${resumeHint}`
|
|
3321
3997
|
);
|
|
@@ -3328,12 +4004,6 @@ export default function (pi: ExtensionAPI) {
|
|
|
3328
4004
|
* The engine picks up the state change on its next poll cycle.
|
|
3329
4005
|
*/
|
|
3330
4006
|
function doOrchSkipTask(taskId: string, ctx: ExtensionContext): string {
|
|
3331
|
-
// TP-077 R001-1: Reject while engine is actively running (no IPC skip path)
|
|
3332
|
-
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
3333
|
-
if (activePhases.has(orchBatchState.phase)) {
|
|
3334
|
-
return `❌ Cannot skip task while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
3335
|
-
}
|
|
3336
|
-
|
|
3337
4007
|
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
3338
4008
|
|
|
3339
4009
|
// Load persisted state
|
|
@@ -3348,6 +4018,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
3348
4018
|
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
3349
4019
|
}
|
|
3350
4020
|
|
|
4021
|
+
// #631: single ownership gate against the PERSISTED target (TP-077 R001-1's
|
|
4022
|
+
// "reject while the engine is running" is case 1 of the gate).
|
|
4023
|
+
{
|
|
4024
|
+
const refusal = recoveryOwnershipGate("orch_skip_task", stateRoot, {
|
|
4025
|
+
batchId: state.batchId,
|
|
4026
|
+
phase: state.phase,
|
|
4027
|
+
});
|
|
4028
|
+
if (refusal) return refusal;
|
|
4029
|
+
}
|
|
4030
|
+
|
|
3351
4031
|
// Find the task
|
|
3352
4032
|
const taskRecord = state.tasks.find((t) => t.taskId === taskId);
|
|
3353
4033
|
if (!taskRecord) {
|
|
@@ -3371,6 +4051,8 @@ export default function (pi: ExtensionAPI) {
|
|
|
3371
4051
|
taskRecord.status = "skipped";
|
|
3372
4052
|
taskRecord.exitReason = "Skipped by supervisor";
|
|
3373
4053
|
taskRecord.endedAt = Date.now();
|
|
4054
|
+
// #629: keep segment records in agreement (segment authority on v2).
|
|
4055
|
+
markTaskSegmentsSkipped(state, taskId, taskRecord.endedAt);
|
|
3374
4056
|
|
|
3375
4057
|
// Adjust counters
|
|
3376
4058
|
state.skippedTasks = (state.skippedTasks ?? 0) + 1;
|
|
@@ -3396,6 +4078,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3396
4078
|
const newBlocked = computeTransitiveDependents(
|
|
3397
4079
|
remainingFailures,
|
|
3398
4080
|
orchBatchState.dependencyGraph,
|
|
4081
|
+
batchTaskScope(state.wavePlan),
|
|
3399
4082
|
);
|
|
3400
4083
|
|
|
3401
4084
|
// Find tasks that were blocked but are now unblocked
|
|
@@ -3478,12 +4161,6 @@ export default function (pi: ExtensionAPI) {
|
|
|
3478
4161
|
skipFailed: boolean,
|
|
3479
4162
|
ctx: ExtensionContext,
|
|
3480
4163
|
): string {
|
|
3481
|
-
// Reject while engine is actively running
|
|
3482
|
-
const activePhases = new Set(["launching", "executing", "merging", "planning"]);
|
|
3483
|
-
if (activePhases.has(orchBatchState.phase)) {
|
|
3484
|
-
return `❌ Cannot force merge while batch is ${orchBatchState.phase}. Pause or wait for the current operation to finish first.`;
|
|
3485
|
-
}
|
|
3486
|
-
|
|
3487
4164
|
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
3488
4165
|
|
|
3489
4166
|
// Load persisted state
|
|
@@ -3498,6 +4175,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
3498
4175
|
return "❌ No batch state found. There is no active or recent batch to modify.";
|
|
3499
4176
|
}
|
|
3500
4177
|
|
|
4178
|
+
// #631: single ownership gate against the PERSISTED target (TP-077 R001-1's
|
|
4179
|
+
// "reject while the engine is running" is case 1 of the gate).
|
|
4180
|
+
{
|
|
4181
|
+
const refusal = recoveryOwnershipGate("orch_force_merge", stateRoot, {
|
|
4182
|
+
batchId: state.batchId,
|
|
4183
|
+
phase: state.phase,
|
|
4184
|
+
});
|
|
4185
|
+
if (refusal) return refusal;
|
|
4186
|
+
}
|
|
4187
|
+
|
|
3501
4188
|
// Force-merge is a recovery action for non-running failed/paused batches.
|
|
3502
4189
|
const resumablePhases = new Set(["paused", "stopped", "failed"]);
|
|
3503
4190
|
if (!resumablePhases.has(state.phase)) {
|
|
@@ -3608,6 +4295,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3608
4295
|
const newBlocked = computeTransitiveDependents(
|
|
3609
4296
|
remainingFailures,
|
|
3610
4297
|
orchBatchState.dependencyGraph,
|
|
4298
|
+
batchTaskScope(state.wavePlan),
|
|
3611
4299
|
);
|
|
3612
4300
|
state.blockedTaskIds = [...newBlocked].sort();
|
|
3613
4301
|
state.blockedTasks = newBlocked.size;
|
|
@@ -3702,8 +4390,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
3702
4390
|
// Resolve integration context
|
|
3703
4391
|
const { repoRoot } = execCtx!;
|
|
3704
4392
|
const stateRoot = execCtx!.workspaceRoot;
|
|
4393
|
+
// #631: batch state is resolved at the CANONICAL root (workspace root in
|
|
4394
|
+
// workspace mode) — the same root the ownership gate authorizes against.
|
|
4395
|
+
{
|
|
4396
|
+
const conflict = conflictingRootsRefusal("orch_integrate", repoRoot);
|
|
4397
|
+
if (conflict) return { message: conflict, error: true };
|
|
4398
|
+
}
|
|
3705
4399
|
const resolution = resolveIntegrationContext(parsed, {
|
|
3706
|
-
loadBatchState: () => loadBatchState(repoRoot),
|
|
4400
|
+
loadBatchState: () => loadBatchState(stateRoot ?? repoRoot),
|
|
3707
4401
|
getCurrentBranch: () => getCurrentBranch(repoRoot),
|
|
3708
4402
|
listOrchBranches: () => {
|
|
3709
4403
|
const result = runGit(["branch", "--list", "orch/*"], repoRoot);
|
|
@@ -3724,11 +4418,83 @@ export default function (pi: ExtensionAPI) {
|
|
|
3724
4418
|
return { message: resolution.error, error: severity !== "info" };
|
|
3725
4419
|
}
|
|
3726
4420
|
|
|
3727
|
-
const { orchBranch, baseBranch,
|
|
3728
|
-
|
|
4421
|
+
const { orchBranch, baseBranch, currentBranch, notices } = resolution as IntegrationContext;
|
|
4422
|
+
let batchId = (resolution as IntegrationContext).batchId;
|
|
3729
4423
|
const outputLines: string[] = [];
|
|
3730
4424
|
let hasWarning = false;
|
|
3731
4425
|
|
|
4426
|
+
// #631: integration merges and cleans up the orch branch. A "completed"
|
|
4427
|
+
// phase on disk is not proof the engine has finished tearing down (or that
|
|
4428
|
+
// an inherited batch's engine is gone). Refuse while it is verifiably alive.
|
|
4429
|
+
// (dead/exited/none proceed — integration does not drive the engine.)
|
|
4430
|
+
{
|
|
4431
|
+
if (engineAttachedHere()) {
|
|
4432
|
+
return {
|
|
4433
|
+
message: `⏳ This session's engine (PID ${activeWorker?.pid ?? process.pid}) is still running/shutting down — integrate after it exits.`,
|
|
4434
|
+
error: true,
|
|
4435
|
+
};
|
|
4436
|
+
}
|
|
4437
|
+
// Ownership is looked up BY THE SELECTED BRANCH at the canonical state root —
|
|
4438
|
+
// not by whichever batch happens to be persisted or reconstructable. The
|
|
4439
|
+
// batch behind `orchBranch` may have no batch-state.json (aborted) and may
|
|
4440
|
+
// not be reconstructable (worker manifests gone) while its engine identity
|
|
4441
|
+
// still records a live pid. Every associated batch must pass the full rule
|
|
4442
|
+
// (alive/none → refuse; dead/exited → proceed). Persisted state for the same
|
|
4443
|
+
// branch is included; a persisted batch for a DIFFERENT branch is irrelevant.
|
|
4444
|
+
const integRoot = stateRoot ?? repoRoot;
|
|
4445
|
+
const associated = new Map<
|
|
4446
|
+
string,
|
|
4447
|
+
{ batchId: string; phase: string; liveness: ReturnType<typeof assessEngineLiveness> }
|
|
4448
|
+
>();
|
|
4449
|
+
for (const b of findBatchesForOrchBranch(integRoot, orchBranch)) {
|
|
4450
|
+
associated.set(b.batchId, { batchId: b.batchId, phase: "unknown", liveness: b.liveness });
|
|
4451
|
+
}
|
|
4452
|
+
try {
|
|
4453
|
+
const persisted = loadBatchState(integRoot);
|
|
4454
|
+
if (persisted && (persisted.orchBranch === orchBranch || persisted.batchId === batchId)) {
|
|
4455
|
+
associated.set(persisted.batchId, {
|
|
4456
|
+
batchId: persisted.batchId,
|
|
4457
|
+
phase: persisted.phase,
|
|
4458
|
+
liveness: assessEngineLiveness(integRoot, persisted.batchId),
|
|
4459
|
+
});
|
|
4460
|
+
}
|
|
4461
|
+
} catch {
|
|
4462
|
+
/* reported elsewhere */
|
|
4463
|
+
}
|
|
4464
|
+
if (batchId && !associated.has(batchId)) {
|
|
4465
|
+
associated.set(batchId, {
|
|
4466
|
+
batchId,
|
|
4467
|
+
phase: "completed",
|
|
4468
|
+
liveness: assessEngineLiveness(integRoot, batchId),
|
|
4469
|
+
});
|
|
4470
|
+
}
|
|
4471
|
+
for (const target of associated.values()) {
|
|
4472
|
+
const decision = decideRecoveryOwnership({
|
|
4473
|
+
operation: "orch_integrate",
|
|
4474
|
+
local: {
|
|
4475
|
+
engineAttached: false,
|
|
4476
|
+
phase: orchBatchState.phase,
|
|
4477
|
+
batchId: orchBatchState.batchId,
|
|
4478
|
+
pid: null,
|
|
4479
|
+
},
|
|
4480
|
+
target: { batchId: target.batchId, phase: target.phase },
|
|
4481
|
+
liveness: target.liveness,
|
|
4482
|
+
priorSupervisor,
|
|
4483
|
+
});
|
|
4484
|
+
if (!decision.proceed) return { message: decision.reason, error: true };
|
|
4485
|
+
}
|
|
4486
|
+
// No batch is associated with this branch (pure branch integration):
|
|
4487
|
+
// nothing an engine could be driving — proceed.
|
|
4488
|
+
|
|
4489
|
+
// Bind cleanup/history to the batch behind THIS branch. The resolver
|
|
4490
|
+
// leaves batchId empty when an explicit branch differs from persisted
|
|
4491
|
+
// state; a unique associated runtime batch fills it in. Ambiguous (>1)
|
|
4492
|
+
// → leave empty: batch-scoped cleanup is skipped rather than guessed.
|
|
4493
|
+
if (!batchId && associated.size === 1) {
|
|
4494
|
+
batchId = [...associated.values()][0].batchId;
|
|
4495
|
+
}
|
|
4496
|
+
}
|
|
4497
|
+
|
|
3732
4498
|
for (const notice of notices) {
|
|
3733
4499
|
outputLines.push(notice);
|
|
3734
4500
|
}
|
|
@@ -3863,7 +4629,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
3863
4629
|
|
|
3864
4630
|
const branchCleanupLines: string[] = [];
|
|
3865
4631
|
for (const repo of allRepos) {
|
|
3866
|
-
const branchCleanup = deleteStaleBranches(repo.root, opId, batchId);
|
|
4632
|
+
const branchCleanup = deleteStaleBranches(repo.root, opId, batchId, stateRoot ?? repoRoot);
|
|
3867
4633
|
const totalDeleted =
|
|
3868
4634
|
branchCleanup.deletedTaskBranches.length + branchCleanup.deletedSavedBranches.length;
|
|
3869
4635
|
if (totalDeleted > 0 || branchCleanup.failedDeletes.length > 0) {
|
|
@@ -3910,6 +4676,17 @@ export default function (pi: ExtensionAPI) {
|
|
|
3910
4676
|
hasWarning = true;
|
|
3911
4677
|
}
|
|
3912
4678
|
|
|
4679
|
+
// #610: the batch is integrated. Any batch-end epilogue still DEFERRED behind
|
|
4680
|
+
// this turn (the engine finished while the supervisor was mid-turn, #621)
|
|
4681
|
+
// would fire at agent_settled with stale content — "Ready for integration /
|
|
4682
|
+
// run orch_integrate()" or the supervised "Integration Plan … merge commit"
|
|
4683
|
+
// prompt for an orch branch that no longer exists. Supersede it now, and mark
|
|
4684
|
+
// the in-memory batch integrated so a not-yet-deferred dispatch also skips.
|
|
4685
|
+
if (orchBatchState.batchId === batchId || !batchId) {
|
|
4686
|
+
orchBatchState.integratedAt = Date.now();
|
|
4687
|
+
}
|
|
4688
|
+
supersedeDeferredEpilogue();
|
|
4689
|
+
|
|
3913
4690
|
// TP-179: Write integratedAt to batch history before deleting state
|
|
3914
4691
|
if (batchId) {
|
|
3915
4692
|
try {
|
|
@@ -3919,8 +4696,16 @@ export default function (pi: ExtensionAPI) {
|
|
|
3919
4696
|
}
|
|
3920
4697
|
}
|
|
3921
4698
|
|
|
4699
|
+
// #631: delete persisted state ONLY if it belongs to the integrated branch/batch.
|
|
4700
|
+
// An explicit branch integration must not erase an unrelated batch's recovery
|
|
4701
|
+
// checkpoint (its engine may even still be alive).
|
|
3922
4702
|
try {
|
|
3923
|
-
|
|
4703
|
+
const deleted = deleteBatchStateIfOwned(stateRoot ?? repoRoot, batchId, orchBranch);
|
|
4704
|
+
if (deleted === false) {
|
|
4705
|
+
outputLines.push(
|
|
4706
|
+
`ℹ️ Persisted batch state belongs to a different batch/branch — left in place (not part of this integration).`,
|
|
4707
|
+
);
|
|
4708
|
+
}
|
|
3924
4709
|
} catch {
|
|
3925
4710
|
/* best effort */
|
|
3926
4711
|
}
|
|
@@ -4161,6 +4946,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
4161
4946
|
|
|
4162
4947
|
ctx.ui.notify(`🔄 **${reason}** Activating supervisor.\n\n` + summary, "info");
|
|
4163
4948
|
|
|
4949
|
+
// #631: record what we know about the previous owner. The engine of a
|
|
4950
|
+
// batch we did not start is NOT attached to this process; the active-
|
|
4951
|
+
// phase guards use this + engine.json to decide whether an inherited
|
|
4952
|
+
// "executing" phase is a resumable orphan.
|
|
4953
|
+
priorSupervisor =
|
|
4954
|
+
"lock" in lockResult && lockResult.lock
|
|
4955
|
+
? { pid: lockResult.lock.pid, alive: isProcessAlive(lockResult.lock.pid) }
|
|
4956
|
+
: null;
|
|
4164
4957
|
// Populate orchBatchState from persisted state
|
|
4165
4958
|
orchBatchState.batchId = batchState.batchId;
|
|
4166
4959
|
orchBatchState.phase = batchState.phase as typeof orchBatchState.phase;
|
|
@@ -4207,6 +5000,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
4207
5000
|
"warning",
|
|
4208
5001
|
);
|
|
4209
5002
|
|
|
5003
|
+
// #631: record what we know about the previous owner. The engine of a
|
|
5004
|
+
// batch we did not start is NOT attached to this process; the active-
|
|
5005
|
+
// phase guards use this + engine.json to decide whether an inherited
|
|
5006
|
+
// "executing" phase is a resumable orphan.
|
|
5007
|
+
priorSupervisor =
|
|
5008
|
+
"lock" in lockResult && lockResult.lock
|
|
5009
|
+
? { pid: lockResult.lock.pid, alive: isProcessAlive(lockResult.lock.pid) }
|
|
5010
|
+
: null;
|
|
4210
5011
|
// Populate orchBatchState from persisted state
|
|
4211
5012
|
orchBatchState.batchId = batchState.batchId;
|
|
4212
5013
|
orchBatchState.phase = batchState.phase as typeof orchBatchState.phase;
|
|
@@ -4709,6 +5510,7 @@ export default function (pi: ExtensionAPI) {
|
|
|
4709
5510
|
"The 'to' parameter must be a valid agent session name from the current batch.",
|
|
4710
5511
|
"Use orch_status() to see active session names.",
|
|
4711
5512
|
"Default type is 'steer' (course correction). Other types: 'query', 'abort', 'info'.",
|
|
5513
|
+
"HOLD CONTRACT (#630): when a worker is holding for a ruling you escalated for, send type='info' to ACKNOWLEDGE (\"received, ruling pending\") — it keeps the worker on hold and resets its relaunch budget. Send type='steer' only for the actual ruling/instruction — it releases the hold.",
|
|
4712
5514
|
"Messages are limited to 4KB. For larger context, write to a file and reference by path.",
|
|
4713
5515
|
],
|
|
4714
5516
|
parameters: Type.Object({
|
|
@@ -4810,6 +5612,29 @@ export default function (pi: ExtensionAPI) {
|
|
|
4810
5612
|
return `❌ Batch ${state.batchId} is in terminal phase (${state.phase}). Start or resume a batch before sending messages.`;
|
|
4811
5613
|
}
|
|
4812
5614
|
|
|
5615
|
+
// #630: a registry agent still marked running whose PROCESS is gone gets a
|
|
5616
|
+
// distinct, actionable error (pid, last-seen, what to do) instead of the
|
|
5617
|
+
// generic "unknown session" that collectKnownAgentIds would otherwise
|
|
5618
|
+
// produce after filtering dead pids. This is the #630 incident: the worker
|
|
5619
|
+
// died after an unanswered escalation; registry stayed `running` frozen.
|
|
5620
|
+
try {
|
|
5621
|
+
const registry = readRegistrySnapshot(stateRoot, state.batchId);
|
|
5622
|
+
const manifest = registry?.agents[to];
|
|
5623
|
+
if (manifest && !isTerminalStatus(manifest.status) && !registryIsProcessAlive(manifest.pid)) {
|
|
5624
|
+
const lastSeen = new Date(registry!.updatedAt).toISOString();
|
|
5625
|
+
return (
|
|
5626
|
+
`❌ Agent "${to}" is DEAD: its process (PID ${manifest.pid}) no longer exists, but the registry still ` +
|
|
5627
|
+
`marks it "${manifest.status}" (last registry update ${lastSeen}${manifest.taskId ? `, task ${manifest.taskId}` : ""}). ` +
|
|
5628
|
+
`No live consumer can receive this message.\n` +
|
|
5629
|
+
` Do NOT hand-edit registry.json. orch_resume(force=true) reconciles the dead worker ` +
|
|
5630
|
+
`(re-execute in its existing worktree; committed work survives) — if the batch is inherited, ` +
|
|
5631
|
+
`the ownership gate must pass first (see the takeover summary's Engine line).`
|
|
5632
|
+
);
|
|
5633
|
+
}
|
|
5634
|
+
} catch {
|
|
5635
|
+
/* registry unreadable — fall through to the normal validation */
|
|
5636
|
+
}
|
|
5637
|
+
|
|
4813
5638
|
// Build valid runtime agent IDs (registry-first, legacy fallback).
|
|
4814
5639
|
const validSessions = new Set<string>(collectKnownAgentIds(stateRoot, state));
|
|
4815
5640
|
|
|
@@ -5255,6 +6080,188 @@ export default function (pi: ExtensionAPI) {
|
|
|
5255
6080
|
return lines.join("\n");
|
|
5256
6081
|
}
|
|
5257
6082
|
|
|
6083
|
+
/**
|
|
6084
|
+
* #631: record operator-verified engine shutdown for the current/persisted
|
|
6085
|
+
* batch. Shared by the supervisor tool and the /orch-confirm-engine-shutdown
|
|
6086
|
+
* command. Always audited.
|
|
6087
|
+
*/
|
|
6088
|
+
function doOrchConfirmEngineShutdown(
|
|
6089
|
+
note: string,
|
|
6090
|
+
stateRoot: string,
|
|
6091
|
+
explicitBatchId?: string,
|
|
6092
|
+
): string {
|
|
6093
|
+
// Target: an EXPLICIT batchId (exactly the batch a refusal named — works for
|
|
6094
|
+
// meta-only legacy batches that are not reconstructable), else the same
|
|
6095
|
+
// target the recovery gates use: persisted → reconstructed → cached.
|
|
6096
|
+
const explicit = (explicitBatchId ?? "").trim();
|
|
6097
|
+
if (explicit && !/^[A-Za-z0-9._-]+$/.test(explicit)) {
|
|
6098
|
+
return `❌ Invalid batchId "${explicit}".`;
|
|
6099
|
+
}
|
|
6100
|
+
const target = explicit ? null : resolveRecoveryTarget(stateRoot, true);
|
|
6101
|
+
const batchId =
|
|
6102
|
+
explicit || target?.batchId || orchBatchState.batchId || supervisorState.batchId || "";
|
|
6103
|
+
if (!batchId) {
|
|
6104
|
+
return (
|
|
6105
|
+
"❌ No batch to confirm shutdown for (no batch on disk, nothing reconstructable, nothing in memory). " +
|
|
6106
|
+
"Pass the batchId named by the refusal explicitly."
|
|
6107
|
+
);
|
|
6108
|
+
}
|
|
6109
|
+
if (!note || note.trim().length < 8) {
|
|
6110
|
+
return "❌ A verification note is required (what you checked and how) — it is written to engine.json and the audit trail.";
|
|
6111
|
+
}
|
|
6112
|
+
const result = recordOperatorConfirmedShutdown(stateRoot, batchId, {
|
|
6113
|
+
supervisorPid: process.pid,
|
|
6114
|
+
note,
|
|
6115
|
+
});
|
|
6116
|
+
logRecoveryAction(stateRoot, batchId, {
|
|
6117
|
+
action: "confirm_engine_shutdown",
|
|
6118
|
+
classification: "destructive",
|
|
6119
|
+
context: `operator-verified engine shutdown for inherited batch: ${note.slice(0, 300)}`,
|
|
6120
|
+
command: "orch_confirm_engine_shutdown",
|
|
6121
|
+
result: result.ok ? "success" : "failure",
|
|
6122
|
+
detail: result.reason,
|
|
6123
|
+
});
|
|
6124
|
+
return result.ok
|
|
6125
|
+
? `✅ ${result.reason}. Recovery tools (orch_resume(force=true), retry/skip/force_merge) will now proceed via the verified path. Audit entry written.`
|
|
6126
|
+
: `❌ Not recorded: ${result.reason}`;
|
|
6127
|
+
}
|
|
6128
|
+
|
|
6129
|
+
pi.registerCommand("orch-confirm-engine-shutdown", {
|
|
6130
|
+
description:
|
|
6131
|
+
"Record operator-verified engine shutdown for a batch with no engine identity (#631): /orch-confirm-engine-shutdown [--batch <batchId>] <what you verified>",
|
|
6132
|
+
handler: async (args, ctx) => {
|
|
6133
|
+
// Syntax: /orch-confirm-engine-shutdown [--batch <batchId>] <note>
|
|
6134
|
+
let raw = (args ?? "").trim();
|
|
6135
|
+
let explicitBatchId: string | undefined;
|
|
6136
|
+
const m = /(?:^|\s)--batch\s+(\S+)/.exec(raw);
|
|
6137
|
+
if (m) {
|
|
6138
|
+
explicitBatchId = m[1];
|
|
6139
|
+
raw = raw.replace(m[0], " ").trim();
|
|
6140
|
+
}
|
|
6141
|
+
const stateRoot = execCtx?.workspaceRoot ?? execCtx?.repoRoot ?? ctx.cwd;
|
|
6142
|
+
const result = doOrchConfirmEngineShutdown(raw, stateRoot, explicitBatchId);
|
|
6143
|
+
ctx.ui.notify(result, result.startsWith("✅") ? "info" : "warning");
|
|
6144
|
+
},
|
|
6145
|
+
});
|
|
6146
|
+
|
|
6147
|
+
// ── #631: explicit, audited legacy shutdown confirmation ──
|
|
6148
|
+
// For a batch with NO engine identity (pre-#631 engine, or one that never
|
|
6149
|
+
// published), recovery tools fail closed: unknown ownership is not confirmed
|
|
6150
|
+
// shutdown. The operator verifies out-of-band that no engine process exists,
|
|
6151
|
+
// then records it here. The record is an `exited` identity (so every gate
|
|
6152
|
+
// proceeds through the normal verified path) AND an audit-trail entry.
|
|
6153
|
+
pi.registerTool({
|
|
6154
|
+
name: "orch_confirm_engine_shutdown",
|
|
6155
|
+
label: "Confirm Engine Shutdown",
|
|
6156
|
+
description:
|
|
6157
|
+
"Record that the operator verified NO engine process is running for the inherited batch (used only when " +
|
|
6158
|
+
"no engine identity is recorded, so the runtime cannot verify it). Unblocks orch_resume/retry/skip/force_merge. " +
|
|
6159
|
+
"Refuses when a real engine identity exists — that path is pid-verified and cannot be overridden.",
|
|
6160
|
+
promptSnippet:
|
|
6161
|
+
"orch_confirm_engine_shutdown(note) — record operator-verified engine shutdown for a batch with no engine identity",
|
|
6162
|
+
promptGuidelines: [
|
|
6163
|
+
"Only after verifying out-of-band that no engine process exists for this repo (e.g. Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -match 'engine-worker' } on Windows; pgrep -af engine-worker on POSIX).",
|
|
6164
|
+
"Never use this to override a refusal that names a live engine PID — wait for it or terminate it.",
|
|
6165
|
+
"The note should say what you checked; it is written to engine.json and the audit trail.",
|
|
6166
|
+
],
|
|
6167
|
+
parameters: Type.Object({
|
|
6168
|
+
note: Type.String({
|
|
6169
|
+
description:
|
|
6170
|
+
"What was verified and how (e.g. 'no engine-worker processes in Get-CimInstance output at 21:32')",
|
|
6171
|
+
}),
|
|
6172
|
+
batchId: Type.Optional(
|
|
6173
|
+
Type.String({
|
|
6174
|
+
description:
|
|
6175
|
+
"Exact batchId to confirm (the one named by the refusal). Omit to use the persisted/reconstructed batch.",
|
|
6176
|
+
}),
|
|
6177
|
+
),
|
|
6178
|
+
}),
|
|
6179
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
6180
|
+
const text = doOrchConfirmEngineShutdown(params.note, resolveToolStateRoot(ctx), params.batchId);
|
|
6181
|
+
return { content: [{ type: "text" as const, text }], details: undefined };
|
|
6182
|
+
},
|
|
6183
|
+
});
|
|
6184
|
+
|
|
6185
|
+
// ── #625: Audit-trail tool ───────────────────────────────────
|
|
6186
|
+
// The supervisor's audit trail (.pi/supervisor/actions.jsonl) was previously
|
|
6187
|
+
// hand-appended via bash per the system-prompt instructions — the LLM invented
|
|
6188
|
+
// the `ts` values (5+ hours off, non-monotonic, crossing calendar days).
|
|
6189
|
+
// logRecoveryAction() existed but nothing exposed it. This tool code-stamps
|
|
6190
|
+
// ts and batchId so the audit trail is trustworthy.
|
|
6191
|
+
pi.registerTool({
|
|
6192
|
+
name: "log_recovery_action",
|
|
6193
|
+
label: "Log Recovery Action",
|
|
6194
|
+
description:
|
|
6195
|
+
"Append an entry to the supervisor audit trail (.pi/supervisor/actions.jsonl). " +
|
|
6196
|
+
"Timestamps and batchId are stamped by code — never hand-write the file.",
|
|
6197
|
+
promptSnippet:
|
|
6198
|
+
"log_recovery_action(action, classification, context, command, result, detail, …) — append audit-trail entry",
|
|
6199
|
+
promptGuidelines: [
|
|
6200
|
+
"Use log_recovery_action for EVERY audit-trail entry — never append to actions.jsonl with bash.",
|
|
6201
|
+
"Timestamps and batchId are stamped automatically; do not supply them.",
|
|
6202
|
+
'For destructive actions: log result="pending" BEFORE executing, then log the real result after.',
|
|
6203
|
+
"classification: diagnostic | tier0_known | destructive.",
|
|
6204
|
+
],
|
|
6205
|
+
parameters: Type.Object({
|
|
6206
|
+
action: Type.String({ description: 'Action identifier, e.g. "merge_retry", "kill_session"' }),
|
|
6207
|
+
classification: Type.Union(
|
|
6208
|
+
[Type.Literal("diagnostic"), Type.Literal("tier0_known"), Type.Literal("destructive")],
|
|
6209
|
+
{ description: "Recovery action classification" },
|
|
6210
|
+
),
|
|
6211
|
+
context: Type.String({ description: "Why this action was taken" }),
|
|
6212
|
+
command: Type.String({ description: "Command or operation executed" }),
|
|
6213
|
+
result: Type.Union(
|
|
6214
|
+
[
|
|
6215
|
+
Type.Literal("pending"),
|
|
6216
|
+
Type.Literal("success"),
|
|
6217
|
+
Type.Literal("failure"),
|
|
6218
|
+
Type.Literal("skipped"),
|
|
6219
|
+
],
|
|
6220
|
+
{ description: "Outcome (pending = before a destructive action)" },
|
|
6221
|
+
),
|
|
6222
|
+
detail: Type.String({ description: "Result detail — error on failure, summary on success" }),
|
|
6223
|
+
waveIndex: Type.Optional(Type.Number({ description: "Wave index if wave-scoped" })),
|
|
6224
|
+
laneNumber: Type.Optional(Type.Number({ description: "Lane number if lane-scoped" })),
|
|
6225
|
+
taskId: Type.Optional(Type.String({ description: "Task ID if task-scoped" })),
|
|
6226
|
+
}),
|
|
6227
|
+
async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
|
|
6228
|
+
try {
|
|
6229
|
+
const stateRoot = resolveToolStateRoot(ctx);
|
|
6230
|
+
const batchId = orchBatchState.batchId || supervisorState.batchId || "unknown";
|
|
6231
|
+
logRecoveryAction(stateRoot, batchId, {
|
|
6232
|
+
action: params.action,
|
|
6233
|
+
classification: params.classification,
|
|
6234
|
+
context: params.context,
|
|
6235
|
+
command: params.command,
|
|
6236
|
+
result: params.result,
|
|
6237
|
+
detail: params.detail,
|
|
6238
|
+
...(params.waveIndex !== undefined ? { waveIndex: params.waveIndex } : {}),
|
|
6239
|
+
...(params.laneNumber !== undefined ? { laneNumber: params.laneNumber } : {}),
|
|
6240
|
+
...(params.taskId !== undefined ? { taskId: params.taskId } : {}),
|
|
6241
|
+
});
|
|
6242
|
+
return {
|
|
6243
|
+
content: [
|
|
6244
|
+
{
|
|
6245
|
+
type: "text" as const,
|
|
6246
|
+
text: `Audit entry logged: ${params.action} (${params.classification}, ${params.result})`,
|
|
6247
|
+
},
|
|
6248
|
+
],
|
|
6249
|
+
details: undefined,
|
|
6250
|
+
};
|
|
6251
|
+
} catch (err) {
|
|
6252
|
+
return {
|
|
6253
|
+
content: [
|
|
6254
|
+
{
|
|
6255
|
+
type: "text" as const,
|
|
6256
|
+
text: `Error logging audit entry: ${err instanceof Error ? err.message : String(err)}`,
|
|
6257
|
+
},
|
|
6258
|
+
],
|
|
6259
|
+
details: undefined,
|
|
6260
|
+
};
|
|
6261
|
+
}
|
|
6262
|
+
},
|
|
6263
|
+
});
|
|
6264
|
+
|
|
5258
6265
|
pi.registerTool({
|
|
5259
6266
|
name: "trigger_wrap_up",
|
|
5260
6267
|
label: "Trigger Wrap Up",
|
|
@@ -5611,6 +6618,33 @@ export default function (pi: ExtensionAPI) {
|
|
|
5611
6618
|
|
|
5612
6619
|
// ── Session Lifecycle ────────────────────────────────────────────
|
|
5613
6620
|
|
|
6621
|
+
// #621 (defense in depth): repair tool_use/tool_result ordering on every
|
|
6622
|
+
// outgoing request. The `context` event fires before each provider call on
|
|
6623
|
+
// the pi-internal AgentMessage[] (before convertToLlm). Any supervisor
|
|
6624
|
+
// `custom` message that was spliced between an assistant tool_use and its
|
|
6625
|
+
// tool_result (from ANY send site — batch summary, integration progress/
|
|
6626
|
+
// result, heartbeat, routing) is moved back after the tool-result group, so
|
|
6627
|
+
// the request is always valid and a mistimed injection can never wedge the
|
|
6628
|
+
// session. Only transforms the per-request context; the persisted tree is
|
|
6629
|
+
// untouched (self-correcting across reloads).
|
|
6630
|
+
pi.on("context", (event: { messages: unknown[] }) => {
|
|
6631
|
+
const repaired = repairToolResultOrdering(event.messages as Array<Record<string, unknown>>);
|
|
6632
|
+
if (repaired !== event.messages) return { messages: repaired };
|
|
6633
|
+
});
|
|
6634
|
+
|
|
6635
|
+
// #621: Flush a deferred batch-end epilogue once the interactive agent has
|
|
6636
|
+
// fully settled (all tool_results appended). Re-check idleness here because a
|
|
6637
|
+
// prior settle handler may have started another run.
|
|
6638
|
+
pi.on("agent_settled", (_event: unknown, ctx: ExtensionContext) => {
|
|
6639
|
+
noticeGate.onSettled(ctx.isIdle(), batchGeneration);
|
|
6640
|
+
});
|
|
6641
|
+
|
|
6642
|
+
// #621: Drop any deferred epilogue and disable the gate on shutdown so a
|
|
6643
|
+
// stale closure cannot fire against a replaced session.
|
|
6644
|
+
pi.on("session_shutdown", () => {
|
|
6645
|
+
noticeGate.dispose();
|
|
6646
|
+
});
|
|
6647
|
+
|
|
5614
6648
|
pi.on("session_start", async (_event, ctx) => {
|
|
5615
6649
|
// Store widget context for dashboard updates (needed even if startup fails)
|
|
5616
6650
|
orchWidgetCtx = ctx;
|
|
@@ -5747,6 +6781,14 @@ export default function (pi: ExtensionAPI) {
|
|
|
5747
6781
|
"info",
|
|
5748
6782
|
);
|
|
5749
6783
|
|
|
6784
|
+
// #631: record what we know about the previous owner. The engine of a
|
|
6785
|
+
// batch we did not start is NOT attached to this process; the active-
|
|
6786
|
+
// phase guards use this + engine.json to decide whether an inherited
|
|
6787
|
+
// "executing" phase is a resumable orphan.
|
|
6788
|
+
priorSupervisor =
|
|
6789
|
+
"lock" in lockResult && lockResult.lock
|
|
6790
|
+
? { pid: lockResult.lock.pid, alive: isProcessAlive(lockResult.lock.pid) }
|
|
6791
|
+
: null;
|
|
5750
6792
|
// Populate orchBatchState from persisted state for the supervisor
|
|
5751
6793
|
// prompt rebuild. We copy the key fields used by the system prompt.
|
|
5752
6794
|
orchBatchState.batchId = batchState.batchId;
|
|
@@ -5798,6 +6840,11 @@ export default function (pi: ExtensionAPI) {
|
|
|
5798
6840
|
|
|
5799
6841
|
// Store the live lock info so the /orch handler can detect it
|
|
5800
6842
|
// (preventing a second /orch from starting a concurrent batch).
|
|
6843
|
+
// #631: the other session's engine is live and NOT ours — record it.
|
|
6844
|
+
priorSupervisor =
|
|
6845
|
+
"lock" in lockResult && lockResult.lock
|
|
6846
|
+
? { pid: lockResult.lock.pid, alive: isProcessAlive(lockResult.lock.pid) }
|
|
6847
|
+
: null;
|
|
5801
6848
|
orchBatchState.batchId = batchState.batchId;
|
|
5802
6849
|
orchBatchState.phase = batchState.phase as typeof orchBatchState.phase;
|
|
5803
6850
|
orchBatchState.baseBranch = batchState.baseBranch;
|