taskplane 0.30.4 → 0.30.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/taskplane/agent-bridge-extension.ts +66 -8
- package/extensions/taskplane/agent-host.ts +170 -9
- package/extensions/taskplane/config-loader.ts +9 -0
- package/extensions/taskplane/config-schema.ts +47 -1
- package/extensions/taskplane/context-repair.ts +158 -0
- package/extensions/taskplane/diagnostic-reports.ts +109 -6
- package/extensions/taskplane/diagnostics.ts +3 -0
- package/extensions/taskplane/engine-identity.ts +401 -0
- package/extensions/taskplane/engine-worker.ts +59 -3
- package/extensions/taskplane/engine.ts +137 -17
- package/extensions/taskplane/execution.ts +89 -8
- package/extensions/taskplane/extension.ts +1288 -241
- package/extensions/taskplane/git.ts +74 -0
- package/extensions/taskplane/lane-runner.ts +971 -95
- package/extensions/taskplane/process-registry.ts +7 -2
- package/extensions/taskplane/resume.ts +559 -23
- package/extensions/taskplane/review-analysis.ts +450 -0
- package/extensions/taskplane/segment-recovery.ts +192 -0
- package/extensions/taskplane/supervisor-dispatch.ts +103 -0
- package/extensions/taskplane/supervisor-primer.md +182 -2
- package/extensions/taskplane/supervisor.ts +247 -24
- package/extensions/taskplane/types.ts +142 -4
- package/extensions/taskplane/worktree.ts +143 -2
- package/package.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
*
|
|
11
11
|
* @module orch/diagnostic-reports
|
|
12
12
|
*/
|
|
13
|
-
import { existsSync, mkdirSync, writeFileSync } from "fs";
|
|
13
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
14
14
|
import { join } from "path";
|
|
15
15
|
|
|
16
16
|
import { execLog } from "./execution.ts";
|
|
@@ -92,6 +92,12 @@ export interface DiagnosticReportInput {
|
|
|
92
92
|
totalTasks: number;
|
|
93
93
|
/** State root path where `.pi/` lives */
|
|
94
94
|
stateRoot: string;
|
|
95
|
+
/**
|
|
96
|
+
* #629: per-task cost (USD) from outcome telemetry for the CURRENT pass.
|
|
97
|
+
* `diagnostics.taskExits` is not populated by the v2 runtime, so without
|
|
98
|
+
* this every report read $0.00. Optional for backward compatibility.
|
|
99
|
+
*/
|
|
100
|
+
taskCostUsd?: Record<string, number>;
|
|
95
101
|
}
|
|
96
102
|
|
|
97
103
|
// ── Diagnostics Directory ────────────────────────────────────────────
|
|
@@ -140,8 +146,8 @@ export function buildDiagnosticEvents(input: DiagnosticReportInput): DiagnosticE
|
|
|
140
146
|
classification = task.exitDiagnostic.classification;
|
|
141
147
|
}
|
|
142
148
|
|
|
143
|
-
// Cost: from taskExits, else 0
|
|
144
|
-
const cost = exitSummary?.cost ?? 0;
|
|
149
|
+
// Cost: from taskExits, else this pass's outcome telemetry, else 0
|
|
150
|
+
const cost = exitSummary?.cost ?? input.taskCostUsd?.[task.taskId] ?? 0;
|
|
145
151
|
|
|
146
152
|
// Duration: from taskExits, else compute from timestamps, else 0
|
|
147
153
|
let durationSec = 0;
|
|
@@ -177,6 +183,79 @@ export function buildDiagnosticEvents(input: DiagnosticReportInput): DiagnosticE
|
|
|
177
183
|
|
|
178
184
|
// ── JSONL Generation ─────────────────────────────────────────────────
|
|
179
185
|
|
|
186
|
+
// ── Cross-pass evidence preservation (#629 side-effect 2) ──
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Does this event carry any execution evidence? A task that was not executed
|
|
190
|
+
* in the current pass (e.g. a no-op resume) produces an evidence-empty event
|
|
191
|
+
* that would otherwise CLOBBER the prior pass's record when the report is
|
|
192
|
+
* rewritten (observed: a $55 / 1h42m run reported as $0 / 0s after two
|
|
193
|
+
* no-op resumes).
|
|
194
|
+
*/
|
|
195
|
+
export function hasExecutionEvidence(evt: DiagnosticEvent): boolean {
|
|
196
|
+
return (
|
|
197
|
+
(evt.classification !== "unknown" && evt.classification !== "") ||
|
|
198
|
+
evt.cost > 0 ||
|
|
199
|
+
evt.durationSec > 0 ||
|
|
200
|
+
evt.retries > 0 ||
|
|
201
|
+
evt.startedAt !== null ||
|
|
202
|
+
evt.endedAt !== null
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Merge the previous report's events into the current pass's events.
|
|
208
|
+
*
|
|
209
|
+
* Rule: CURRENT STATE always comes from the new pass (`status`, `phase`,
|
|
210
|
+
* `mode`, `batchId`) — a legitimate retry or skip must be reflected. Execution
|
|
211
|
+
* EVIDENCE is merged FIELD-WISE: each evidence field takes the new pass's
|
|
212
|
+
* value when it is present/non-zero, else the previous value. This matters
|
|
213
|
+
* because resume's reconciliation synthesizes outcomes for tasks it did NOT
|
|
214
|
+
* execute that still carry the persisted classification and a fresh
|
|
215
|
+
* `endTime` — an all-or-nothing rule would treat those as "evidence" and
|
|
216
|
+
* erase the prior cost. Cost is never summed across passes (each pass's
|
|
217
|
+
* telemetry is that attempt's cost; no double counting). Tasks present only
|
|
218
|
+
* in the previous report are dropped (the new wave plan is authoritative for
|
|
219
|
+
* membership). Pure; deterministic order follows `next`.
|
|
220
|
+
*/
|
|
221
|
+
export function mergeDiagnosticEvents(
|
|
222
|
+
previous: DiagnosticEvent[],
|
|
223
|
+
next: DiagnosticEvent[],
|
|
224
|
+
): DiagnosticEvent[] {
|
|
225
|
+
const prevByTask = new Map<string, DiagnosticEvent>();
|
|
226
|
+
for (const p of previous) prevByTask.set(p.taskId, p);
|
|
227
|
+
return next.map((n) => {
|
|
228
|
+
const p = prevByTask.get(n.taskId);
|
|
229
|
+
if (!p) return n;
|
|
230
|
+
const knownClass = n.classification && n.classification !== "unknown";
|
|
231
|
+
return {
|
|
232
|
+
...n,
|
|
233
|
+
classification: knownClass ? n.classification : p.classification,
|
|
234
|
+
cost: n.cost > 0 ? n.cost : p.cost,
|
|
235
|
+
durationSec: n.durationSec > 0 ? n.durationSec : p.durationSec,
|
|
236
|
+
retries: n.retries > 0 ? n.retries : p.retries,
|
|
237
|
+
exitReason: n.exitReason || p.exitReason,
|
|
238
|
+
startedAt: n.startedAt ?? p.startedAt,
|
|
239
|
+
endedAt: n.endedAt ?? p.endedAt,
|
|
240
|
+
};
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Parse a previously written events JSONL file (tolerant: bad lines skipped). */
|
|
245
|
+
export function parseEventsJsonl(content: string): DiagnosticEvent[] {
|
|
246
|
+
const out: DiagnosticEvent[] = [];
|
|
247
|
+
for (const line of content.split(/\r?\n/)) {
|
|
248
|
+
if (!line.trim()) continue;
|
|
249
|
+
try {
|
|
250
|
+
const obj = JSON.parse(line) as Partial<DiagnosticEvent>;
|
|
251
|
+
if (typeof obj.taskId === "string") out.push(obj as DiagnosticEvent);
|
|
252
|
+
} catch {
|
|
253
|
+
/* skip malformed */
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
return out;
|
|
257
|
+
}
|
|
258
|
+
|
|
180
259
|
/**
|
|
181
260
|
* Serialize diagnostic events to JSONL format (one JSON object per line).
|
|
182
261
|
*/
|
|
@@ -222,7 +301,12 @@ export function buildMarkdownReport(
|
|
|
222
301
|
const { succeededTasks, failedTasks, skippedTasks, blockedTasks, totalTasks } = input;
|
|
223
302
|
|
|
224
303
|
const batchDurationSec = endedAt ? Math.round((endedAt - startedAt) / 1000) : 0;
|
|
225
|
-
|
|
304
|
+
// #629: header cost from batch diagnostics when populated, else the sum of
|
|
305
|
+
// per-task evidence (which survives no-op passes via mergeDiagnosticEvents).
|
|
306
|
+
const batchCost =
|
|
307
|
+
diagnostics.batchCost && diagnostics.batchCost > 0
|
|
308
|
+
? diagnostics.batchCost
|
|
309
|
+
: events.reduce((sum, e) => sum + (Number.isFinite(e.cost) ? e.cost : 0), 0);
|
|
226
310
|
|
|
227
311
|
const lines: string[] = [];
|
|
228
312
|
|
|
@@ -337,10 +421,21 @@ export function emitDiagnosticReports(input: DiagnosticReportInput): void {
|
|
|
337
421
|
const opId = resolveOperatorId(input.orchConfig);
|
|
338
422
|
const dir = ensureDiagnosticsDir(input.stateRoot);
|
|
339
423
|
|
|
340
|
-
const
|
|
424
|
+
const jsonlPath = join(dir, `${opId}-${input.batchId}-events.jsonl`);
|
|
425
|
+
|
|
426
|
+
// #629: preserve prior-pass execution evidence for tasks this pass did
|
|
427
|
+
// not execute (state fields still come from this pass).
|
|
428
|
+
let events = buildDiagnosticEvents(input);
|
|
429
|
+
if (existsSync(jsonlPath)) {
|
|
430
|
+
try {
|
|
431
|
+
const previous = parseEventsJsonl(readFileSync(jsonlPath, "utf-8"));
|
|
432
|
+
events = mergeDiagnosticEvents(previous, events);
|
|
433
|
+
} catch {
|
|
434
|
+
/* unreadable prior report — proceed with this pass's events */
|
|
435
|
+
}
|
|
436
|
+
}
|
|
341
437
|
|
|
342
438
|
// ── JSONL event log ──
|
|
343
|
-
const jsonlPath = join(dir, `${opId}-${input.batchId}-events.jsonl`);
|
|
344
439
|
const jsonlContent = eventsToJsonl(events);
|
|
345
440
|
writeFileSync(jsonlPath, jsonlContent, "utf-8");
|
|
346
441
|
|
|
@@ -471,5 +566,13 @@ export function assembleDiagnosticInput(
|
|
|
471
566
|
blockedTasks: batchState.blockedTasks,
|
|
472
567
|
totalTasks: batchState.totalTasks,
|
|
473
568
|
stateRoot,
|
|
569
|
+
taskCostUsd: (() => {
|
|
570
|
+
const costs: Record<string, number> = {};
|
|
571
|
+
for (const [taskId, outcome] of outcomeByTaskId) {
|
|
572
|
+
const c = outcome.telemetry?.costUsd;
|
|
573
|
+
if (typeof c === "number" && Number.isFinite(c) && c > 0) costs[taskId] = c;
|
|
574
|
+
}
|
|
575
|
+
return costs;
|
|
576
|
+
})(),
|
|
474
577
|
};
|
|
475
578
|
}
|
|
@@ -53,6 +53,7 @@ export interface SessionTokenCounts {
|
|
|
53
53
|
* | `stall_timeout` | No STATUS.md progress for stall_timeout minutes |
|
|
54
54
|
* | `user_killed` | User manually killed the session (e.g., forced process kill) |
|
|
55
55
|
* | `spawn_failure` | Worker process never spawned (e.g., Pi CLI not findable, worktree provisioning) |
|
|
56
|
+
* | `review_gate_refusal`| Governance refusal: finalize blocked by an outstanding REVISE/RETHINK review verdict (#626/#629). The worker exited cleanly — NOT a crash, never auto-retried (the review file must change first) |
|
|
56
57
|
* | `unknown` | Could not determine cause |
|
|
57
58
|
*
|
|
58
59
|
* Note: `spawn_failure` (TP-190, #561) is set BEFORE any agent process exists —
|
|
@@ -72,6 +73,7 @@ export type ExitClassification =
|
|
|
72
73
|
| "stall_timeout"
|
|
73
74
|
| "user_killed"
|
|
74
75
|
| "spawn_failure"
|
|
76
|
+
| "review_gate_refusal"
|
|
75
77
|
| "unknown";
|
|
76
78
|
|
|
77
79
|
/**
|
|
@@ -88,6 +90,7 @@ export const EXIT_CLASSIFICATIONS: readonly ExitClassification[] = [
|
|
|
88
90
|
"stall_timeout",
|
|
89
91
|
"user_killed",
|
|
90
92
|
"spawn_failure",
|
|
93
|
+
"review_gate_refusal",
|
|
91
94
|
"unknown",
|
|
92
95
|
] as const;
|
|
93
96
|
|
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* engine-identity.ts — Persisted identity of the batch ENGINE process (#631).
|
|
3
|
+
*
|
|
4
|
+
* The orchestration engine runs as a forked child process of the supervisor's
|
|
5
|
+
* Pi session. If that session dies or is replaced while the batch is
|
|
6
|
+
* `executing`, the replacement supervisor inherits persisted state that says
|
|
7
|
+
* "executing" — but it has no engine attached, and it cannot tell from the
|
|
8
|
+
* supervisor lock alone whether the ORIGINAL engine is still alive (a dead
|
|
9
|
+
* supervisor pid does not imply a dead engine; a forked child survives its
|
|
10
|
+
* parent on Windows, and a wedged supervisor may leave a healthy engine
|
|
11
|
+
* driving lanes).
|
|
12
|
+
*
|
|
13
|
+
* This module gives a replacement session a verifiable answer:
|
|
14
|
+
*
|
|
15
|
+
* - `alive` the recorded engine pid still exists → refuse recovery
|
|
16
|
+
* mutations (double-drive risk); tell the operator the pid.
|
|
17
|
+
* - `dead` the recorded pid no longer exists → confirmed orphan; the
|
|
18
|
+
* persisted-state eligibility rules (`checkResumeEligibility`)
|
|
19
|
+
* may proceed.
|
|
20
|
+
* - `exited` the engine recorded its own exit (clean or crash) → same as dead.
|
|
21
|
+
* - `none` no identity recorded (older batch, or the engine never forked)
|
|
22
|
+
* → callers fall back to supervisor-lock evidence.
|
|
23
|
+
*
|
|
24
|
+
* Verified shutdown, not inferred inactivity: timestamps in batch-state.json
|
|
25
|
+
* are checkpoint times, not heartbeats, and are refreshed by recovery tools
|
|
26
|
+
* themselves — they are NOT used here.
|
|
27
|
+
*
|
|
28
|
+
* File: `{stateRoot}/.pi/runtime/{batchId}/engine.json` (additive; runtime
|
|
29
|
+
* dir already hosts registry.json). Best-effort I/O: a write failure never
|
|
30
|
+
* blocks batch start.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { createHash } from "crypto";
|
|
34
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "fs";
|
|
35
|
+
import { dirname, join } from "path";
|
|
36
|
+
import { fileURLToPath } from "url";
|
|
37
|
+
|
|
38
|
+
import { isProcessAlive } from "./process-registry.ts";
|
|
39
|
+
import { runtimeRoot } from "./types.ts";
|
|
40
|
+
|
|
41
|
+
export interface EngineIdentity {
|
|
42
|
+
batchId: string;
|
|
43
|
+
/** Engine (forked child) pid */
|
|
44
|
+
pid: number;
|
|
45
|
+
/** Pid of the supervisor session that forked it */
|
|
46
|
+
supervisorPid: number;
|
|
47
|
+
/** Epoch ms when the fork happened */
|
|
48
|
+
startedAt: number;
|
|
49
|
+
/** Epoch ms when the engine exited (set by the parent on child exit) */
|
|
50
|
+
exitedAt?: number;
|
|
51
|
+
/** Exit code when known */
|
|
52
|
+
exitCode?: number | null;
|
|
53
|
+
/** Why the exit was recorded (e.g. "child-exit", "session-end-kill", "abort") */
|
|
54
|
+
exitReason?: string;
|
|
55
|
+
/**
|
|
56
|
+
* Build marker (Penster feedback on #632): which Taskplane code is actually
|
|
57
|
+
* driving this batch. `version` is package.json's; `build` is a short
|
|
58
|
+
* content fingerprint of the loaded extension source (sha256 prefix), so a
|
|
59
|
+
* local pre-release deploy is distinguishable from the published version
|
|
60
|
+
* even when the version string has not been bumped.
|
|
61
|
+
*/
|
|
62
|
+
taskplaneVersion?: string;
|
|
63
|
+
taskplaneBuild?: string;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Compute the build marker for the currently loaded Taskplane: package.json
|
|
68
|
+
* version + a short sha256 of the extension entry sources. Cached. Never throws.
|
|
69
|
+
*/
|
|
70
|
+
let cachedBuildMarker: { taskplaneVersion: string; taskplaneBuild: string } | null = null;
|
|
71
|
+
export function taskplaneBuildMarker(): { taskplaneVersion: string; taskplaneBuild: string } {
|
|
72
|
+
if (cachedBuildMarker) return cachedBuildMarker;
|
|
73
|
+
let version = "unknown";
|
|
74
|
+
let build = "unknown";
|
|
75
|
+
try {
|
|
76
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
77
|
+
const pkg = JSON.parse(readFileSync(join(here, "..", "..", "package.json"), "utf-8")) as {
|
|
78
|
+
version?: string;
|
|
79
|
+
};
|
|
80
|
+
if (typeof pkg.version === "string") version = pkg.version;
|
|
81
|
+
const h = createHash("sha256");
|
|
82
|
+
for (const file of [
|
|
83
|
+
"extension.ts",
|
|
84
|
+
"lane-runner.ts",
|
|
85
|
+
"engine.ts",
|
|
86
|
+
"resume.ts",
|
|
87
|
+
"engine-identity.ts",
|
|
88
|
+
]) {
|
|
89
|
+
try {
|
|
90
|
+
h.update(readFileSync(join(here, file)));
|
|
91
|
+
} catch {
|
|
92
|
+
/* skip */
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
build = h.digest("hex").slice(0, 12);
|
|
96
|
+
} catch {
|
|
97
|
+
/* best effort */
|
|
98
|
+
}
|
|
99
|
+
cachedBuildMarker = { taskplaneVersion: version, taskplaneBuild: build };
|
|
100
|
+
return cachedBuildMarker;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* #631: ownership evidence ASSOCIATED WITH AN ORCH BRANCH, independent of full
|
|
105
|
+
* state reconstruction. Integration acts on a branch; the batch behind it may
|
|
106
|
+
* have no batch-state.json (aborted) and may not be *reconstructable* (worker
|
|
107
|
+
* manifests gone) while its engine identity still records a live pid.
|
|
108
|
+
* Reconstructability is a resumability requirement, not a prerequisite for
|
|
109
|
+
* recognising ownership. Scans `.pi/runtime/<batchId>/batch-meta.json` for
|
|
110
|
+
* `orchBranch === branch` and returns each such batch with its liveness.
|
|
111
|
+
* Unreadable/absent meta is skipped (no association can be established).
|
|
112
|
+
*/
|
|
113
|
+
export function findBatchesForOrchBranch(
|
|
114
|
+
stateRoot: string,
|
|
115
|
+
orchBranch: string,
|
|
116
|
+
probe: (pid: number) => boolean = isProcessAlive,
|
|
117
|
+
): Array<{ batchId: string; liveness: EngineLiveness }> {
|
|
118
|
+
const out: Array<{ batchId: string; liveness: EngineLiveness }> = [];
|
|
119
|
+
const runtimeDir = join(stateRoot, ".pi", "runtime");
|
|
120
|
+
if (!existsSync(runtimeDir)) return out;
|
|
121
|
+
let entries: string[] = [];
|
|
122
|
+
try {
|
|
123
|
+
entries = readdirSync(runtimeDir);
|
|
124
|
+
} catch {
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
for (const batchId of entries) {
|
|
128
|
+
try {
|
|
129
|
+
const metaPath = join(runtimeDir, batchId, "batch-meta.json");
|
|
130
|
+
if (!existsSync(metaPath)) continue;
|
|
131
|
+
const meta = JSON.parse(readFileSync(metaPath, "utf-8")) as {
|
|
132
|
+
orchBranch?: unknown;
|
|
133
|
+
batchId?: unknown;
|
|
134
|
+
};
|
|
135
|
+
if (meta.orchBranch !== orchBranch) continue;
|
|
136
|
+
const id = typeof meta.batchId === "string" && meta.batchId ? meta.batchId : batchId;
|
|
137
|
+
out.push({ batchId: id, liveness: assessEngineLiveness(stateRoot, id, probe) });
|
|
138
|
+
} catch {
|
|
139
|
+
/* skip unreadable runtime dir */
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export type EngineLivenessStatus = "alive" | "dead" | "exited" | "none";
|
|
146
|
+
|
|
147
|
+
export interface EngineLiveness {
|
|
148
|
+
status: EngineLivenessStatus;
|
|
149
|
+
identity: EngineIdentity | null;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
export function engineIdentityPath(stateRoot: string, batchId: string): string {
|
|
153
|
+
return join(runtimeRoot(stateRoot, batchId), "engine.json");
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Publish a freshly started engine's identity. Returns false when it could not
|
|
158
|
+
* be written — callers treat that as "cannot own this batch verifiably" and
|
|
159
|
+
* refuse to start the engine (the identity is what lets a successor prove
|
|
160
|
+
* shutdown; an engine without one is an unownable orphan-in-waiting).
|
|
161
|
+
* The write goes through a temp file + rename so a reader never sees a
|
|
162
|
+
* partial record.
|
|
163
|
+
*/
|
|
164
|
+
export function writeEngineIdentity(
|
|
165
|
+
stateRoot: string,
|
|
166
|
+
identity: Omit<EngineIdentity, "exitedAt" | "exitCode" | "exitReason">,
|
|
167
|
+
): boolean {
|
|
168
|
+
try {
|
|
169
|
+
const path = engineIdentityPath(stateRoot, identity.batchId);
|
|
170
|
+
mkdirSync(join(path, ".."), { recursive: true });
|
|
171
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
172
|
+
writeFileSync(tmp, JSON.stringify({ ...identity, ...taskplaneBuildMarker() }, null, 2), "utf-8");
|
|
173
|
+
renameSync(tmp, path);
|
|
174
|
+
// Read-back: the record on disk must be the one we just published.
|
|
175
|
+
const check = readEngineIdentity(stateRoot, identity.batchId);
|
|
176
|
+
return check !== null && check.pid === identity.pid && check.exitedAt === undefined;
|
|
177
|
+
} catch {
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Mark the recorded engine as exited (keeps the file for forensics). Best effort.
|
|
184
|
+
*
|
|
185
|
+
* ATTEMPT-SCOPED: only marks the file when its recorded `pid` matches the
|
|
186
|
+
* exiting engine's pid. A delayed exit callback from an OLD parent must never
|
|
187
|
+
* mark a NEW live engine (which has since overwritten the identity) as exited.
|
|
188
|
+
* Returns true when the mark was applied.
|
|
189
|
+
*/
|
|
190
|
+
export function markEngineExited(
|
|
191
|
+
stateRoot: string,
|
|
192
|
+
batchId: string,
|
|
193
|
+
info: { pid: number; exitCode?: number | null; exitReason: string; exitedAt?: number },
|
|
194
|
+
): boolean {
|
|
195
|
+
try {
|
|
196
|
+
const path = engineIdentityPath(stateRoot, batchId);
|
|
197
|
+
if (!existsSync(path)) return false;
|
|
198
|
+
const current = JSON.parse(readFileSync(path, "utf-8")) as EngineIdentity;
|
|
199
|
+
if (current.pid !== info.pid) return false; // a newer engine owns this file
|
|
200
|
+
current.exitedAt = info.exitedAt ?? Date.now();
|
|
201
|
+
current.exitCode = info.exitCode ?? null;
|
|
202
|
+
current.exitReason = info.exitReason;
|
|
203
|
+
writeFileSync(path, JSON.stringify(current, null, 2), "utf-8");
|
|
204
|
+
return true;
|
|
205
|
+
} catch {
|
|
206
|
+
return false;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Marker pid for an identity synthesized by operator confirmation (no real engine pid). */
|
|
211
|
+
export const OPERATOR_CONFIRMED_PID = 0;
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* The explicit, auditable LEGACY path (#631): for a batch with no engine
|
|
215
|
+
* identity (pre-#631 engine, or an engine that never got far enough to publish
|
|
216
|
+
* one), the runtime cannot verify shutdown and MUST fail closed. The operator
|
|
217
|
+
* verifies out-of-band that no engine process exists for this repo and records
|
|
218
|
+
* that confirmation here; it becomes an `exited` identity so every recovery
|
|
219
|
+
* gate proceeds through the normal verified path. Refuses (returns false) if a
|
|
220
|
+
* REAL identity exists — confirmation cannot override an alive engine.
|
|
221
|
+
*/
|
|
222
|
+
export function recordOperatorConfirmedShutdown(
|
|
223
|
+
stateRoot: string,
|
|
224
|
+
batchId: string,
|
|
225
|
+
confirmedBy: { supervisorPid: number; note?: string },
|
|
226
|
+
): { ok: boolean; reason: string } {
|
|
227
|
+
const existing = readEngineIdentity(stateRoot, batchId);
|
|
228
|
+
if (existing && existing.pid !== OPERATOR_CONFIRMED_PID) {
|
|
229
|
+
return {
|
|
230
|
+
ok: false,
|
|
231
|
+
reason: `an engine identity IS recorded (PID ${existing.pid}${existing.exitedAt ? ", exited" : ""}); confirmation is only for batches with no identity — use the pid-verified path`,
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
try {
|
|
235
|
+
const path = engineIdentityPath(stateRoot, batchId);
|
|
236
|
+
mkdirSync(join(path, ".."), { recursive: true });
|
|
237
|
+
const now = Date.now();
|
|
238
|
+
const identity: EngineIdentity = {
|
|
239
|
+
batchId,
|
|
240
|
+
pid: OPERATOR_CONFIRMED_PID,
|
|
241
|
+
supervisorPid: confirmedBy.supervisorPid,
|
|
242
|
+
startedAt: now,
|
|
243
|
+
exitedAt: now,
|
|
244
|
+
exitCode: null,
|
|
245
|
+
exitReason: `operator-confirmed-shutdown${confirmedBy.note ? `: ${confirmedBy.note.slice(0, 200)}` : ""}`,
|
|
246
|
+
};
|
|
247
|
+
writeFileSync(path, JSON.stringify(identity, null, 2), "utf-8");
|
|
248
|
+
return { ok: true, reason: `recorded operator-confirmed shutdown for ${batchId}` };
|
|
249
|
+
} catch (err) {
|
|
250
|
+
return { ok: false, reason: err instanceof Error ? err.message : String(err) };
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
export function readEngineIdentity(stateRoot: string, batchId: string): EngineIdentity | null {
|
|
255
|
+
try {
|
|
256
|
+
const path = engineIdentityPath(stateRoot, batchId);
|
|
257
|
+
if (!existsSync(path)) return null;
|
|
258
|
+
const parsed = JSON.parse(readFileSync(path, "utf-8")) as Partial<EngineIdentity>;
|
|
259
|
+
if (typeof parsed.pid !== "number" || typeof parsed.batchId !== "string") return null;
|
|
260
|
+
return parsed as EngineIdentity;
|
|
261
|
+
} catch {
|
|
262
|
+
return null;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Is the recorded engine for this batch still running?
|
|
268
|
+
*
|
|
269
|
+
* `probe` is injectable for tests; defaults to a real pid probe.
|
|
270
|
+
*/
|
|
271
|
+
export function assessEngineLiveness(
|
|
272
|
+
stateRoot: string,
|
|
273
|
+
batchId: string,
|
|
274
|
+
probe: (pid: number) => boolean = isProcessAlive,
|
|
275
|
+
): EngineLiveness {
|
|
276
|
+
const identity = readEngineIdentity(stateRoot, batchId);
|
|
277
|
+
if (!identity) return { status: "none", identity: null };
|
|
278
|
+
if (identity.exitedAt !== undefined) return { status: "exited", identity };
|
|
279
|
+
return { status: probe(identity.pid) ? "alive" : "dead", identity };
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Decide whether a recovery mutation (resume / retry / skip / force-merge /
|
|
284
|
+
* administrative pause) may proceed against a batch whose persisted phase is
|
|
285
|
+
* ACTIVE but which has no engine attached to THIS process.
|
|
286
|
+
*
|
|
287
|
+
* Pure decision function (no I/O) so the policy is unit-testable:
|
|
288
|
+
*
|
|
289
|
+
* - engine `alive` → refuse (double-drive)
|
|
290
|
+
* - engine `dead` | `exited` → proceed (verified shutdown)
|
|
291
|
+
* - engine `none` → REFUSE, always. Unknown ownership is
|
|
292
|
+
* not confirmed shutdown (a forked
|
|
293
|
+
* engine outlives a dead supervisor).
|
|
294
|
+
* The message names the explicit,
|
|
295
|
+
* auditable legacy path:
|
|
296
|
+
* orch_confirm_engine_shutdown after
|
|
297
|
+
* out-of-band verification.
|
|
298
|
+
*
|
|
299
|
+
* `force` deliberately does NOT bypass an alive engine — that is the one case
|
|
300
|
+
* where proceeding can corrupt state.
|
|
301
|
+
*/
|
|
302
|
+
/**
|
|
303
|
+
* THE recovery-ownership rule (#631), as a pure function so every bypass
|
|
304
|
+
* scenario is unit-testable. `extension.ts` wraps it with I/O (engine
|
|
305
|
+
* liveness probe, logging).
|
|
306
|
+
*
|
|
307
|
+
* 1. an engine runs IN THIS PROCESS (forked child not yet terminated, or the
|
|
308
|
+
* main-thread fallback) → refuse — even when the cached phase already
|
|
309
|
+
* reads paused/failed/completed (teardown still in flight).
|
|
310
|
+
* 2–4. otherwise defer to `decideInheritedActivePhase` against the ACTUAL
|
|
311
|
+
* target: alive elsewhere → refuse; none → refuse (confirm path);
|
|
312
|
+
* dead/exited → proceed.
|
|
313
|
+
*/
|
|
314
|
+
export function decideRecoveryOwnership(input: {
|
|
315
|
+
operation: string;
|
|
316
|
+
local: { engineAttached: boolean; phase: string; batchId: string; pid: number | null };
|
|
317
|
+
target: { batchId: string; phase: string };
|
|
318
|
+
liveness: EngineLiveness;
|
|
319
|
+
priorSupervisor: { pid: number; alive: boolean } | null;
|
|
320
|
+
}): InheritedPhaseDecision {
|
|
321
|
+
const { operation, local, target, liveness, priorSupervisor } = input;
|
|
322
|
+
if (local.engineAttached) {
|
|
323
|
+
const pid = local.pid ?? "?";
|
|
324
|
+
const terminalCache =
|
|
325
|
+
local.phase === "paused" ||
|
|
326
|
+
local.phase === "failed" ||
|
|
327
|
+
local.phase === "stopped" ||
|
|
328
|
+
local.phase === "completed";
|
|
329
|
+
return {
|
|
330
|
+
proceed: false,
|
|
331
|
+
reason: terminalCache
|
|
332
|
+
? `⏳ This session's engine (PID ${pid}) for batch ${local.batchId} is still shutting down — ${operation} would race its teardown. Retry in a moment.`
|
|
333
|
+
: `❌ Cannot ${operation} while batch ${local.batchId} is ${local.phase} in this session (engine PID ${pid}). Pause or wait for the current operation to finish first.`,
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
return decideInheritedActivePhase({
|
|
337
|
+
phase: target.phase,
|
|
338
|
+
batchId: target.batchId,
|
|
339
|
+
liveness,
|
|
340
|
+
priorSupervisor,
|
|
341
|
+
operation,
|
|
342
|
+
});
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export interface InheritedPhaseDecision {
|
|
346
|
+
proceed: boolean;
|
|
347
|
+
/** Operator-facing explanation (refusal reason or proceed rationale) */
|
|
348
|
+
reason: string;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
export function decideInheritedActivePhase(input: {
|
|
352
|
+
phase: string;
|
|
353
|
+
batchId: string;
|
|
354
|
+
liveness: EngineLiveness;
|
|
355
|
+
priorSupervisor: { pid: number; alive: boolean } | null;
|
|
356
|
+
operation: string;
|
|
357
|
+
}): InheritedPhaseDecision {
|
|
358
|
+
const { phase, batchId, liveness, priorSupervisor, operation } = input;
|
|
359
|
+
const id = liveness.identity;
|
|
360
|
+
switch (liveness.status) {
|
|
361
|
+
case "alive":
|
|
362
|
+
return {
|
|
363
|
+
proceed: false,
|
|
364
|
+
reason:
|
|
365
|
+
`❌ Batch ${batchId} is "${phase}" and its engine process (PID ${id!.pid}, forked ` +
|
|
366
|
+
`${new Date(id!.startedAt).toISOString()} by supervisor PID ${id!.supervisorPid}) is still ALIVE ` +
|
|
367
|
+
`in another process. This session has no engine attached and cannot drive or signal it; ` +
|
|
368
|
+
`${operation} now would double-drive the batch.\n` +
|
|
369
|
+
` Wait for that engine to finish or wind down (it pauses itself when its supervisor ` +
|
|
370
|
+
`disconnects), or terminate it explicitly (Windows: taskkill /PID ${id!.pid} /T; ` +
|
|
371
|
+
`POSIX: kill ${id!.pid}) and re-run ${operation}. force does not bypass this check.`,
|
|
372
|
+
};
|
|
373
|
+
case "dead":
|
|
374
|
+
case "exited":
|
|
375
|
+
return {
|
|
376
|
+
proceed: true,
|
|
377
|
+
reason:
|
|
378
|
+
`engine PID ${id!.pid} is ${liveness.status === "exited" ? `exited (${id!.exitReason ?? "recorded"})` : "dead"} ` +
|
|
379
|
+
`— inherited "${phase}" phase treated as disconnected; persisted-state eligibility applies`,
|
|
380
|
+
};
|
|
381
|
+
case "none": {
|
|
382
|
+
const supervisorNote = priorSupervisor
|
|
383
|
+
? priorSupervisor.alive
|
|
384
|
+
? `The previous supervisor (PID ${priorSupervisor.pid}) is still ALIVE, so its engine may well be driving the batch.`
|
|
385
|
+
: `The previous supervisor (PID ${priorSupervisor.pid}) is dead — but a forked engine can outlive its supervisor, so that alone does not prove the engine is gone.`
|
|
386
|
+
: `No previous-supervisor record is available either.`;
|
|
387
|
+
return {
|
|
388
|
+
proceed: false,
|
|
389
|
+
reason:
|
|
390
|
+
`❌ Batch ${batchId} is "${phase}" but this session has no engine attached and NO engine identity is ` +
|
|
391
|
+
`recorded for it (pre-#631 engine, or it never published one). ${supervisorNote} ` +
|
|
392
|
+
`Refusing ${operation}: unknown ownership is not confirmed shutdown.
|
|
393
|
+
` +
|
|
394
|
+
` Verify out-of-band that no engine process exists for this repo — Windows: ` +
|
|
395
|
+
`Get-CimInstance Win32_Process | Where-Object { $_.CommandLine -match "engine-worker" } ; ` +
|
|
396
|
+
`POSIX: pgrep -af engine-worker — then record it with orch_confirm_engine_shutdown(note) ` +
|
|
397
|
+
`(audited) and re-run ${operation}. No hand-edit of batch-state.json is needed.`,
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|