pi-braid 0.40.6 → 0.41.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1649 @@
1
+ /**
2
+ * The fleet supervisor: one process, no model, holding a cap of node sessions.
3
+ *
4
+ * Its whole job is to keep one statement true across a crash: no completed
5
+ * effect is repeated and no node is silently lost. Everything else here is
6
+ * scheduling taste.
7
+ *
8
+ * How that statement stays true:
9
+ *
10
+ * - A turn's prompt is journalled before the child is spawned
11
+ * (`turns.prompt_text`), and the child's process group is journalled with
12
+ * it. A restart therefore knows what was sent and what to fence.
13
+ * - Effects are journalled from the driver's own tool events: the attempt
14
+ * before the tool runs, the result after it. An attempt without a result
15
+ * is the crash residue, and reconciliation turns it into `unknown` rather
16
+ * than into a repeat.
17
+ * - A node resumes by reopening its recorded session id. Its recovery turn
18
+ * carries the journal's account of the interrupted turn: which effects
19
+ * completed, which are unknown. The supervisor never re-sends a prompt it
20
+ * already recorded as sent - that single rule is the difference between
21
+ * resuming and restarting.
22
+ * - The supervisor's epoch is journalled. A restart fences the process
23
+ * groups of every turn its predecessor left open, so a stale drive cannot
24
+ * keep writing while the new one resumes the same session.
25
+ *
26
+ * The naive replay path (`naive: true`) exists to be wrong on purpose: it is
27
+ * the arm the crash drill measures duplication with.
28
+ */
29
+
30
+ import { execFileSync } from "node:child_process";
31
+ import { createHash, randomBytes } from "node:crypto";
32
+ import { chmodSync, existsSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
33
+ import { createServer, connect } from "node:net";
34
+ import { tmpdir } from "node:os";
35
+ import { join } from "node:path";
36
+ import { pathToFileURL } from "node:url";
37
+
38
+ import { BUSY_TIMEOUT_MS, openJournal, replayPolicy, resolveFleetRoot, SCHEMA_VERSION } from "./journal.mjs";
39
+ import { createRpcDriver, GUARD_EXTENSION, guardEnvironment } from "./driver-rpc.mjs";
40
+ import { createScriptedDriver, ASK_SENTINEL, DONE_SENTINEL } from "./driver-scripted.mjs";
41
+ import { groupIsAlive, pidIsAlive, residentMb } from "./procs.mjs";
42
+
43
+ export { ASK_SENTINEL, DONE_SENTINEL };
44
+
45
+ export const NODE_STATES = ["settled", "waiting_on_child", "waiting_on_human", "failed", "done"];
46
+
47
+ /**
48
+ * Where a maintenance fact goes when the store will not take the write: beside
49
+ * the journal, because a fact about a wedged store is the one fact the store
50
+ * cannot hold. `status` reads it, and the first pass that gets through copies it
51
+ * into the journal and removes it.
52
+ */
53
+ export const DEFERRED_FILE = "store-deferred.json";
54
+
55
+ /**
56
+ * Why no future turn can carry a message for this node, or null.
57
+ *
58
+ * `done` and `failed`-at-its-attempt-bound are the two states the scheduler
59
+ * skips for good, so a message queued for one of them is not early, it is
60
+ * undeliverable. Every other state can still be driven - an open ask gets an
61
+ * answer, a turn in flight ends - so a message queued there is only waiting.
62
+ *
63
+ * `maxAttempts` is the caller's policy, and a caller that does not have one
64
+ * passes null and learns only the `done` half: the command line cannot see the
65
+ * bound a supervisor is running with.
66
+ */
67
+ export function undeliverableReason(node, maxAttempts = null) {
68
+ if (node.state === "done") return "the node is done";
69
+ if (maxAttempts != null && node.state === "failed" && node.attempts >= maxAttempts) {
70
+ return `the node failed after ${node.attempts} attempts`;
71
+ }
72
+ return null;
73
+ }
74
+
75
+ export const DEFAULT_ANSWER = "No decision arrived before the deadline. Continue with the charter's stated default.";
76
+
77
+ /**
78
+ * Is this error SQLite saying "another process holds the write lock"?
79
+ *
80
+ * The primary result codes are what decide: 5 is SQLITE_BUSY, 6 is
81
+ * SQLITE_LOCKED, and 517 is SQLITE_BUSY_SNAPSHOT, which node:sqlite can report
82
+ * as an extended code. The message is the fallback for a driver that hands over
83
+ * a bare Error, and it matches SQLite's own strings rather than the word
84
+ * "locked" on its own, so a schema complaint cannot be mistaken for contention.
85
+ */
86
+ export function isBusyError(error) {
87
+ const code = Number(error?.errcode);
88
+ if (code === 5 || code === 6 || code === 517) return true;
89
+ return /database is locked|database table is locked|SQLITE_BUSY/i.test(String(error?.message ?? ""));
90
+ }
91
+
92
+ /**
93
+ * Block the current thread for `ms`. The drain is synchronous on purpose: its
94
+ * claim and its queue are one transaction, and a promise there would mean the
95
+ * transaction spans an await, which is the shape that lets a second writer in
96
+ * between the two halves. So the pause between two attempts blocks.
97
+ */
98
+ function sleepSync(ms) {
99
+ if (ms <= 0) return;
100
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
101
+ }
102
+
103
+ /** A JSON meta value, or null when the key is absent or unreadable. */
104
+ function jsonMeta(meta, key) {
105
+ if (!meta[key]) return null;
106
+ try {
107
+ return JSON.parse(meta[key]);
108
+ } catch {
109
+ return null;
110
+ }
111
+ }
112
+
113
+ /**
114
+ * The deferral a wedged or dead supervisor left beside the store, or null.
115
+ *
116
+ * Exported because `status` reads it too: when the store cannot be opened at all,
117
+ * this is the one fact about why the fleet is idle that is still readable.
118
+ */
119
+ export function deferralFromDisk(root) {
120
+ try {
121
+ return JSON.parse(readFileSync(join(root, DEFERRED_FILE), "utf8"));
122
+ } catch {
123
+ return null;
124
+ }
125
+ }
126
+
127
+ /**
128
+ * Where the control socket lives.
129
+ *
130
+ * A unix socket path is capped (104 bytes on macOS, 108 on Linux), and a fleet
131
+ * root can be deeper than that - a nested project, a scratch tree, a CI
132
+ * workspace. The socket therefore falls back to a short hashed name in the
133
+ * system temp directory, and both the supervisor and the commands derive the
134
+ * same path from the root, so the channel does not depend on where the store
135
+ * happens to sit. The chosen path is journalled as `control_socket`.
136
+ */
137
+ export function controlSocketPath(root) {
138
+ const absolute = resolveFleetRoot(root);
139
+ const local = join(absolute, "control.sock");
140
+ if (Buffer.byteLength(local) <= 100) return local;
141
+ const hash = createHash("sha256").update(absolute).digest("hex").slice(0, 12);
142
+ return join(tmpdir(), `pi-fleet-${hash}.sock`);
143
+ }
144
+
145
+ /**
146
+ * How long the arrival probe waits for an answer before it treats the path as a
147
+ * departed binder's leavings. A supervisor answers a ping in microseconds; the
148
+ * wait is long enough that a busy tick is not mistaken for a dead channel.
149
+ */
150
+ const CONTROL_PROBE_MS = 1_000;
151
+
152
+ /**
153
+ * The identity of whatever file sits at `path`: a unix socket's device and
154
+ * inode are stable for the life of the file, and a rebind at the same path
155
+ * creates a different inode. Used to answer "is the path still the one this
156
+ * process bound?" from the filesystem rather than from memory of a name.
157
+ *
158
+ * Returns null when nothing is there, and throws when the question cannot be
159
+ * answered at all: a caller that cannot tell must not delete.
160
+ */
161
+ function fileIdentity(path) {
162
+ try {
163
+ const stat = statSync(path);
164
+ return { dev: stat.dev, ino: stat.ino };
165
+ } catch (error) {
166
+ if (error.code === "ENOENT") return null;
167
+ throw error;
168
+ }
169
+ }
170
+
171
+ /**
172
+ * One request, one response line, over a unix socket path. Resolves `null` when
173
+ * the path does not answer within the timeout: absent, refused, or silent. The
174
+ * wire format lives with the server it speaks to, so the supervisor's own
175
+ * arrival probe and the commands share one implementation.
176
+ */
177
+ export function requestOnControlSocket(path, request, { timeoutMs = 10_000 } = {}) {
178
+ return new Promise((resolveReply) => {
179
+ const socket = connect(path);
180
+ let buffer = "";
181
+ const finish = (reply) => {
182
+ socket.destroy();
183
+ resolveReply(reply);
184
+ };
185
+ socket.setTimeout(timeoutMs, () => finish(null));
186
+ socket.on("connect", () => socket.write(JSON.stringify(request) + "\n"));
187
+ socket.on("data", (chunk) => {
188
+ buffer += chunk.toString("utf8");
189
+ const index = buffer.indexOf("\n");
190
+ if (index < 0) return;
191
+ try {
192
+ finish(JSON.parse(buffer.slice(0, index)));
193
+ } catch {
194
+ finish(null);
195
+ }
196
+ });
197
+ socket.on("error", () => finish(null));
198
+ });
199
+ }
200
+
201
+ export const CHILD_FLAGS = [
202
+ "--mode",
203
+ "rpc",
204
+ "--no-mcp",
205
+ "--no-context-files",
206
+ "--no-extensions",
207
+ "--no-skills",
208
+ "--no-prompt-templates",
209
+ "--thinking",
210
+ "off",
211
+ ];
212
+
213
+ /**
214
+ * The queue section of one prompt: the lines it appends for the messages the
215
+ * turn carries, and the ids those lines are.
216
+ *
217
+ * Both halves come out of one array here, and no other code builds the section
218
+ * by hand, so the answer to "what did this turn carry?" is the section that went
219
+ * into the prompt rather than a second reading of the finished text. That is the
220
+ * difference between a delivery and a coincidence: a charter written beside a
221
+ * queued message very often repeats the operator's instruction word for word,
222
+ * and a body found by search is not a message the model was given.
223
+ */
224
+ export function queueSection(header, pending) {
225
+ const entries = pending.map((message) => ({ msgId: message.msg_id, line: `- ${message.body}` }));
226
+ return { header, lines: entries.map((entry) => entry.line), entries };
227
+ }
228
+
229
+ /**
230
+ * Which of the section's messages the prompt really carries.
231
+ *
232
+ * The builder already says what it appended, and the text has to agree: the
233
+ * section's own sentence must be there, and so must the message's own line. A
234
+ * turn kind that appends no section carries nothing, and a prompt that repeats a
235
+ * body in another section - above all a charter - is not read as one.
236
+ */
237
+ export function carriedBy(prompt, section) {
238
+ if (section.entries.length === 0) return [];
239
+ if (!prompt.includes(section.header)) return [];
240
+ return section.entries.filter((entry) => prompt.includes(entry.line)).map((entry) => entry.msgId);
241
+ }
242
+
243
+ /** A prompt assembled around its queue section: the head, the section, the tail. */
244
+ function withSection(head, section, tail = []) {
245
+ const lines = [...head];
246
+ if (section.entries.length > 0) {
247
+ if (lines.length > 0) lines.push("");
248
+ lines.push(section.header, ...section.lines);
249
+ }
250
+ return { prompt: [...lines, ...tail].join("\n"), section };
251
+ }
252
+
253
+ /** The fleet preamble a node's first turn carries, before its charter body. */
254
+ export function charterPrompt(node, body, pending = []) {
255
+ const lines = [
256
+ `You are node ${node.node_id} of a fleet. Your charter follows.`,
257
+ `Work the charter with your tools. One tool call per step, in order.`,
258
+ `When the charter's steps are complete, end your turn with the single line ${DONE_SENTINEL}.`,
259
+ `If you cannot continue without a decision from a human, end your turn with one line:`,
260
+ `${ASK_SENTINEL} <question> | default: <the answer to use if nobody replies> | priority: <number, 0 when unsure>`,
261
+ "",
262
+ "CHARTER:",
263
+ body.trim(),
264
+ ];
265
+ // A queued message is an operator's instruction for this node, and the first
266
+ // turn is a turn: it rides along instead of waiting for a later one. The
267
+ // charter keeps its shape, and the queue is a section of its own after it.
268
+ return withSection(
269
+ lines,
270
+ queueSection("The operator sent this before your first turn. It applies to this turn as well:", pending),
271
+ );
272
+ }
273
+
274
+ /** The between-turns turn: the queued messages, and nothing else to read. */
275
+ export function messagePrompt(pending) {
276
+ return withSection([], queueSection("FLEET MESSAGE.", pending), ["", "Act on it, then end your turn."]);
277
+ }
278
+
279
+ /**
280
+ * The recovery prompt is built from the journal, not from the charter. A node
281
+ * that sees its own completed effects listed cannot be told to start over.
282
+ *
283
+ * `note` is the journal account, and it can be suppressed: the note-removal arm
284
+ * measures what the resumed turn does when the prompt is silent about the
285
+ * effects already applied. Nothing but that arm sets it to false.
286
+ */
287
+ export function recoveryPrompt({ node, interruptedTurn, completed, unknown, pending, note = true }) {
288
+ const lines = [
289
+ `FLEET RESUME. Node ${node.node_id} keeps the same session. Turn ${interruptedTurn.turn_id} was interrupted while it was running.`,
290
+ ];
291
+ if (note && completed.length > 0) {
292
+ lines.push("", "Effects already applied and recorded (do NOT repeat them):");
293
+ for (const effect of completed) {
294
+ lines.push(`- seq ${effect.seq} ${effect.tool_name} completed (idempotency ${effect.idempotency_key})`);
295
+ }
296
+ }
297
+ if (note && unknown.length > 0) {
298
+ lines.push("", "Effects whose outcome is unknown (the tool started and never reported):");
299
+ for (const effect of unknown) {
300
+ lines.push(`- seq ${effect.seq} ${effect.tool_name} unknown (idempotency ${effect.idempotency_key})`);
301
+ }
302
+ // The guard refuses a repeat of these, and the note has to say which gate is
303
+ // closed: a node told to "check the world" without being told what will stop
304
+ // it reads the line as a caution, and one that discovers the refusal by
305
+ // hitting it spends a turn learning what the prompt could have said. The
306
+ // release is named as an operator's decision because it is: a node that can
307
+ // retire its own rows is policing itself.
308
+ lines.push(
309
+ "Re-issuing one of these is refused: an effect whose outcome is unknown counts as history the guard will not let you repeat, and releasing it is an operator's decision, not this turn's. Check the world before acting on these; never repeat a completed effect.",
310
+ );
311
+ }
312
+ if (note && completed.length === 0 && unknown.length === 0) {
313
+ lines.push("", "No effect had been recorded when the turn was interrupted.");
314
+ }
315
+ return withSection(lines, queueSection("Messages waiting for you:", pending), [
316
+ "",
317
+ "Continue the charter from its first unfinished step.",
318
+ ]);
319
+ }
320
+
321
+ /**
322
+ * How much of a child's stderr one turn's log line shows. The driver keeps the
323
+ * last 8000 characters, and the whole of it stays on the driver object; a log
324
+ * line is a report, not a transcript, so it is capped here and names what it
325
+ * withheld rather than trailing off.
326
+ */
327
+ const CHILD_STDERR_SHOWN = 400;
328
+
329
+ /**
330
+ * The log line for a child that wrote to stderr, or null when it wrote nothing.
331
+ *
332
+ * A child's stderr is where a guard that failed open says so, and a fail-open
333
+ * guard has no other visible product: it refuses nothing, records nothing, and
334
+ * the turn looks exactly like one the guard watched and found clean. A wrong
335
+ * FLEET_GUARD_ROOT, the likeliest way to be unguarded, leaves its only trace
336
+ * here, so it is reported on every turn that produced any stderr at all.
337
+ *
338
+ * Collapsed to one line and tail-only, because the alternative is a noisy child
339
+ * deciding how large the supervisor's log gets.
340
+ */
341
+ export function childStderrReport(stderr, { limit = CHILD_STDERR_SHOWN } = {}) {
342
+ const text = String(stderr ?? "");
343
+ if (!text.trim()) return null;
344
+ const collapsed = text.replace(/\s+/g, " ").trim();
345
+ const over = Math.max(0, collapsed.length - limit);
346
+ return {
347
+ chars: text.length,
348
+ withheld: over,
349
+ shown: over > 0 ? `[${over} chars withheld] ${collapsed.slice(-limit)}` : collapsed,
350
+ };
351
+ }
352
+
353
+ export function parseAsk(text) {
354
+ const line = String(text ?? "")
355
+ .split("\n")
356
+ .find((candidate) => candidate.includes(ASK_SENTINEL));
357
+ if (!line) return null;
358
+ const rest = line.slice(line.indexOf(ASK_SENTINEL) + ASK_SENTINEL.length).trim();
359
+ const [question, ...tail] = rest.split("|");
360
+ let defaultAnswer = "";
361
+ let priority = 0;
362
+ for (const field of tail) {
363
+ const [rawKey, ...rawValue] = field.split(":");
364
+ const key = rawKey.trim().toLowerCase();
365
+ const value = rawValue.join(":").trim();
366
+ if (key === "default") defaultAnswer = value;
367
+ if (key === "priority") priority = Number(value) || 0;
368
+ }
369
+ return { question: question.trim(), defaultAnswer: defaultAnswer || DEFAULT_ANSWER, priority };
370
+ }
371
+
372
+ export function isDone(text) {
373
+ return String(text ?? "").includes(DONE_SENTINEL);
374
+ }
375
+
376
+ /** Text content of an assistant message, with thinking blocks dropped. */
377
+ export function assistantText(message) {
378
+ const content = message?.content;
379
+ if (typeof content === "string") return content;
380
+ if (!Array.isArray(content)) return "";
381
+ return content
382
+ .filter((block) => block?.type === "text" && typeof block.text === "string")
383
+ .map((block) => block.text)
384
+ .join("\n");
385
+ }
386
+
387
+ export class Supervisor {
388
+ constructor(options) {
389
+ const defaults = {
390
+ cap: 3,
391
+ tickMs: 250,
392
+ askTtlMs: 600_000,
393
+ turnTimeoutMs: 300_000,
394
+ maxAttempts: 3,
395
+ idleExitMs: 5_000,
396
+ // The maintenance budget: how long a pass may spend meeting another
397
+ // process's write lock before it is recorded as deferred and left to a
398
+ // later tick. Two windows at ten seconds plus one backoff is the most a
399
+ // tick's maintenance can spend waiting. Nothing here bounds a lease: a
400
+ // lease is exclusive for as long as its owner answers (journal.mjs).
401
+ drainAttempts: 2,
402
+ drainBusyTimeoutMs: BUSY_TIMEOUT_MS,
403
+ drainBackoffMs: 250,
404
+ // How long a single drain may block before the fact is journalled, even
405
+ // though it completed: one attempt that waited 7s for the lock is a fact
406
+ // an operator wants, not a silent success.
407
+ drainWaitReportMs: 1_000,
408
+ driver: "rpc",
409
+ model: "opencode-go/deepseek-v4.1-flash",
410
+ piBin: "pi",
411
+ naive: false,
412
+ control: true,
413
+ // What a prompt appends for the messages it carries. The only caller that
414
+ // turns it off is the queue arm, which needs a prompt whose body appears
415
+ // somewhere else in the text; no flag maps to it, and it can only make a
416
+ // turn carry less.
417
+ queueSection: true,
418
+ extensions: [],
419
+ extraArgs: [],
420
+ };
421
+ // An explicit `undefined` option means "not given", never "override with
422
+ // nothing": the command line produces undefined for every flag it omits.
423
+ this.options = { ...defaults };
424
+ for (const [key, value] of Object.entries(options ?? {})) {
425
+ if (value !== undefined) this.options[key] = value;
426
+ }
427
+ this.journal = openJournal(this.options.root, {
428
+ projectRoot: this.options.projectRoot ?? this.options.root,
429
+ driver: this.options.driver,
430
+ childArgs: this.options.extraArgs ?? [],
431
+ piVersion: this.options.piVersion ?? "unset",
432
+ });
433
+ this.root = this.journal.root;
434
+ this.epoch = null;
435
+ this.active = new Map();
436
+ this.timer = null;
437
+ this.server = null;
438
+ // The file this process bound, as the filesystem names it. A name is not an
439
+ // identity: another process can rebind the same path, so the stop below
440
+ // compares this against the path rather than trusting the name.
441
+ this.boundSocket = null;
442
+ this.stopping = false;
443
+ this.fences = [];
444
+ this.idleSince = null;
445
+ // A maintenance pass that could not write: the store was locked when the
446
+ // fact was made. It is written beside the store at once and journalled at
447
+ // the first pass that gets through, so it outlives this process - the
448
+ // measured case is a tick that threw `database is locked` and left an
449
+ // overdue ask open with nothing saying why.
450
+ this.pendingStoreDeferral = null;
451
+ // Resolved when this process has stopped itself because the fleet had
452
+ // nothing left to do; the entry point exits on it rather than on a guess.
453
+ this.done = new Promise((resolveDone) => {
454
+ this.doneResolve = resolveDone;
455
+ });
456
+ this.log = this.options.log ?? ((line) => console.log(line));
457
+ }
458
+
459
+ async start() {
460
+ // The journal records the driver and the version it ran under, so the
461
+ // choice between an RPC child and an in-process SDK stays reversible.
462
+ if (!this.options.piVersion && this.options.driver !== "scripted") {
463
+ try {
464
+ this.options.piVersion = execFileSync(this.options.piBin, ["--version"], { encoding: "utf8" }).trim();
465
+ } catch (error) {
466
+ this.options.piVersion = `unavailable: ${error.code ?? error.message}`;
467
+ }
468
+ this.journal.setMeta("pi_version", this.options.piVersion);
469
+ }
470
+ this.epoch = this.journal.registerSupervisor({
471
+ pid: process.pid,
472
+ driver: this.options.driver,
473
+ childArgs: this.options.extraArgs ?? [],
474
+ piVersion: this.options.piVersion,
475
+ });
476
+ this.log(`fleet: supervisor pid ${process.pid} epoch ${this.epoch} store ${this.journal.path}`);
477
+ await this.reconcile();
478
+ if (this.options.control) await this._listen();
479
+ this._schedule();
480
+ return this;
481
+ }
482
+
483
+ // ── reconciliation ───────────────────────────────────────────────────────
484
+
485
+ /**
486
+ * Repair what a dead predecessor left: dead epoch rows, expired leases, and
487
+ * every turn still marked in-progress. The turn's process group is fenced
488
+ * before its effects are read, so the next drive cannot race a writer that
489
+ * is still alive.
490
+ *
491
+ * A lease and a turn left behind by the same owner are the same owner's, so
492
+ * both loops ask `ownerLiveness` whether that owner is still there. Fencing a
493
+ * live owner's turn kills a child that is still working and records a
494
+ * recovery that never happened.
495
+ */
496
+ async reconcile() {
497
+ const supervisors = this.journal.supervisors();
498
+ for (const supervisor of supervisors) {
499
+ if (supervisor.pid === process.pid) continue;
500
+ if (supervisor.state === "live" && !pidIsAlive(supervisor.pid)) {
501
+ this.journal.setSupervisorState(supervisor.pid, "dead");
502
+ }
503
+ }
504
+ const owners = this.ownerLiveness();
505
+ for (const lease of this.journal.liveLeases()) {
506
+ if (lease.epoch === this.epoch) continue;
507
+ if (owners.get(lease.epoch)) continue;
508
+ this.journal.expireLease(lease.lease_id);
509
+ }
510
+ let skippedTurns = 0;
511
+ const openTurns = this.journal.allOpenTurns();
512
+ for (const turn of openTurns) {
513
+ if (turn.supervisor_epoch === this.epoch) continue;
514
+ if (owners.get(turn.supervisor_epoch)) {
515
+ skippedTurns += 1;
516
+ continue;
517
+ }
518
+ const fence = await this.fenceGroup(turn);
519
+ const unknown = this.journal.markUnknownEffects(turn.turn_id);
520
+ this.journal.endTurn(turn.turn_id, { state: "interrupted" });
521
+ // Messages the interrupted turn had already recorded as carried go back to
522
+ // the queue: the prompt went out and the turn did not finish, so the
523
+ // recovery turn below is the first place they can be acted on.
524
+ const abandoned = this.journal
525
+ .messages(turn.node_id)
526
+ .filter((message) => message.state === "delivered" && message.delivered_turn_id === turn.turn_id)
527
+ .map((message) => message.msg_id);
528
+ const requeued = this.journal.releaseMessages(abandoned);
529
+ this.journal.bumpAttempt(turn.node_id);
530
+ const node = this.journal.getNode(turn.node_id);
531
+ if (node && node.state !== "done") {
532
+ // The bound closes the crash loop: without it a node that kills its
533
+ // supervisor every time is interrupted, settled, resumed, and
534
+ // interrupted again, for ever.
535
+ if (node.attempts >= this.options.maxAttempts) {
536
+ this.journal.setNodeState(turn.node_id, "failed", {
537
+ done_reason: `attempt bound: ${node.attempts} failed or interrupted attempts reached maxAttempts ${this.options.maxAttempts}`,
538
+ });
539
+ this.log(
540
+ `fleet: node ${turn.node_id} reached the attempt bound (${node.attempts}/${this.options.maxAttempts}); not resuming it`,
541
+ );
542
+ } else {
543
+ this.journal.setNodeState(turn.node_id, "settled");
544
+ }
545
+ }
546
+ this.journal.setMeta("fences", JSON.stringify(this.fences));
547
+ this.log(
548
+ `fleet: reconciled turn ${turn.turn_id} (node ${turn.node_id}) fence=${fence.killed ? "killed" : fence.reason} unknown_effects=${unknown.length} requeued_messages=${requeued}`,
549
+ );
550
+ }
551
+ this.journal.setMeta("reconcile_skipped_turns", String(skippedTurns));
552
+ this.drainExpiredAsks();
553
+ }
554
+
555
+ /**
556
+ * Which supervisor epochs are still alive, by the one predicate both repair
557
+ * loops need: a row that says `live` whose pid answers. The repair pass above
558
+ * has already demoted a live row whose pid is gone, so a `live` row here is a
559
+ * process the box still holds.
560
+ */
561
+ ownerLiveness() {
562
+ const owners = new Map();
563
+ for (const row of this.journal.supervisors()) {
564
+ owners.set(row.epoch, row.state === "live" && pidIsAlive(row.pid));
565
+ }
566
+ return owners;
567
+ }
568
+
569
+ // ── the ask deadline ─────────────────────────────────────────────────────
570
+
571
+ /**
572
+ * The one drain of expired asks. Resolving an ask to its default is only half
573
+ * the job: nothing tells the node to move unless the default is also queued
574
+ * as an ordinary message. Both callers - the tick, and the restart that finds
575
+ * a deadline passed while no supervisor was running - come through here, so
576
+ * the two paths cannot drift into one of them doing half the work.
577
+ *
578
+ * Only what `expireAsks()` claimed is queued. It returns the rows this call
579
+ * actually transitioned, not the rows it read, so a second drainer of the same
580
+ * ask claims nothing and queues nothing: the queue decision rests on the
581
+ * conditional update, not on a shared read.
582
+ *
583
+ * Contention is waited out, not thrown. A drain that meets the store's write
584
+ * lock waits out the journal's busy timeout, retries with a backoff, and
585
+ * reports a drain it could not finish as a fact (`store_deferred`) instead of
586
+ * as an exception from a tick. The earlier shape was measured: two drainers on
587
+ * one store, a full-suite run, one of them `database is locked` and gone, and
588
+ * the asks it was there to drain stayed held. A supervisor whose drain throws
589
+ * does not drain that ask, and its caller is a tick handler, so the failure
590
+ * was a log line and a skipped maintenance pass.
591
+ */
592
+ drainExpiredAsks() {
593
+ const started = Date.now();
594
+ const attempts = Math.max(1, Math.floor(this.options.drainAttempts));
595
+ for (let attempt = 1; attempt <= attempts; attempt += 1) {
596
+ try {
597
+ const { expired, notes } = this.journal.transaction(() => this._claimExpiredAsks(), {
598
+ busyTimeoutMs: this.options.drainBusyTimeoutMs,
599
+ });
600
+ // After the commit, so a retried attempt cannot log the same claim
601
+ // twice, and so the log line describes a decision that survived.
602
+ for (const note of notes) this.log(note);
603
+ this._recordDrainContention({ attempt, waitedMs: Date.now() - started });
604
+ return expired;
605
+ } catch (error) {
606
+ // A locked store is the one error this drain answers itself. Anything
607
+ // else is a defect in the body, and swallowing it would hide a real
608
+ // failure behind a maintenance pass nobody reads.
609
+ if (!isBusyError(error)) throw error;
610
+ const why = error.errstr ?? error.message;
611
+ if (attempt >= attempts) {
612
+ this._recordStoreDeferral({ what: "drain", attempts: attempt, waitedMs: Date.now() - started, error: why });
613
+ return [];
614
+ }
615
+ const backoffMs = this.options.drainBackoffMs * attempt;
616
+ this.log(
617
+ `fleet: drain met ${why} on attempt ${attempt}/${attempts}; waiting ${backoffMs} ms and retrying`,
618
+ );
619
+ sleepSync(backoffMs);
620
+ }
621
+ }
622
+ return [];
623
+ }
624
+
625
+ /**
626
+ * The drain's body: claim the overdue asks and queue one message per claimed
627
+ * row. It runs inside the transaction, so it must stay synchronous and free
628
+ * of side effects that a rollback cannot undo - the log lines are returned to
629
+ * the caller rather than printed here.
630
+ */
631
+ _claimExpiredAsks() {
632
+ // Claim and queue as one transaction. A supervisor killed between the two
633
+ // leaves an ask answered with no message queued for it, and a node parked
634
+ // on that ask then has no open ask, no pending message and no ready kind,
635
+ // so nothing ever drives it again: the parked-forever shape this drain
636
+ // exists to prevent, reached through the drain itself. BEGIN IMMEDIATE also
637
+ // serializes two drainers, so the read that decides a claim cannot land
638
+ // inside the other process's claims.
639
+ const expired = this.journal.expireAsks();
640
+ const notes = [];
641
+ for (const ask of expired) {
642
+ // The ask is claimed and answered by now; what is left is whether the
643
+ // node should be woken. A done node must not be resurrected: the fence
644
+ // loop guards this, and the drain queued for it anyway, so an overdue
645
+ // ask on a finished node flipped it back to `settled` and the default
646
+ // then drove it a second time.
647
+ const node = this.journal.getNode(ask.node_id);
648
+ if (!node || node.state === "done") {
649
+ notes.push(
650
+ `fleet: ask ${ask.token} expired for node ${ask.node_id}, which is ${node ? "done" : "missing"}; default not queued`,
651
+ );
652
+ continue;
653
+ }
654
+ this.journal.addMessage({
655
+ msgId: `m-${randomBytes(6).toString("hex")}`,
656
+ nodeId: ask.node_id,
657
+ kind: "answer",
658
+ body: ask.default_answer,
659
+ token: ask.token,
660
+ });
661
+ this.journal.setNodeState(ask.node_id, "settled");
662
+ notes.push(`fleet: ask ${ask.token} hit its deadline; default answer queued`);
663
+ }
664
+ return { expired, notes };
665
+ }
666
+
667
+ /**
668
+ * Record that a drain waited for the lock, and flush a deferral a previous
669
+ * drain could not write.
670
+ *
671
+ * Neither write is allowed to throw: this runs after a successful drain, and
672
+ * losing the note because another writer holds the lock again is not a reason
673
+ * to report the drain itself as failed. The deferral stays pending until a
674
+ * write lands, which is the earliest point the store can hold the fact at all.
675
+ */
676
+ _recordDrainContention({ attempt, waitedMs }) {
677
+ this._flushStoreDeferral(Date.now());
678
+ if (attempt <= 1 && waitedMs < this.options.drainWaitReportMs) return;
679
+ try {
680
+ this.journal.transaction(
681
+ () =>
682
+ this.journal.setMeta("drain_contention", JSON.stringify({ at: Date.now(), attempts: attempt, waitedMs })),
683
+ { busyTimeoutMs: Math.max(100, this.options.drainBackoffMs) },
684
+ );
685
+ } catch (error) {
686
+ // Best effort: the drain's decision is already committed and correct.
687
+ this.log(`fleet: the drain could not record its wait (${error.message}); the next drain retries the note`);
688
+ }
689
+ }
690
+
691
+ /** The path a fact goes to when the store will not take it. */
692
+ deferredPath() {
693
+ return join(this.root, DEFERRED_FILE);
694
+ }
695
+
696
+ /**
697
+ * The deferral as it stands: this process's, or one an earlier process left
698
+ * beside the store. Reading the file is what makes the fact survive a death,
699
+ * and it is also what lets the next supervisor finish the job.
700
+ */
701
+ _pendingDeferral() {
702
+ if (this.pendingStoreDeferral) return this.pendingStoreDeferral;
703
+ const onDisk = deferralFromDisk(this.root);
704
+ if (onDisk) this.pendingStoreDeferral = onDisk;
705
+ return this.pendingStoreDeferral;
706
+ }
707
+
708
+ /**
709
+ * Put the fact where it can be read with no lock: a file beside the journal.
710
+ *
711
+ * There is no second try if this fails - the store is unwritable and so is the
712
+ * root - so the log line is the last surface left, and it says so.
713
+ */
714
+ _writeDeferralFile(fact) {
715
+ const path = this.deferredPath();
716
+ try {
717
+ writeFileSync(`${path}.tmp`, `${JSON.stringify(fact, null, 2)}\n`);
718
+ renameSync(`${path}.tmp`, path);
719
+ } catch (error) {
720
+ this.log(`fleet: the deferral could not be written to ${path} (${error.code ?? error.message})`);
721
+ }
722
+ }
723
+
724
+ /**
725
+ * Copy a pending deferral into the journal, if one is pending. `recoveredAt`
726
+ * says whether a later pass got the work done anyway, which is what tells a
727
+ * reader whether the store was merely busy or the work is still waiting.
728
+ *
729
+ * The file is removed only after the journal has the fact, so a crash between
730
+ * the two leaves it readable rather than losing it; the next pass writes it
731
+ * again.
732
+ */
733
+ _flushStoreDeferral(recoveredAt) {
734
+ const pending = this._pendingDeferral();
735
+ if (!pending) return false;
736
+ const fact = recoveredAt == null ? pending : { ...pending, recoveredAt };
737
+ try {
738
+ this.journal.transaction(() => this.journal.setMeta("store_deferred", JSON.stringify(fact)), {
739
+ busyTimeoutMs: Math.max(100, this.options.drainBackoffMs),
740
+ });
741
+ } catch (error) {
742
+ this.log(`fleet: the deferral could not be journalled yet (${error.message}); the next pass retries it`);
743
+ return false;
744
+ }
745
+ this.pendingStoreDeferral = null;
746
+ rmSync(this.deferredPath(), { force: true });
747
+ if (recoveredAt != null) this.log("fleet: the earlier deferral was journalled; a later pass completed the work");
748
+ return true;
749
+ }
750
+
751
+ /**
752
+ * A maintenance pass that gave up on a locked store. The store cannot hold
753
+ * this fact - that is the fact - so it goes to the file first, then into the
754
+ * journal at the first moment the store accepts a write. It is not an
755
+ * exception: the tick keeps ticking, and the pass is retried.
756
+ */
757
+ _recordStoreDeferral({ what, attempts, waitedMs, error }) {
758
+ const fact = { at: Date.now(), what, attempts, waitedMs, error };
759
+ this.pendingStoreDeferral = fact;
760
+ this._writeDeferralFile(fact);
761
+ this.log(
762
+ `fleet: ${what} could not write after ${attempts} attempt(s) in ${waitedMs} ms (${error}); recorded in ${DEFERRED_FILE} until the store takes it`,
763
+ );
764
+ this._flushStoreDeferral(null);
765
+ }
766
+
767
+ /**
768
+ * Kill the process group a dead turn left behind, and wait until the group is
769
+ * gone before reading its effects. A signal that was sent is not a fence that
770
+ * held, so the record carries what was observed after the wait.
771
+ */
772
+ async fenceGroup(turn) {
773
+ const pgid = turn.child_pgid;
774
+ const record = {
775
+ turnId: turn.turn_id,
776
+ nodeId: turn.node_id,
777
+ pgid,
778
+ killed: false,
779
+ reason: "",
780
+ at: Date.now(),
781
+ waitedMs: 0,
782
+ };
783
+ if (!pgid) {
784
+ record.reason = "no child recorded";
785
+ } else if (!groupIsAlive(pgid)) {
786
+ record.reason = "group already gone";
787
+ } else {
788
+ try {
789
+ process.kill(-pgid, "SIGKILL");
790
+ record.killed = true;
791
+ record.reason = "SIGKILL to process group";
792
+ } catch (error) {
793
+ record.reason = `kill failed: ${error.code ?? error.message}`;
794
+ }
795
+ const started = Date.now();
796
+ while (groupIsAlive(pgid) && Date.now() - started < 500) {
797
+ await new Promise((resolveWait) => setTimeout(resolveWait, 10));
798
+ }
799
+ record.waitedMs = Date.now() - started;
800
+ }
801
+ record.aliveAfter = groupIsAlive(pgid);
802
+ this.fences.push(record);
803
+ return record;
804
+ }
805
+
806
+ // ── scheduling ───────────────────────────────────────────────────────────
807
+
808
+ _schedule() {
809
+ if (this.stopping) return;
810
+ this.timer = setTimeout(() => {
811
+ this.tick()
812
+ .catch((error) => this.log(`fleet: tick failed: ${error.message}`))
813
+ .finally(() => this._schedule());
814
+ }, this.options.tickMs);
815
+ }
816
+
817
+ /**
818
+ * The tick's maintenance pass: the expired asks, then the queued messages no
819
+ * turn will ever carry.
820
+ *
821
+ * One named pass because it is the part of a tick another process's write lock
822
+ * can stall, and because both halves are retried by the next tick rather than
823
+ * failed for good - which is only true if the failure is recorded. Returns what
824
+ * the drain claimed, so a caller can say what the pass did.
825
+ */
826
+ maintenance() {
827
+ const expired = this.drainExpiredAsks();
828
+ this._maintenanceWrite("undeliverable sweep", () => this.retireUndeliverableMessages());
829
+ return expired;
830
+ }
831
+
832
+ /**
833
+ * One maintenance write, with a busy store recorded instead of thrown.
834
+ *
835
+ * The drain answers its own contention; the pass after it meets the same lock
836
+ * and has no policy of its own, and it used to throw into the tick's catch: one
837
+ * log line, no fact, and the rest of the tick skipped. A store that will not
838
+ * take the write is a fact an operator can read, and the next tick retries the
839
+ * pass. Anything that is not contention is a defect and stays an exception.
840
+ */
841
+ _maintenanceWrite(what, body) {
842
+ const started = Date.now();
843
+ try {
844
+ this.journal.withBusyTimeout(Math.max(100, this.options.drainBusyTimeoutMs), body);
845
+ } catch (error) {
846
+ if (!isBusyError(error)) throw error;
847
+ this._recordStoreDeferral({
848
+ what,
849
+ attempts: 1,
850
+ waitedMs: Date.now() - started,
851
+ error: error.errstr ?? error.message,
852
+ });
853
+ }
854
+ }
855
+
856
+ async tick() {
857
+ this.journal.heartbeatSupervisor(process.pid);
858
+ for (const [nodeId, ctx] of this.active) {
859
+ const renewed = this.journal.acquireLease({
860
+ nodeId,
861
+ ownerPid: process.pid,
862
+ epoch: this.epoch,
863
+ });
864
+ if (!renewed) this.log(`fleet: lease for ${nodeId} could not be renewed (ctx ${ctx.turnId})`);
865
+ }
866
+ this.maintenance();
867
+ if (this.active.size < this.options.cap) {
868
+ for (const node of this.journal.listNodes()) {
869
+ if (this.stopping || this.active.size >= this.options.cap) break;
870
+ if (this.active.has(node.node_id)) continue;
871
+ const kind = this.readyKind(node);
872
+ if (!kind) continue;
873
+ this.driveNode(node, kind).catch((error) => {
874
+ this.log(`fleet: drive ${node.node_id} threw: ${error.message}`);
875
+ });
876
+ }
877
+ }
878
+ await this.idleCheck();
879
+ }
880
+
881
+ // ── stopping when there is nothing to do ────────────────────────────────
882
+
883
+ /**
884
+ * Whether the fleet has anything left to do. Three things are work at any
885
+ * capacity: a turn this process is running, an open ask (a cap-0 supervisor
886
+ * still drains an expired one, because the drain runs before the cap gate),
887
+ * and a lease this epoch has not let go of.
888
+ *
889
+ * A runnable node is work only when this process could drive it. At cap 0 the
890
+ * drive loop never runs, so a node the scheduler would name is a node this
891
+ * supervisor is structurally unable to start; counting it as work is how a
892
+ * cap-0 supervisor outlives its usefulness holding a journal full of nodes it
893
+ * will never drive.
894
+ */
895
+ hasWork() {
896
+ if (this.active.size > 0) return true;
897
+ if (this.journal.openAsks().length > 0) return true;
898
+ if (this.journal.liveLeases().some((lease) => lease.epoch === this.epoch)) return true;
899
+ if (this.options.cap <= 0) return false;
900
+ return this.journal.listNodes().some((node) => this.readyKind(node) !== null);
901
+ }
902
+
903
+ /**
904
+ * The idle exit: nothing to do for a sustained window means stop and let the
905
+ * process end. A single empty sample is not that window - a node is
906
+ * registered before the supervisor that drives it is started, so a fresh
907
+ * supervisor's first tick legitimately finds nothing ready - so the window is
908
+ * re-sampled on every tick and any work resets it. `idleExitMs: 0` disables
909
+ * the exit for a supervisor meant to wait.
910
+ */
911
+ async idleCheck() {
912
+ const windowMs = this.options.idleExitMs;
913
+ if (!windowMs || this.stopping) return;
914
+ if (this.hasWork()) {
915
+ this.idleSince = null;
916
+ return;
917
+ }
918
+ if (this.idleSince == null) {
919
+ this.idleSince = Date.now();
920
+ return;
921
+ }
922
+ const idleMs = Date.now() - this.idleSince;
923
+ if (idleMs < windowMs) return;
924
+ // One last look in the same synchronous stretch as the stop, so a node that
925
+ // arrived while the window elapsed is not exited from.
926
+ if (this.hasWork()) {
927
+ this.idleSince = null;
928
+ return;
929
+ }
930
+ this.log(`fleet: no work left for ${idleMs} ms; stopping`);
931
+ try {
932
+ await this.stop();
933
+ } catch (error) {
934
+ this.log(`fleet: stop during the idle exit failed: ${error.message}`);
935
+ this.doneResolve(false);
936
+ }
937
+ }
938
+
939
+ /**
940
+ * Why a node is ready to be driven right now, or null. The five states are
941
+ * the design's; `waiting_on_child` is repaired here rather than trusted,
942
+ * because a killed supervisor leaves it behind.
943
+ *
944
+ * Order matters: an interrupted turn outranks a queued message, because the
945
+ * recovery prompt carries the queued messages with it; a node that has never
946
+ * been driven takes its charter before anything else.
947
+ */
948
+ readyKind(node) {
949
+ if (this.journal.openAsks(node.node_id).length > 0) return null;
950
+ if (undeliverableReason(node, this.options.maxAttempts)) return null;
951
+ const turns = this.journal.turnsFor(node.node_id);
952
+ const last = turns.at(-1) ?? null;
953
+ if (last && last.state === "in_progress") return null;
954
+ if (turns.length === 0) return "charter";
955
+ if (last && last.state === "interrupted") return this.options.naive ? "naive" : "recovery";
956
+ if (this.journal.pendingMessages(node.node_id).length > 0) return "message";
957
+ return null;
958
+ }
959
+
960
+ /**
961
+ * Queued messages on a node that will never run again take a terminal state
962
+ * of their own.
963
+ *
964
+ * `recorded` is a promise that some turn will carry the message. When the node
965
+ * it was routed to is done, or failed at its attempt bound, no turn will ever
966
+ * do that, and leaving the row in `recorded` is the same silent lie as
967
+ * retiring a message that never reached a prompt. The sweep runs on every
968
+ * tick, so a message is stamped as soon as a supervisor can see that it cannot
969
+ * be delivered - including one that was queued while the node was still
970
+ * running and whose later turn ended the node instead of carrying it.
971
+ */
972
+ retireUndeliverableMessages() {
973
+ for (const node of this.journal.listNodes()) {
974
+ if (this.active.has(node.node_id)) continue;
975
+ const why = undeliverableReason(node, this.options.maxAttempts);
976
+ if (!why) continue;
977
+ const retired = this.journal.markMessagesUndeliverable(node.node_id);
978
+ if (retired.length > 0) {
979
+ this.log(
980
+ `fleet: ${why}; ${retired.length} queued message(s) for node ${node.node_id} can never be delivered: ${retired
981
+ .map((message) => message.msg_id)
982
+ .join(", ")}`,
983
+ );
984
+ }
985
+ }
986
+ }
987
+
988
+ // ── driving one node ─────────────────────────────────────────────────────
989
+
990
+ createDriver(node, turnId) {
991
+ const config = JSON.parse(node.driver_config || "{}");
992
+ const sessionId = node.session_id ?? `fleet-${node.node_id}-${randomBytes(3).toString("hex")}`;
993
+ if (node.driver === "scripted") {
994
+ return createScriptedDriver({
995
+ steps: config.steps ?? [],
996
+ worldFile: config.worldFile ?? join(this.root, "world.log"),
997
+ sessionId,
998
+ sessionFile: node.session_file ?? join(this.root, "sessions", `${sessionId}.jsonl`),
999
+ });
1000
+ }
1001
+ return createRpcDriver({
1002
+ piBin: this.options.piBin,
1003
+ cwd: node.project_root,
1004
+ sessionId,
1005
+ sessionDir: join(this.root, "sessions"),
1006
+ model: this.options.model,
1007
+ extensions: this.options.extensions,
1008
+ guardExtension: GUARD_EXTENSION,
1009
+ env: guardEnvironment({ root: this.root, nodeId: node.node_id, turnId }),
1010
+ extraArgs: this.options.extraArgs,
1011
+ fullDiscovery: this.options.fullDiscovery,
1012
+ thinking: this.options.thinking,
1013
+ });
1014
+ }
1015
+
1016
+ async driveNode(node, kind) {
1017
+ const lease = this.journal.acquireLease({
1018
+ nodeId: node.node_id,
1019
+ ownerPid: process.pid,
1020
+ epoch: this.epoch,
1021
+ });
1022
+ if (!lease) return;
1023
+ const ctx = {
1024
+ nodeId: node.node_id,
1025
+ turnId: null,
1026
+ driver: null,
1027
+ kind,
1028
+ toolEnds: 0,
1029
+ requestSeq: 0,
1030
+ effectSeq: 0,
1031
+ usage: { input: 0, output: 0 },
1032
+ settled: false,
1033
+ settleResolve: null,
1034
+ finalText: "",
1035
+ lastAssistantText: "",
1036
+ error: null,
1037
+ startedAt: Date.now(),
1038
+ // What this turn's prompt carries, and what was steered into it. Both are
1039
+ // decided by the turn itself; the settle gate retires exactly these.
1040
+ carried: [],
1041
+ steeredIds: [],
1042
+ };
1043
+ ctx.settlePromise = new Promise((resolveSettle) => {
1044
+ ctx.settleResolve = resolveSettle;
1045
+ });
1046
+ this.active.set(node.node_id, ctx);
1047
+ try {
1048
+ const refusal = this.journal.budgetRefusal(node.project_root, node.node_id);
1049
+ if (refusal) {
1050
+ this.journal.setNodeState(node.node_id, "failed", { done_reason: `budget: ${refusal}` });
1051
+ return;
1052
+ }
1053
+ const pending = this.journal.pendingMessages(node.node_id);
1054
+ // What the prompt is built with. The queue arm is the only caller that
1055
+ // passes none while messages are waiting, and it does so to build the
1056
+ // prompt the old carrier could not tell from a delivery: the body present
1057
+ // in the text, and no section that put it there.
1058
+ const queued = this.options.queueSection === false ? [] : pending;
1059
+ const turns = this.journal.turnsFor(node.node_id);
1060
+ const interrupted = turns.at(-1)?.state === "interrupted" ? turns.at(-1) : null;
1061
+ let built;
1062
+ let promptKind = kind;
1063
+ if (kind === "recovery" && interrupted) {
1064
+ built = recoveryPrompt({
1065
+ node,
1066
+ interruptedTurn: interrupted,
1067
+ completed: this.journal.settledEffects(interrupted.turn_id),
1068
+ unknown: this.journal
1069
+ .effects(interrupted.turn_id)
1070
+ .filter((effect) => effect.state === "unknown" || effect.state === "attempted"),
1071
+ pending: queued,
1072
+ // In-process only, and no command-line flag maps to it: the
1073
+ // note-removal arm is the single caller that turns the note off.
1074
+ note: this.options.recoveryNote !== false,
1075
+ });
1076
+ } else if (kind === "message") {
1077
+ built = messagePrompt(queued);
1078
+ } else {
1079
+ built = charterPrompt(node, this._charter(node), queued);
1080
+ if (kind === "naive") promptKind = "naive-replay";
1081
+ }
1082
+ const prompt = built.prompt;
1083
+ // The messages this turn carried, from the section the builder appended
1084
+ // and the text that holds it - never from a search of the finished prompt
1085
+ // for a body. A charter that repeats an operator instruction is not a
1086
+ // delivery of the message that happens to say the same thing: the row
1087
+ // keeps waiting, and the next turn that carries it retires it.
1088
+ ctx.carried = carriedBy(prompt, built.section);
1089
+ this.journal.setNodeState(node.node_id, "waiting_on_child");
1090
+ const turnId = this.journal.beginTurn(node.node_id, {
1091
+ kind: promptKind,
1092
+ promptText: prompt,
1093
+ sessionId: node.session_id,
1094
+ supervisorEpoch: this.epoch,
1095
+ });
1096
+ ctx.turnId = turnId;
1097
+ for (const msgId of ctx.carried) {
1098
+ this.journal.markMessageDelivered(msgId, {
1099
+ tier: "B",
1100
+ turnId,
1101
+ afterEffectSeq: 0,
1102
+ beforeRequestSeq: 0,
1103
+ });
1104
+ }
1105
+ const driver = this.createDriver(node, turnId);
1106
+ ctx.driver = driver;
1107
+ driver.onEvent((event) => this._onDriverEvent(ctx, event));
1108
+ const { sessionId, sessionFile } = await driver.start();
1109
+ ctx.sessionId = sessionId;
1110
+ this.journal.setNodeSession(node.node_id, { sessionId, sessionFile });
1111
+ if (driver.pid) this.journal.setTurnChild(turnId, { pid: driver.pid, pgid: driver.pgid });
1112
+ await driver.prompt(prompt);
1113
+ await this._waitForSettle(ctx);
1114
+ await this._settleTurn(node, ctx);
1115
+ } catch (error) {
1116
+ ctx.error = error.message;
1117
+ if (ctx.turnId) {
1118
+ this.journal.markUnknownEffects(ctx.turnId);
1119
+ this.journal.endTurn(ctx.turnId, { state: "failed", ...this._usage(ctx) });
1120
+ // The prompt went out and the turn did not finish it, so its messages
1121
+ // are queued again rather than left in a `delivered` row nothing will
1122
+ // ever read.
1123
+ this.journal.releaseMessages([...ctx.carried, ...ctx.steeredIds]);
1124
+ this.journal.bumpAttempt(node.node_id);
1125
+ }
1126
+ this.journal.setNodeState(node.node_id, "failed", { done_reason: `turn failed: ${error.message}` });
1127
+ this.log(`fleet: node ${node.node_id} turn failed: ${error.message}`);
1128
+ } finally {
1129
+ if (ctx.driver) await ctx.driver.stop().catch(() => {});
1130
+ // After the child is gone, so stderr written on the way out is included.
1131
+ const report = childStderrReport(ctx.driver?.stderr);
1132
+ if (report) {
1133
+ this.log(
1134
+ `fleet: node ${node.node_id} turn ${ctx.turnId ?? "-"} child stderr ${report.chars} chars: ${report.shown}`,
1135
+ );
1136
+ }
1137
+ this.journal.releaseLease(node.node_id, this.epoch);
1138
+ this.active.delete(node.node_id);
1139
+ }
1140
+ }
1141
+
1142
+ _charter(node) {
1143
+ try {
1144
+ return readFileSync(node.charter_path, "utf8");
1145
+ } catch (error) {
1146
+ return `(charter unreadable at ${node.charter_path}: ${error.code ?? error.message})`;
1147
+ }
1148
+ }
1149
+
1150
+ _usage(ctx) {
1151
+ return {
1152
+ tokensIn: ctx.usage.input,
1153
+ tokensOut: ctx.usage.output,
1154
+ spendUsd: 0,
1155
+ };
1156
+ }
1157
+
1158
+ _onDriverEvent(ctx, event) {
1159
+ if (!ctx.turnId) return;
1160
+ switch (event.type) {
1161
+ case "tool_start":
1162
+ ctx.effectSeq += 1;
1163
+ this.journal.effectAttempted({
1164
+ turnId: ctx.turnId,
1165
+ nodeId: ctx.nodeId,
1166
+ toolCallId: event.toolCallId,
1167
+ toolName: event.toolName,
1168
+ args: event.args,
1169
+ replay: replayPolicy(event.toolName),
1170
+ seq: ctx.effectSeq,
1171
+ });
1172
+ break;
1173
+ case "tool_end":
1174
+ ctx.toolEnds += 1;
1175
+ this.journal.effectSettled({
1176
+ turnId: ctx.turnId,
1177
+ toolCallId: event.toolCallId,
1178
+ result: event.result,
1179
+ isError: event.isError,
1180
+ });
1181
+ break;
1182
+ case "model_request":
1183
+ ctx.requestSeq = this.journal.recordModelRequest(ctx.turnId, {
1184
+ messageCount: event.messageCount,
1185
+ toolCount: event.toolCount,
1186
+ });
1187
+ break;
1188
+ case "message_end":
1189
+ if (event.role === "assistant") {
1190
+ ctx.lastAssistantText = assistantText(event.message);
1191
+ const usage = event.message?.usage;
1192
+ if (usage) {
1193
+ ctx.usage.input += Number(usage.input ?? 0);
1194
+ ctx.usage.output += Number(usage.output ?? 0);
1195
+ }
1196
+ }
1197
+ break;
1198
+ case "final_text":
1199
+ ctx.finalText = event.text;
1200
+ ctx.settled = true;
1201
+ ctx.settleResolve?.();
1202
+ break;
1203
+ case "steer_delivered": {
1204
+ // The only delivery this fleet can testify to: the child ran the text it
1205
+ // was given, in this turn. Rows are matched by the body the child
1206
+ // carried, so a delivery cannot be charged to a message that was never
1207
+ // run. Every message is recorded before the steer is requested, which
1208
+ // is what makes an event that arrives before the ack findable.
1209
+ const waiting = this.journal.pendingMessages(ctx.nodeId);
1210
+ if (event.text == null) {
1211
+ // A driver that cannot name the body leaves the oldest waiting row as
1212
+ // the only honest candidate; saying so is better than a silent guess.
1213
+ this.log(`fleet: node ${ctx.nodeId} reported a steer with no body; the oldest waiting message is charged with it`);
1214
+ }
1215
+ const matched = event.text == null ? waiting.slice(0, 1) : waiting.filter((message) => message.body === event.text);
1216
+ if (event.text != null && matched.length === 0) {
1217
+ this.log(`fleet: node ${ctx.nodeId} ran a steered message no queued row carried, so no row is marked delivered`);
1218
+ }
1219
+ for (const message of matched) {
1220
+ this.journal.markMessageDelivered(message.msg_id, {
1221
+ tier: "A",
1222
+ turnId: ctx.turnId,
1223
+ afterEffectSeq: event.afterEffectSeq,
1224
+ beforeRequestSeq: ctx.requestSeq,
1225
+ });
1226
+ if (!ctx.steeredIds.includes(message.msg_id)) ctx.steeredIds.push(message.msg_id);
1227
+ }
1228
+ break;
1229
+ }
1230
+ case "settled":
1231
+ ctx.finalText = ctx.finalText || ctx.lastAssistantText;
1232
+ ctx.settled = true;
1233
+ ctx.settleResolve?.();
1234
+ break;
1235
+ default:
1236
+ break;
1237
+ }
1238
+ }
1239
+
1240
+ async _waitForSettle(ctx) {
1241
+ const deadline = Date.now() + this.options.turnTimeoutMs;
1242
+ while (!ctx.settled) {
1243
+ if (Date.now() > deadline) throw new Error(`turn exceeded ${this.options.turnTimeoutMs} ms`);
1244
+ if (ctx.driver?.exit) {
1245
+ const exited = await Promise.race([ctx.driver.exit.then(() => true), new Promise((resolveWait) => setTimeout(() => resolveWait(false), 50))]);
1246
+ if (exited && !ctx.settled) throw new Error("driver exited before settling");
1247
+ }
1248
+ await new Promise((resolveWait) => setTimeout(resolveWait, 50));
1249
+ }
1250
+ }
1251
+
1252
+ async _settleTurn(node, ctx) {
1253
+ const text = ctx.finalText || ctx.lastAssistantText;
1254
+ const ask = parseAsk(text);
1255
+ let turnState = "settled";
1256
+ let nodeState = "settled";
1257
+ let doneReason = null;
1258
+ if (ask) {
1259
+ const token = `ask-${randomBytes(6).toString("hex")}`;
1260
+ this.journal.recordAsk({
1261
+ token,
1262
+ nodeId: node.node_id,
1263
+ turnId: ctx.turnId,
1264
+ priority: ask.priority,
1265
+ question: ask.question,
1266
+ defaultAnswer: ask.defaultAnswer,
1267
+ deadlineAt: Date.now() + this.options.askTtlMs,
1268
+ });
1269
+ turnState = "parked";
1270
+ nodeState = "waiting_on_human";
1271
+ doneReason = `parked on ${token}`;
1272
+ this.log(`fleet: node ${node.node_id} parked on ${token}: ${ask.question}`);
1273
+ } else if (isDone(text)) {
1274
+ turnState = "done";
1275
+ nodeState = "done";
1276
+ doneReason = "charter reported done";
1277
+ }
1278
+ this.journal.endTurn(ctx.turnId, { state: turnState, ...this._usage(ctx), sessionId: ctx.sessionId });
1279
+ this.journal.addNodeUsage(node.node_id, {
1280
+ tokensIn: this._usage(ctx).tokensIn,
1281
+ tokensOut: this._usage(ctx).tokensOut,
1282
+ projectRoot: node.project_root,
1283
+ });
1284
+ // Only what this turn's prompt carried is retired. A message whose body was
1285
+ // not in the prompt keeps the state that makes the next turn pick it up,
1286
+ // which is what makes `consumed` a record of what a model read.
1287
+ this.journal.consumeMessages([...ctx.carried, ...ctx.steeredIds]);
1288
+ this.journal.setNodeState(node.node_id, nodeState, doneReason ? { done_reason: doneReason } : {});
1289
+ this.log(
1290
+ `fleet: node ${node.node_id} turn ${ctx.turnId} ${turnState} effects=${this.journal.effects(ctx.turnId).length} requests=${ctx.requestSeq}`,
1291
+ );
1292
+ }
1293
+
1294
+ // ── control channel ──────────────────────────────────────────────────────
1295
+
1296
+ socketPath() {
1297
+ return controlSocketPath(this.root);
1298
+ }
1299
+
1300
+ /**
1301
+ * The live responder on `path`, or null when nothing answered: no file, no
1302
+ * listener behind the file, or a reply this process wrote itself.
1303
+ */
1304
+ async _responderOn(path) {
1305
+ if (!existsSync(path)) return null;
1306
+ const reply = await requestOnControlSocket(path, { op: "ping" }, { timeoutMs: CONTROL_PROBE_MS });
1307
+ const data = reply?.ok ? reply.data : null;
1308
+ if (!data || data.pid == null || data.pid === process.pid) return null;
1309
+ return data;
1310
+ }
1311
+
1312
+ async _listen() {
1313
+ const path = this.socketPath();
1314
+ const incumbent = await this._responderOn(path);
1315
+ if (incumbent) {
1316
+ // A live responder with another pid is the owner of this root's channel.
1317
+ // Binding over it would leave a supervisor that runs and works while
1318
+ // nothing can reach it, and nothing would say so for the rest of its
1319
+ // life, so this process refuses instead of clobbering. The journal row
1320
+ // follows: `live` for a process that never served would be the same lie.
1321
+ this.journal.setSupervisorState(process.pid, "refused");
1322
+ throw new Error(
1323
+ `control channel ${path} is served by supervisor pid ${incumbent.pid} (epoch ${incumbent.epoch}); refusing to bind over it`,
1324
+ );
1325
+ }
1326
+ if (existsSync(path)) {
1327
+ // A socket file outlives the process that bound it, so a path nobody
1328
+ // answers is a departed binder's leavings. Reclaiming it is what lets the
1329
+ // channel come back after a crash; refusing to would leave a root that no
1330
+ // supervisor can ever serve again.
1331
+ this.log(`fleet: no answer on ${path}; reclaiming the path`);
1332
+ rmSync(path, { force: true });
1333
+ }
1334
+ this.server = createServer((socket) => {
1335
+ let buffer = "";
1336
+ socket.on("data", (chunk) => {
1337
+ buffer += chunk.toString("utf8");
1338
+ let index;
1339
+ while ((index = buffer.indexOf("\n")) >= 0) {
1340
+ const line = buffer.slice(0, index);
1341
+ buffer = buffer.slice(index + 1);
1342
+ if (!line.trim()) continue;
1343
+ this._handleControl(line)
1344
+ .then((reply) => socket.write(JSON.stringify(reply) + "\n"))
1345
+ .catch((error) => socket.write(JSON.stringify({ ok: false, error: error.message }) + "\n"));
1346
+ }
1347
+ });
1348
+ socket.on("error", () => socket.destroy());
1349
+ });
1350
+ await new Promise((resolveListen, rejectListen) => {
1351
+ this.server.once("error", rejectListen);
1352
+ this.server.listen(path, () => {
1353
+ this.server.removeListener("error", rejectListen);
1354
+ chmodSync(path, 0o600);
1355
+ resolveListen();
1356
+ });
1357
+ });
1358
+ const identity = fileIdentity(path);
1359
+ this.boundSocket = identity ? { path, dev: identity.dev, ino: identity.ino } : null;
1360
+ this.journal.setMeta("control_socket", path);
1361
+ return path;
1362
+ }
1363
+
1364
+ async _handleControl(line) {
1365
+ const request = JSON.parse(line);
1366
+ switch (request.op) {
1367
+ case "ping":
1368
+ return { ok: true, data: { pid: process.pid, epoch: this.epoch, active: [...this.active.keys()] } };
1369
+ case "status":
1370
+ return { ok: true, data: buildStatus(this.journal, this) };
1371
+ case "inject": {
1372
+ const node = this.journal.getNode(request.node);
1373
+ if (!node) return { ok: false, error: `no such node: ${request.node}` };
1374
+ // A message for a node no future turn will drive cannot be delivered,
1375
+ // and recording it would be the command claiming it can. Refusing here
1376
+ // is what turns that into an answer the operator sees now.
1377
+ const retired = undeliverableReason(node, this.options.maxAttempts);
1378
+ if (retired) return { ok: false, error: `cannot queue a message for ${node.node_id}: ${retired}; no turn will ever carry it` };
1379
+ const msgId = request.msgId ?? `m-${randomBytes(6).toString("hex")}`;
1380
+ let kind = "user";
1381
+ if (request.token) {
1382
+ const ask = this.journal.getAsk(request.token);
1383
+ if (!ask || ask.state !== "open") return { ok: false, error: `no open ask: ${request.token}` };
1384
+ this.journal.resolveAsk(request.token, { answerText: request.text, answerSource: "fleet-inject" });
1385
+ kind = "answer";
1386
+ if (node.state === "waiting_on_human") this.journal.setNodeState(node.node_id, "settled");
1387
+ }
1388
+ const message = this.journal.addMessage({
1389
+ msgId,
1390
+ nodeId: node.node_id,
1391
+ kind,
1392
+ body: request.text,
1393
+ token: request.token,
1394
+ });
1395
+ const ctx = this.active.get(node.node_id);
1396
+ if (ctx?.driver && !ctx.settled && ctx.turnId) {
1397
+ try {
1398
+ const ack = await ctx.driver.steer(request.text);
1399
+ const disposition = ack?.data?.disposition ?? ack?.disposition ?? "queued";
1400
+ // The ack is the child's queue talking, not a delivery. pi answers
1401
+ // `steer` once the text is queued and hands it to the model later,
1402
+ // or never, so the row stays `recorded` until a `steer_delivered`
1403
+ // event matches its body. That is what keeps a steer the turn never
1404
+ // ran: the next turn carries it, instead of the row being retired
1405
+ // as read by a model that never saw it.
1406
+ return {
1407
+ ok: true,
1408
+ data: { msgId, steered: true, disposition, turnId: ctx.turnId, effectSeq: ctx.toolEnds, message },
1409
+ };
1410
+ } catch (error) {
1411
+ this.log(`fleet: steer for ${node.node_id} refused: ${error.message}`);
1412
+ return { ok: true, data: { msgId, tier: "B", reason: `steer refused: ${error.message}`, message } };
1413
+ }
1414
+ }
1415
+ // Tier B is journalled and prepended to the next drive's prompt. It is
1416
+ // deliberately not pi's `nextTurn` delivery: that waits for the next
1417
+ // user prompt and can land after the topic has moved, which is drift.
1418
+ return { ok: true, data: { msgId, tier: "B", reason: "no turn in flight", message } };
1419
+ }
1420
+ case "stop":
1421
+ setTimeout(() => {
1422
+ this.stop().catch((error) => this.log(`fleet: stop failed: ${error.message}`));
1423
+ }, 10);
1424
+ return { ok: true, data: { stopping: true } };
1425
+ default:
1426
+ return { ok: false, error: `unknown op: ${request.op}` };
1427
+ }
1428
+ }
1429
+
1430
+ /**
1431
+ * Whether the path still resolves to the file this process bound, and why
1432
+ * not when it does not. An unreadable path is not an ownership claim either:
1433
+ * a claim this process cannot check is a claim it does not have.
1434
+ */
1435
+ _socketPathState(path) {
1436
+ const mine = this.boundSocket;
1437
+ if (!mine || mine.path !== path || mine.ino == null) return { owned: false, why: "this process never bound it" };
1438
+ let current;
1439
+ try {
1440
+ current = fileIdentity(path);
1441
+ } catch (error) {
1442
+ return { owned: false, why: `it cannot be read (${error.code ?? error.message})` };
1443
+ }
1444
+ if (!current) return { owned: false, why: "the file is gone" };
1445
+ if (current.dev !== mine.dev || current.ino !== mine.ino) return { owned: false, why: "another process bound it" };
1446
+ return { owned: true, why: "this process bound it and nothing has replaced it" };
1447
+ }
1448
+
1449
+ /**
1450
+ * Give up the control path, if it is still this process's to give up.
1451
+ *
1452
+ * Closing a unix-socket server removes the path it bound, whatever is there
1453
+ * now: `net.Server.close()` unlinks by name, so a server closed after another
1454
+ * process rebound the path deletes that process's socket and leaves a live
1455
+ * supervisor unreachable. The close is therefore the deletion that has to be
1456
+ * gated, not only the unlink after it. When the path is no longer this
1457
+ * process's, the listener is abandoned instead: it is unreachable either way,
1458
+ * and it keeps the event loop referenced, so the only way out of this process
1459
+ * is an explicit `process.exit()` - which does not run the close that would
1460
+ * delete the file that replaced ours.
1461
+ */
1462
+ async _releaseSocketPath() {
1463
+ const path = this.socketPath();
1464
+ const state = this._socketPathState(path);
1465
+ if (!state.owned) {
1466
+ this.log(`fleet: leaving control socket ${path} alone: ${state.why}`);
1467
+ return;
1468
+ }
1469
+ await new Promise((resolveClose) => this.server.close(resolveClose));
1470
+ this.server = null;
1471
+ // The close above removed the path on this platform; the unlink is for a
1472
+ // runtime whose close leaves it behind, and it is safe only because the
1473
+ // identity was confirmed first.
1474
+ rmSync(path, { force: true });
1475
+ }
1476
+
1477
+ async stop() {
1478
+ if (this.stopping) return;
1479
+ this.stopping = true;
1480
+ if (this.timer) clearTimeout(this.timer);
1481
+ if (this.server) await this._releaseSocketPath();
1482
+ for (const [nodeId, ctx] of [...this.active]) {
1483
+ ctx.driver?.killGroup?.("SIGKILL");
1484
+ if (ctx.turnId) {
1485
+ this.journal.markUnknownEffects(ctx.turnId);
1486
+ this.journal.endTurn(ctx.turnId, { state: "interrupted", ...this._usage(ctx) });
1487
+ // The same rule as a failed turn: the prompt went out and the turn did
1488
+ // not settle, so its messages are queued for the next drive rather than
1489
+ // left in a `delivered` row nothing will read again.
1490
+ this.journal.releaseMessages([...ctx.carried, ...ctx.steeredIds]);
1491
+ }
1492
+ this.journal.setNodeState(nodeId, "settled");
1493
+ this.journal.releaseLease(nodeId, this.epoch);
1494
+ this.active.delete(nodeId);
1495
+ }
1496
+ this.journal.setSupervisorState(process.pid, "stopped");
1497
+ this.journal.setMeta("fences", JSON.stringify(this.fences));
1498
+ this.journal.close();
1499
+ // Every stop ends the process the entry point is waiting on: the idle exit,
1500
+ // the control channel's `stop`, and a signal all leave the same way. Stop
1501
+ // is the exit, so nothing has to guess whether the process should still be
1502
+ // up.
1503
+ this.doneResolve(true);
1504
+ }
1505
+ }
1506
+
1507
+ /**
1508
+ * One readable snapshot of the fleet, assembled from the journal plus `ps`.
1509
+ * Shared by `fleet status` and the control channel so both print the same
1510
+ * numbers instead of two nearly-equal views.
1511
+ */
1512
+ export function buildStatus(journal, supervisor = null) {
1513
+ const nodes = journal.listNodes().map((node) => {
1514
+ const turns = journal.turnsFor(node.node_id);
1515
+ const last = turns.at(-1) ?? null;
1516
+ const effects = last ? journal.effects(last.turn_id) : [];
1517
+ const counts = effects.reduce((acc, effect) => {
1518
+ acc[effect.state] = (acc[effect.state] ?? 0) + 1;
1519
+ return acc;
1520
+ }, {});
1521
+ return {
1522
+ nodeId: node.node_id,
1523
+ parentId: node.parent_id,
1524
+ state: node.state,
1525
+ attempts: node.attempts,
1526
+ turns: node.turn_count,
1527
+ sessionId: node.session_id,
1528
+ sessionFile: node.session_file,
1529
+ doneReason: node.done_reason,
1530
+ lastTurn: last ? { turnId: last.turn_id, kind: last.kind, state: last.state, effects: counts } : null,
1531
+ openAsks: journal.openAsks(node.node_id).map((ask) => ask.token),
1532
+ lease: journal.liveLease(node.node_id)?.epoch ?? null,
1533
+ pending: journal.pendingMessages(node.node_id).length,
1534
+ // Queued messages that no turn will ever carry: a count an operator can
1535
+ // read without opening SQLite.
1536
+ undeliverable: journal.messages(node.node_id).filter((message) => message.state === "undeliverable").length,
1537
+ };
1538
+ });
1539
+ const liveSupervisors = journal.supervisors().filter((row) => row.state === "live" && pidIsAlive(row.pid));
1540
+ const childPids = journal
1541
+ .allOpenTurns()
1542
+ .map((turn) => turn.child_pid)
1543
+ .filter(Boolean);
1544
+ const resident = residentMb([...liveSupervisors.map((row) => row.pid), ...childPids]);
1545
+ const meta = journal.meta();
1546
+ return {
1547
+ store: journal.path,
1548
+ schemaVersion: SCHEMA_VERSION,
1549
+ meta,
1550
+ epoch: supervisor?.epoch ?? null,
1551
+ cap: supervisor?.options.cap ?? null,
1552
+ active: supervisor ? [...supervisor.active.keys()] : [],
1553
+ nodes,
1554
+ leases: journal.liveLeases().map((lease) => ({ nodeId: lease.node_id, epoch: lease.epoch, pid: lease.owner_pid })),
1555
+ childPids,
1556
+ supervisors: journal.supervisors().map((row) => ({ pid: row.pid, epoch: row.epoch, state: row.state, driver: row.driver })),
1557
+ asks: journal.openAsks().map((ask) => ({
1558
+ token: ask.token,
1559
+ nodeId: ask.node_id,
1560
+ priority: ask.priority,
1561
+ question: ask.question,
1562
+ defaultAnswer: ask.default_answer,
1563
+ deadlineAt: ask.deadline_at,
1564
+ })),
1565
+ fences: JSON.parse(meta.fences ?? "[]"),
1566
+ // The store's own account of a pass it would not take: a maintenance pass
1567
+ // that could not finish, and one that had to wait for the lock. The journal
1568
+ // is the first place to look and the file beside it the second, because the
1569
+ // deferral is written while the journal cannot be - a reader that stops at
1570
+ // the journal would see nothing exactly when the store is wedged.
1571
+ drain: {
1572
+ deferred:
1573
+ jsonMeta(meta, "store_deferred") ?? jsonMeta(meta, "drain_deferred") ?? deferralFromDisk(journal.root),
1574
+ waited: jsonMeta(meta, "drain_contention"),
1575
+ },
1576
+ resident,
1577
+ };
1578
+ }
1579
+
1580
+ // ── entry point ──────────────────────────────────────────────────────────────
1581
+
1582
+ function parseArgs(argv) {
1583
+ const args = { _: [] };
1584
+ for (let i = 0; i < argv.length; i += 1) {
1585
+ const token = argv[i];
1586
+ if (!token.startsWith("--")) {
1587
+ args._.push(token);
1588
+ continue;
1589
+ }
1590
+ const [key, inline] = token.slice(2).split("=");
1591
+ if (inline != null) {
1592
+ args[key] = inline;
1593
+ continue;
1594
+ }
1595
+ const next = argv[i + 1];
1596
+ if (next == null || next.startsWith("--")) {
1597
+ args[key] = true;
1598
+ continue;
1599
+ }
1600
+ args[key] = next;
1601
+ i += 1;
1602
+ }
1603
+ return args;
1604
+ }
1605
+
1606
+ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
1607
+ const args = parseArgs(process.argv.slice(2));
1608
+ if (!args.root) {
1609
+ console.error(
1610
+ "usage: supervisor.mjs --root <fleet-root> [--cap N] [--driver rpc|scripted] [--naive] [--idle-exit-ms N]\n" +
1611
+ "\n" +
1612
+ "Each child loads the duplication guard. FLEET_GUARD=off in this process's environment disables it for every child it spawns.",
1613
+ );
1614
+ process.exit(2);
1615
+ }
1616
+ const supervisor = new Supervisor({
1617
+ root: String(args.root),
1618
+ cap: args.cap ? Number(args.cap) : undefined,
1619
+ driver: args.driver ? String(args.driver) : undefined,
1620
+ model: args.model ? String(args.model) : undefined,
1621
+ naive: Boolean(args.naive),
1622
+ turnTimeoutMs: args["turn-timeout-ms"] ? Number(args["turn-timeout-ms"]) : undefined,
1623
+ askTtlMs: args["ask-ttl-ms"] ? Number(args["ask-ttl-ms"]) : undefined,
1624
+ idleExitMs: args["idle-exit-ms"] ? Number(args["idle-exit-ms"]) : undefined,
1625
+ extensions: args.extension ? [].concat(args.extension) : [],
1626
+ fullDiscovery: Boolean(args["full-discovery"]),
1627
+ piVersion: args["pi-version"] ? String(args["pi-version"]) : undefined,
1628
+ });
1629
+ try {
1630
+ await supervisor.start();
1631
+ } catch (error) {
1632
+ // A start that cannot happen is an operator's answer, not a stack trace:
1633
+ // the loudest thing this process can do with a refusal is print it and go.
1634
+ console.error(`fleet: ${error.message}`);
1635
+ process.exit(1);
1636
+ }
1637
+ const shutdown = async (signal) => {
1638
+ console.log(`fleet: ${signal} received, stopping`);
1639
+ await supervisor.stop();
1640
+ process.exit(0);
1641
+ };
1642
+ process.on("SIGTERM", () => shutdown("SIGTERM"));
1643
+ process.on("SIGINT", () => shutdown("SIGINT"));
1644
+ // The tick timer is what held the process up; it stops rescheduling itself
1645
+ // when the fleet has nothing left to do. `done` resolves after that stop has
1646
+ // finished, so this is an exit and not an intention to exit.
1647
+ const cleanExit = await supervisor.done;
1648
+ process.exit(cleanExit ? 0 : 1);
1649
+ }