pi-braid 0.40.6 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1494 @@
1
+ /**
2
+ * The fleet supervisor: one process, no model, holding a cap of node sessions.
3
+ *
4
+ * Its whole job is to keep one statement true across a crash: no completed
5
+ * effect is repeated and no node is silently lost. Everything else here is
6
+ * scheduling taste.
7
+ *
8
+ * How that statement stays true:
9
+ *
10
+ * - A turn's prompt is journalled before the child is spawned
11
+ * (`turns.prompt_text`), and the child's process group is journalled with
12
+ * it. A restart therefore knows what was sent and what to fence.
13
+ * - Effects are journalled from the driver's own tool events: the attempt
14
+ * before the tool runs, the result after it. An attempt without a result
15
+ * is the crash residue, and reconciliation turns it into `unknown` rather
16
+ * than into a repeat.
17
+ * - A node resumes by reopening its recorded session id. Its recovery turn
18
+ * carries the journal's account of the interrupted turn: which effects
19
+ * completed, which are unknown. The supervisor never re-sends a prompt it
20
+ * already recorded as sent - that single rule is the difference between
21
+ * resuming and restarting.
22
+ * - The supervisor's epoch is journalled. A restart fences the process
23
+ * groups of every turn its predecessor left open, so a stale drive cannot
24
+ * keep writing while the new one resumes the same session.
25
+ *
26
+ * The naive replay path (`naive: true`) exists to be wrong on purpose: it is
27
+ * the arm the crash drill measures duplication with.
28
+ */
29
+
30
+ import { execFileSync } from "node:child_process";
31
+ import { createHash, randomBytes } from "node:crypto";
32
+ import { chmodSync, existsSync, readFileSync, rmSync, statSync } from "node:fs";
33
+ import { createServer, connect } from "node:net";
34
+ import { tmpdir } from "node:os";
35
+ import { join } from "node:path";
36
+ import { pathToFileURL } from "node:url";
37
+
38
+ import { BUSY_TIMEOUT_MS, openJournal, replayPolicy, resolveFleetRoot, SCHEMA_VERSION } from "./journal.mjs";
39
+ import { createRpcDriver, GUARD_EXTENSION, guardEnvironment } from "./driver-rpc.mjs";
40
+ import { createScriptedDriver, ASK_SENTINEL, DONE_SENTINEL } from "./driver-scripted.mjs";
41
+ import { groupIsAlive, pidIsAlive, residentMb } from "./procs.mjs";
42
+
43
+ export { ASK_SENTINEL, DONE_SENTINEL };
44
+
45
+ export const NODE_STATES = ["settled", "waiting_on_child", "waiting_on_human", "failed", "done"];
46
+
47
+ /**
48
+ * Why no future turn can carry a message for this node, or null.
49
+ *
50
+ * `done` and `failed`-at-its-attempt-bound are the two states the scheduler
51
+ * skips for good, so a message queued for one of them is not early, it is
52
+ * undeliverable. Every other state can still be driven - an open ask gets an
53
+ * answer, a turn in flight ends - so a message queued there is only waiting.
54
+ *
55
+ * `maxAttempts` is the caller's policy, and a caller that does not have one
56
+ * passes null and learns only the `done` half: the command line cannot see the
57
+ * bound a supervisor is running with.
58
+ */
59
+ export function undeliverableReason(node, maxAttempts = null) {
60
+ if (node.state === "done") return "the node is done";
61
+ if (maxAttempts != null && node.state === "failed" && node.attempts >= maxAttempts) {
62
+ return `the node failed after ${node.attempts} attempts`;
63
+ }
64
+ return null;
65
+ }
66
+
67
+ export const DEFAULT_ANSWER = "No decision arrived before the deadline. Continue with the charter's stated default.";
68
+
69
+ /**
70
+ * Is this error SQLite saying "another process holds the write lock"?
71
+ *
72
+ * The primary result codes are what decide: 5 is SQLITE_BUSY, 6 is
73
+ * SQLITE_LOCKED, and 517 is SQLITE_BUSY_SNAPSHOT, which node:sqlite can report
74
+ * as an extended code. The message is the fallback for a driver that hands over
75
+ * a bare Error, and it matches SQLite's own strings rather than the word
76
+ * "locked" on its own, so a schema complaint cannot be mistaken for contention.
77
+ */
78
+ export function isBusyError(error) {
79
+ const code = Number(error?.errcode);
80
+ if (code === 5 || code === 6 || code === 517) return true;
81
+ return /database is locked|database table is locked|SQLITE_BUSY/i.test(String(error?.message ?? ""));
82
+ }
83
+
84
+ /**
85
+ * Block the current thread for `ms`. The drain is synchronous on purpose: its
86
+ * claim and its queue are one transaction, and a promise there would mean the
87
+ * transaction spans an await, which is the shape that lets a second writer in
88
+ * between the two halves. So the pause between two attempts blocks.
89
+ */
90
+ function sleepSync(ms) {
91
+ if (ms <= 0) return;
92
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
93
+ }
94
+
95
+ /** A JSON meta value, or null when the key is absent or unreadable. */
96
+ function jsonMeta(meta, key) {
97
+ if (!meta[key]) return null;
98
+ try {
99
+ return JSON.parse(meta[key]);
100
+ } catch {
101
+ return null;
102
+ }
103
+ }
104
+
105
+ /**
106
+ * Where the control socket lives.
107
+ *
108
+ * A unix socket path is capped (104 bytes on macOS, 108 on Linux), and a fleet
109
+ * root can be deeper than that - a nested project, a scratch tree, a CI
110
+ * workspace. The socket therefore falls back to a short hashed name in the
111
+ * system temp directory, and both the supervisor and the commands derive the
112
+ * same path from the root, so the channel does not depend on where the store
113
+ * happens to sit. The chosen path is journalled as `control_socket`.
114
+ */
115
+ export function controlSocketPath(root) {
116
+ const absolute = resolveFleetRoot(root);
117
+ const local = join(absolute, "control.sock");
118
+ if (Buffer.byteLength(local) <= 100) return local;
119
+ const hash = createHash("sha256").update(absolute).digest("hex").slice(0, 12);
120
+ return join(tmpdir(), `pi-fleet-${hash}.sock`);
121
+ }
122
+
123
+ /**
124
+ * How long the arrival probe waits for an answer before it treats the path as a
125
+ * departed binder's leavings. A supervisor answers a ping in microseconds; the
126
+ * wait is long enough that a busy tick is not mistaken for a dead channel.
127
+ */
128
+ const CONTROL_PROBE_MS = 1_000;
129
+
130
+ /**
131
+ * The identity of whatever file sits at `path`: a unix socket's device and
132
+ * inode are stable for the life of the file, and a rebind at the same path
133
+ * creates a different inode. Used to answer "is the path still the one this
134
+ * process bound?" from the filesystem rather than from memory of a name.
135
+ *
136
+ * Returns null when nothing is there, and throws when the question cannot be
137
+ * answered at all: a caller that cannot tell must not delete.
138
+ */
139
+ function fileIdentity(path) {
140
+ try {
141
+ const stat = statSync(path);
142
+ return { dev: stat.dev, ino: stat.ino };
143
+ } catch (error) {
144
+ if (error.code === "ENOENT") return null;
145
+ throw error;
146
+ }
147
+ }
148
+
149
+ /**
150
+ * One request, one response line, over a unix socket path. Resolves `null` when
151
+ * the path does not answer within the timeout: absent, refused, or silent. The
152
+ * wire format lives with the server it speaks to, so the supervisor's own
153
+ * arrival probe and the commands share one implementation.
154
+ */
155
+ export function requestOnControlSocket(path, request, { timeoutMs = 10_000 } = {}) {
156
+ return new Promise((resolveReply) => {
157
+ const socket = connect(path);
158
+ let buffer = "";
159
+ const finish = (reply) => {
160
+ socket.destroy();
161
+ resolveReply(reply);
162
+ };
163
+ socket.setTimeout(timeoutMs, () => finish(null));
164
+ socket.on("connect", () => socket.write(JSON.stringify(request) + "\n"));
165
+ socket.on("data", (chunk) => {
166
+ buffer += chunk.toString("utf8");
167
+ const index = buffer.indexOf("\n");
168
+ if (index < 0) return;
169
+ try {
170
+ finish(JSON.parse(buffer.slice(0, index)));
171
+ } catch {
172
+ finish(null);
173
+ }
174
+ });
175
+ socket.on("error", () => finish(null));
176
+ });
177
+ }
178
+
179
+ export const CHILD_FLAGS = [
180
+ "--mode",
181
+ "rpc",
182
+ "--no-mcp",
183
+ "--no-context-files",
184
+ "--no-extensions",
185
+ "--no-skills",
186
+ "--no-prompt-templates",
187
+ "--thinking",
188
+ "off",
189
+ ];
190
+
191
+ /** The fleet preamble a node's first turn carries, before its charter body. */
192
+ export function charterPrompt(node, body, pending = []) {
193
+ const lines = [
194
+ `You are node ${node.node_id} of a fleet. Your charter follows.`,
195
+ `Work the charter with your tools. One tool call per step, in order.`,
196
+ `When the charter's steps are complete, end your turn with the single line ${DONE_SENTINEL}.`,
197
+ `If you cannot continue without a decision from a human, end your turn with one line:`,
198
+ `${ASK_SENTINEL} <question> | default: <the answer to use if nobody replies> | priority: <number, 0 when unsure>`,
199
+ "",
200
+ "CHARTER:",
201
+ body.trim(),
202
+ ];
203
+ // A queued message is an operator's instruction for this node, and the first
204
+ // turn is a turn: it rides along instead of waiting for a later one. The
205
+ // charter keeps its shape, and the queue is a section of its own after it.
206
+ if (pending.length > 0) {
207
+ lines.push("", "The operator sent this before your first turn. It applies to this turn as well:");
208
+ for (const message of pending) lines.push(`- ${message.body}`);
209
+ }
210
+ return lines.join("\n");
211
+ }
212
+
213
+ /**
214
+ * The recovery prompt is built from the journal, not from the charter. A node
215
+ * that sees its own completed effects listed cannot be told to start over.
216
+ *
217
+ * `note` is the journal account, and it can be suppressed: the note-removal arm
218
+ * measures what the resumed turn does when the prompt is silent about the
219
+ * effects already applied. Nothing but that arm sets it to false.
220
+ */
221
+ export function recoveryPrompt({ node, interruptedTurn, completed, unknown, pending, note = true }) {
222
+ const lines = [
223
+ `FLEET RESUME. Node ${node.node_id} keeps the same session. Turn ${interruptedTurn.turn_id} was interrupted while it was running.`,
224
+ ];
225
+ if (note && completed.length > 0) {
226
+ lines.push("", "Effects already applied and recorded (do NOT repeat them):");
227
+ for (const effect of completed) {
228
+ lines.push(`- seq ${effect.seq} ${effect.tool_name} completed (idempotency ${effect.idempotency_key})`);
229
+ }
230
+ }
231
+ if (note && unknown.length > 0) {
232
+ lines.push("", "Effects whose outcome is unknown (the tool started and never reported):");
233
+ for (const effect of unknown) {
234
+ lines.push(`- seq ${effect.seq} ${effect.tool_name} unknown (idempotency ${effect.idempotency_key})`);
235
+ }
236
+ // The guard refuses a repeat of these, and the note has to say which gate is
237
+ // closed: a node told to "check the world" without being told what will stop
238
+ // it reads the line as a caution, and one that discovers the refusal by
239
+ // hitting it spends a turn learning what the prompt could have said. The
240
+ // release is named as an operator's decision because it is: a node that can
241
+ // retire its own rows is policing itself.
242
+ lines.push(
243
+ "Re-issuing one of these is refused: an effect whose outcome is unknown counts as history the guard will not let you repeat, and releasing it is an operator's decision, not this turn's. Check the world before acting on these; never repeat a completed effect.",
244
+ );
245
+ }
246
+ if (note && completed.length === 0 && unknown.length === 0) {
247
+ lines.push("", "No effect had been recorded when the turn was interrupted.");
248
+ }
249
+ if (pending.length > 0) {
250
+ lines.push("", "Messages waiting for you:");
251
+ for (const message of pending) lines.push(`- ${message.body}`);
252
+ }
253
+ lines.push("", "Continue the charter from its first unfinished step.");
254
+ return lines.join("\n");
255
+ }
256
+
257
+ /**
258
+ * How much of a child's stderr one turn's log line shows. The driver keeps the
259
+ * last 8000 characters, and the whole of it stays on the driver object; a log
260
+ * line is a report, not a transcript, so it is capped here and names what it
261
+ * withheld rather than trailing off.
262
+ */
263
+ const CHILD_STDERR_SHOWN = 400;
264
+
265
+ /**
266
+ * The log line for a child that wrote to stderr, or null when it wrote nothing.
267
+ *
268
+ * A child's stderr is where a guard that failed open says so, and a fail-open
269
+ * guard has no other visible product: it refuses nothing, records nothing, and
270
+ * the turn looks exactly like one the guard watched and found clean. A wrong
271
+ * FLEET_GUARD_ROOT, the likeliest way to be unguarded, leaves its only trace
272
+ * here, so it is reported on every turn that produced any stderr at all.
273
+ *
274
+ * Collapsed to one line and tail-only, because the alternative is a noisy child
275
+ * deciding how large the supervisor's log gets.
276
+ */
277
+ export function childStderrReport(stderr, { limit = CHILD_STDERR_SHOWN } = {}) {
278
+ const text = String(stderr ?? "");
279
+ if (!text.trim()) return null;
280
+ const collapsed = text.replace(/\s+/g, " ").trim();
281
+ const over = Math.max(0, collapsed.length - limit);
282
+ return {
283
+ chars: text.length,
284
+ withheld: over,
285
+ shown: over > 0 ? `[${over} chars withheld] ${collapsed.slice(-limit)}` : collapsed,
286
+ };
287
+ }
288
+
289
+ export function parseAsk(text) {
290
+ const line = String(text ?? "")
291
+ .split("\n")
292
+ .find((candidate) => candidate.includes(ASK_SENTINEL));
293
+ if (!line) return null;
294
+ const rest = line.slice(line.indexOf(ASK_SENTINEL) + ASK_SENTINEL.length).trim();
295
+ const [question, ...tail] = rest.split("|");
296
+ let defaultAnswer = "";
297
+ let priority = 0;
298
+ for (const field of tail) {
299
+ const [rawKey, ...rawValue] = field.split(":");
300
+ const key = rawKey.trim().toLowerCase();
301
+ const value = rawValue.join(":").trim();
302
+ if (key === "default") defaultAnswer = value;
303
+ if (key === "priority") priority = Number(value) || 0;
304
+ }
305
+ return { question: question.trim(), defaultAnswer: defaultAnswer || DEFAULT_ANSWER, priority };
306
+ }
307
+
308
+ export function isDone(text) {
309
+ return String(text ?? "").includes(DONE_SENTINEL);
310
+ }
311
+
312
+ /** Text content of an assistant message, with thinking blocks dropped. */
313
+ export function assistantText(message) {
314
+ const content = message?.content;
315
+ if (typeof content === "string") return content;
316
+ if (!Array.isArray(content)) return "";
317
+ return content
318
+ .filter((block) => block?.type === "text" && typeof block.text === "string")
319
+ .map((block) => block.text)
320
+ .join("\n");
321
+ }
322
+
323
+ export class Supervisor {
324
+ constructor(options) {
325
+ const defaults = {
326
+ cap: 3,
327
+ tickMs: 250,
328
+ leaseTtlMs: 30_000,
329
+ askTtlMs: 600_000,
330
+ turnTimeoutMs: 300_000,
331
+ maxAttempts: 3,
332
+ idleExitMs: 5_000,
333
+ // The drain's contention policy. Two windows at the journal's own 10s
334
+ // timeout plus one backoff stay inside the 30s lease TTL, so a drain that
335
+ // is waiting cannot let its own lease lapse.
336
+ drainAttempts: 2,
337
+ drainBusyTimeoutMs: BUSY_TIMEOUT_MS,
338
+ drainBackoffMs: 250,
339
+ // How long a single drain may block before the fact is journalled, even
340
+ // though it completed: one attempt that waited 7s for the lock is a fact
341
+ // an operator wants, not a silent success.
342
+ drainWaitReportMs: 1_000,
343
+ driver: "rpc",
344
+ model: "opencode-go/deepseek-v4.1-flash",
345
+ piBin: "pi",
346
+ naive: false,
347
+ control: true,
348
+ extensions: [],
349
+ extraArgs: [],
350
+ };
351
+ // An explicit `undefined` option means "not given", never "override with
352
+ // nothing": the command line produces undefined for every flag it omits.
353
+ this.options = { ...defaults };
354
+ for (const [key, value] of Object.entries(options ?? {})) {
355
+ if (value !== undefined) this.options[key] = value;
356
+ }
357
+ this.journal = openJournal(this.options.root, {
358
+ projectRoot: this.options.projectRoot ?? this.options.root,
359
+ driver: this.options.driver,
360
+ childArgs: this.options.extraArgs ?? [],
361
+ piVersion: this.options.piVersion ?? "unset",
362
+ });
363
+ this.root = this.journal.root;
364
+ this.epoch = null;
365
+ this.active = new Map();
366
+ this.timer = null;
367
+ this.server = null;
368
+ // The file this process bound, as the filesystem names it. A name is not an
369
+ // identity: another process can rebind the same path, so the stop below
370
+ // compares this against the path rather than trusting the name.
371
+ this.boundSocket = null;
372
+ this.stopping = false;
373
+ this.fences = [];
374
+ this.idleSince = null;
375
+ // A drain that gave up and could not write its own record: the store was
376
+ // still locked. The fact is journalled at the first drain that can write.
377
+ this.pendingDrainDeferral = null;
378
+ // Resolved when this process has stopped itself because the fleet had
379
+ // nothing left to do; the entry point exits on it rather than on a guess.
380
+ this.done = new Promise((resolveDone) => {
381
+ this.doneResolve = resolveDone;
382
+ });
383
+ this.log = this.options.log ?? ((line) => console.log(line));
384
+ }
385
+
386
+ async start() {
387
+ // The journal records the driver and the version it ran under, so the
388
+ // choice between an RPC child and an in-process SDK stays reversible.
389
+ if (!this.options.piVersion && this.options.driver !== "scripted") {
390
+ try {
391
+ this.options.piVersion = execFileSync(this.options.piBin, ["--version"], { encoding: "utf8" }).trim();
392
+ } catch (error) {
393
+ this.options.piVersion = `unavailable: ${error.code ?? error.message}`;
394
+ }
395
+ this.journal.setMeta("pi_version", this.options.piVersion);
396
+ }
397
+ this.epoch = this.journal.registerSupervisor({
398
+ pid: process.pid,
399
+ driver: this.options.driver,
400
+ childArgs: this.options.extraArgs ?? [],
401
+ piVersion: this.options.piVersion,
402
+ });
403
+ this.log(`fleet: supervisor pid ${process.pid} epoch ${this.epoch} store ${this.journal.path}`);
404
+ await this.reconcile();
405
+ if (this.options.control) await this._listen();
406
+ this._schedule();
407
+ return this;
408
+ }
409
+
410
+ // ── reconciliation ───────────────────────────────────────────────────────
411
+
412
+ /**
413
+ * Repair what a dead predecessor left: dead epoch rows, expired leases, and
414
+ * every turn still marked in-progress. The turn's process group is fenced
415
+ * before its effects are read, so the next drive cannot race a writer that
416
+ * is still alive.
417
+ *
418
+ * A lease and a turn left behind by the same owner are the same owner's, so
419
+ * both loops ask `ownerLiveness` whether that owner is still there. Fencing a
420
+ * live owner's turn kills a child that is still working and records a
421
+ * recovery that never happened.
422
+ */
423
+ async reconcile() {
424
+ const supervisors = this.journal.supervisors();
425
+ for (const supervisor of supervisors) {
426
+ if (supervisor.pid === process.pid) continue;
427
+ if (supervisor.state === "live" && !pidIsAlive(supervisor.pid)) {
428
+ this.journal.setSupervisorState(supervisor.pid, "dead");
429
+ }
430
+ }
431
+ const owners = this.ownerLiveness();
432
+ for (const lease of this.journal.liveLeases()) {
433
+ if (lease.epoch === this.epoch) continue;
434
+ if (owners.get(lease.epoch)) continue;
435
+ this.journal.expireLease(lease.lease_id);
436
+ }
437
+ let skippedTurns = 0;
438
+ const openTurns = this.journal.allOpenTurns();
439
+ for (const turn of openTurns) {
440
+ if (turn.supervisor_epoch === this.epoch) continue;
441
+ if (owners.get(turn.supervisor_epoch)) {
442
+ skippedTurns += 1;
443
+ continue;
444
+ }
445
+ const fence = await this.fenceGroup(turn);
446
+ const unknown = this.journal.markUnknownEffects(turn.turn_id);
447
+ this.journal.endTurn(turn.turn_id, { state: "interrupted" });
448
+ // Messages the interrupted turn had already recorded as carried go back to
449
+ // the queue: the prompt went out and the turn did not finish, so the
450
+ // recovery turn below is the first place they can be acted on.
451
+ const abandoned = this.journal
452
+ .messages(turn.node_id)
453
+ .filter((message) => message.state === "delivered" && message.delivered_turn_id === turn.turn_id)
454
+ .map((message) => message.msg_id);
455
+ const requeued = this.journal.releaseMessages(abandoned);
456
+ this.journal.bumpAttempt(turn.node_id);
457
+ const node = this.journal.getNode(turn.node_id);
458
+ if (node && node.state !== "done") {
459
+ // The bound closes the crash loop: without it a node that kills its
460
+ // supervisor every time is interrupted, settled, resumed, and
461
+ // interrupted again, for ever.
462
+ if (node.attempts >= this.options.maxAttempts) {
463
+ this.journal.setNodeState(turn.node_id, "failed", {
464
+ done_reason: `attempt bound: ${node.attempts} failed or interrupted attempts reached maxAttempts ${this.options.maxAttempts}`,
465
+ });
466
+ this.log(
467
+ `fleet: node ${turn.node_id} reached the attempt bound (${node.attempts}/${this.options.maxAttempts}); not resuming it`,
468
+ );
469
+ } else {
470
+ this.journal.setNodeState(turn.node_id, "settled");
471
+ }
472
+ }
473
+ this.journal.setMeta("fences", JSON.stringify(this.fences));
474
+ this.log(
475
+ `fleet: reconciled turn ${turn.turn_id} (node ${turn.node_id}) fence=${fence.killed ? "killed" : fence.reason} unknown_effects=${unknown.length} requeued_messages=${requeued}`,
476
+ );
477
+ }
478
+ this.journal.setMeta("reconcile_skipped_turns", String(skippedTurns));
479
+ this.drainExpiredAsks();
480
+ }
481
+
482
+ /**
483
+ * Which supervisor epochs are still alive, by the one predicate both repair
484
+ * loops need: a row that says `live` whose pid answers. The repair pass above
485
+ * has already demoted a live row whose pid is gone, so a `live` row here is a
486
+ * process the box still holds.
487
+ */
488
+ ownerLiveness() {
489
+ const owners = new Map();
490
+ for (const row of this.journal.supervisors()) {
491
+ owners.set(row.epoch, row.state === "live" && pidIsAlive(row.pid));
492
+ }
493
+ return owners;
494
+ }
495
+
496
+ // ── the ask deadline ─────────────────────────────────────────────────────
497
+
498
+ /**
499
+ * The one drain of expired asks. Resolving an ask to its default is only half
500
+ * the job: nothing tells the node to move unless the default is also queued
501
+ * as an ordinary message. Both callers - the tick, and the restart that finds
502
+ * a deadline passed while no supervisor was running - come through here, so
503
+ * the two paths cannot drift into one of them doing half the work.
504
+ *
505
+ * Only what `expireAsks()` claimed is queued. It returns the rows this call
506
+ * actually transitioned, not the rows it read, so a second drainer of the same
507
+ * ask claims nothing and queues nothing: the queue decision rests on the
508
+ * conditional update, not on a shared read.
509
+ *
510
+ * Contention is waited out, not thrown. A drain that meets the store's write
511
+ * lock waits out the journal's busy timeout, retries with a backoff, and
512
+ * reports a drain it could not finish as a journal fact
513
+ * (`drain_deferred`) instead of as an exception from a tick. The earlier
514
+ * shape was measured: two drainers on one store, a full-suite run, one of them
515
+ * `database is locked` and gone, and the asks it was there to drain stayed
516
+ * held. A supervisor whose drain throws does not drain that ask, and its
517
+ * caller is a tick handler, so the failure was a log line and a skipped
518
+ * maintenance pass.
519
+ */
520
+ drainExpiredAsks() {
521
+ const started = Date.now();
522
+ const attempts = Math.max(1, Math.floor(this.options.drainAttempts));
523
+ for (let attempt = 1; attempt <= attempts; attempt += 1) {
524
+ try {
525
+ const { expired, notes } = this.journal.transaction(() => this._claimExpiredAsks(), {
526
+ busyTimeoutMs: this.options.drainBusyTimeoutMs,
527
+ });
528
+ // After the commit, so a retried attempt cannot log the same claim
529
+ // twice, and so the log line describes a decision that survived.
530
+ for (const note of notes) this.log(note);
531
+ this._recordDrainContention({ attempt, waitedMs: Date.now() - started });
532
+ return expired;
533
+ } catch (error) {
534
+ // A locked store is the one error this drain answers itself. Anything
535
+ // else is a defect in the body, and swallowing it would hide a real
536
+ // failure behind a maintenance pass nobody reads.
537
+ if (!isBusyError(error)) throw error;
538
+ const why = error.errstr ?? error.message;
539
+ if (attempt >= attempts) {
540
+ this._recordDrainDeferral({ attempts: attempt, waitedMs: Date.now() - started, error: why });
541
+ return [];
542
+ }
543
+ const backoffMs = this.options.drainBackoffMs * attempt;
544
+ this.log(
545
+ `fleet: drain met ${why} on attempt ${attempt}/${attempts}; waiting ${backoffMs} ms and retrying`,
546
+ );
547
+ sleepSync(backoffMs);
548
+ }
549
+ }
550
+ return [];
551
+ }
552
+
553
+ /**
554
+ * The drain's body: claim the overdue asks and queue one message per claimed
555
+ * row. It runs inside the transaction, so it must stay synchronous and free
556
+ * of side effects that a rollback cannot undo - the log lines are returned to
557
+ * the caller rather than printed here.
558
+ */
559
+ _claimExpiredAsks() {
560
+ // Claim and queue as one transaction. A supervisor killed between the two
561
+ // leaves an ask answered with no message queued for it, and a node parked
562
+ // on that ask then has no open ask, no pending message and no ready kind,
563
+ // so nothing ever drives it again: the parked-forever shape this drain
564
+ // exists to prevent, reached through the drain itself. BEGIN IMMEDIATE also
565
+ // serializes two drainers, so the read that decides a claim cannot land
566
+ // inside the other process's claims.
567
+ const expired = this.journal.expireAsks();
568
+ const notes = [];
569
+ for (const ask of expired) {
570
+ // The ask is claimed and answered by now; what is left is whether the
571
+ // node should be woken. A done node must not be resurrected: the fence
572
+ // loop guards this, and the drain queued for it anyway, so an overdue
573
+ // ask on a finished node flipped it back to `settled` and the default
574
+ // then drove it a second time.
575
+ const node = this.journal.getNode(ask.node_id);
576
+ if (!node || node.state === "done") {
577
+ notes.push(
578
+ `fleet: ask ${ask.token} expired for node ${ask.node_id}, which is ${node ? "done" : "missing"}; default not queued`,
579
+ );
580
+ continue;
581
+ }
582
+ this.journal.addMessage({
583
+ msgId: `m-${randomBytes(6).toString("hex")}`,
584
+ nodeId: ask.node_id,
585
+ kind: "answer",
586
+ body: ask.default_answer,
587
+ token: ask.token,
588
+ });
589
+ this.journal.setNodeState(ask.node_id, "settled");
590
+ notes.push(`fleet: ask ${ask.token} hit its deadline; default answer queued`);
591
+ }
592
+ return { expired, notes };
593
+ }
594
+
595
+ /**
596
+ * Record that a drain waited for the lock, and flush a deferral a previous
597
+ * drain could not write.
598
+ *
599
+ * Neither write is allowed to throw: this runs after a successful drain, and
600
+ * losing the note because another writer holds the lock again is not a reason
601
+ * to report the drain itself as failed. The deferral stays pending until a
602
+ * write lands, which is the earliest point the store can hold the fact at all.
603
+ */
604
+ _recordDrainContention({ attempt, waitedMs }) {
605
+ this._flushDrainDeferral(Date.now());
606
+ if (attempt <= 1 && waitedMs < this.options.drainWaitReportMs) return;
607
+ try {
608
+ this.journal.transaction(
609
+ () =>
610
+ this.journal.setMeta("drain_contention", JSON.stringify({ at: Date.now(), attempts: attempt, waitedMs })),
611
+ { busyTimeoutMs: Math.max(100, this.options.drainBackoffMs) },
612
+ );
613
+ } catch (error) {
614
+ // Best effort: the drain's decision is already committed and correct.
615
+ this.log(`fleet: the drain could not record its wait (${error.message}); the next drain retries the note`);
616
+ }
617
+ }
618
+
619
+ /**
620
+ * Write a pending deferral, if one is pending. `recoveredAt` says whether a
621
+ * later drain got through, which is what tells a reader of the fact whether
622
+ * the work was done after all.
623
+ */
624
+ _flushDrainDeferral(recoveredAt) {
625
+ if (!this.pendingDrainDeferral) return false;
626
+ const fact = recoveredAt == null ? this.pendingDrainDeferral : { ...this.pendingDrainDeferral, recoveredAt };
627
+ try {
628
+ this.journal.transaction(() => this.journal.setMeta("drain_deferred", JSON.stringify(fact)), {
629
+ busyTimeoutMs: Math.max(100, this.options.drainBackoffMs),
630
+ });
631
+ this.pendingDrainDeferral = null;
632
+ if (recoveredAt != null) this.log("fleet: the earlier drain deferral was journalled; a later drain completed the work");
633
+ return true;
634
+ } catch (error) {
635
+ this.log(`fleet: the drain deferral could not be journalled yet (${error.message}); the next drain retries it`);
636
+ return false;
637
+ }
638
+ }
639
+
640
+ /**
641
+ * A drain that gave up after its last attempt: the store stayed locked for
642
+ * the whole budget. This is a journal fact, not an exception, so the tick
643
+ * keeps ticking; the write lands here if the lock has freed in the meantime
644
+ * and is otherwise retried by the next drain that can write.
645
+ */
646
+ _recordDrainDeferral({ attempts, waitedMs, error }) {
647
+ this.pendingDrainDeferral = { at: Date.now(), attempts, waitedMs, error };
648
+ this.log(
649
+ `fleet: drain gave up after ${attempts} attempt(s) in ${waitedMs} ms (${error}); the deferral is journalled when the store accepts a write`,
650
+ );
651
+ this._flushDrainDeferral(null);
652
+ }
653
+
654
+ /**
655
+ * Kill the process group a dead turn left behind, and wait until the group is
656
+ * gone before reading its effects. A signal that was sent is not a fence that
657
+ * held, so the record carries what was observed after the wait.
658
+ */
659
+ async fenceGroup(turn) {
660
+ const pgid = turn.child_pgid;
661
+ const record = {
662
+ turnId: turn.turn_id,
663
+ nodeId: turn.node_id,
664
+ pgid,
665
+ killed: false,
666
+ reason: "",
667
+ at: Date.now(),
668
+ waitedMs: 0,
669
+ };
670
+ if (!pgid) {
671
+ record.reason = "no child recorded";
672
+ } else if (!groupIsAlive(pgid)) {
673
+ record.reason = "group already gone";
674
+ } else {
675
+ try {
676
+ process.kill(-pgid, "SIGKILL");
677
+ record.killed = true;
678
+ record.reason = "SIGKILL to process group";
679
+ } catch (error) {
680
+ record.reason = `kill failed: ${error.code ?? error.message}`;
681
+ }
682
+ const started = Date.now();
683
+ while (groupIsAlive(pgid) && Date.now() - started < 500) {
684
+ await new Promise((resolveWait) => setTimeout(resolveWait, 10));
685
+ }
686
+ record.waitedMs = Date.now() - started;
687
+ }
688
+ record.aliveAfter = groupIsAlive(pgid);
689
+ this.fences.push(record);
690
+ return record;
691
+ }
692
+
693
+ // ── scheduling ───────────────────────────────────────────────────────────
694
+
695
+ _schedule() {
696
+ if (this.stopping) return;
697
+ this.timer = setTimeout(() => {
698
+ this.tick()
699
+ .catch((error) => this.log(`fleet: tick failed: ${error.message}`))
700
+ .finally(() => this._schedule());
701
+ }, this.options.tickMs);
702
+ }
703
+
704
+ async tick() {
705
+ this.journal.heartbeatSupervisor(process.pid);
706
+ for (const [nodeId, ctx] of this.active) {
707
+ const renewed = this.journal.acquireLease({
708
+ nodeId,
709
+ ownerPid: process.pid,
710
+ epoch: this.epoch,
711
+ ttlMs: this.options.leaseTtlMs,
712
+ });
713
+ if (!renewed) this.log(`fleet: lease for ${nodeId} could not be renewed (ctx ${ctx.turnId})`);
714
+ }
715
+ this.drainExpiredAsks();
716
+ this.retireUndeliverableMessages();
717
+ if (this.active.size < this.options.cap) {
718
+ for (const node of this.journal.listNodes()) {
719
+ if (this.stopping || this.active.size >= this.options.cap) break;
720
+ if (this.active.has(node.node_id)) continue;
721
+ const kind = this.readyKind(node);
722
+ if (!kind) continue;
723
+ this.driveNode(node, kind).catch((error) => {
724
+ this.log(`fleet: drive ${node.node_id} threw: ${error.message}`);
725
+ });
726
+ }
727
+ }
728
+ await this.idleCheck();
729
+ }
730
+
731
+ // ── stopping when there is nothing to do ────────────────────────────────
732
+
733
+ /**
734
+ * Whether the fleet has anything left to do. Three things are work at any
735
+ * capacity: a turn this process is running, an open ask (a cap-0 supervisor
736
+ * still drains an expired one, because the drain runs before the cap gate),
737
+ * and a lease this epoch has not let go of.
738
+ *
739
+ * A runnable node is work only when this process could drive it. At cap 0 the
740
+ * drive loop never runs, so a node the scheduler would name is a node this
741
+ * supervisor is structurally unable to start; counting it as work is how a
742
+ * cap-0 supervisor outlives its usefulness holding a journal full of nodes it
743
+ * will never drive.
744
+ */
745
+ hasWork() {
746
+ if (this.active.size > 0) return true;
747
+ if (this.journal.openAsks().length > 0) return true;
748
+ if (this.journal.liveLeases().some((lease) => lease.epoch === this.epoch)) return true;
749
+ if (this.options.cap <= 0) return false;
750
+ return this.journal.listNodes().some((node) => this.readyKind(node) !== null);
751
+ }
752
+
753
+ /**
754
+ * The idle exit: nothing to do for a sustained window means stop and let the
755
+ * process end. A single empty sample is not that window - a node is
756
+ * registered before the supervisor that drives it is started, so a fresh
757
+ * supervisor's first tick legitimately finds nothing ready - so the window is
758
+ * re-sampled on every tick and any work resets it. `idleExitMs: 0` disables
759
+ * the exit for a supervisor meant to wait.
760
+ */
761
+ async idleCheck() {
762
+ const windowMs = this.options.idleExitMs;
763
+ if (!windowMs || this.stopping) return;
764
+ if (this.hasWork()) {
765
+ this.idleSince = null;
766
+ return;
767
+ }
768
+ if (this.idleSince == null) {
769
+ this.idleSince = Date.now();
770
+ return;
771
+ }
772
+ const idleMs = Date.now() - this.idleSince;
773
+ if (idleMs < windowMs) return;
774
+ // One last look in the same synchronous stretch as the stop, so a node that
775
+ // arrived while the window elapsed is not exited from.
776
+ if (this.hasWork()) {
777
+ this.idleSince = null;
778
+ return;
779
+ }
780
+ this.log(`fleet: no work left for ${idleMs} ms; stopping`);
781
+ try {
782
+ await this.stop();
783
+ } catch (error) {
784
+ this.log(`fleet: stop during the idle exit failed: ${error.message}`);
785
+ this.doneResolve(false);
786
+ }
787
+ }
788
+
789
+ /**
790
+ * Why a node is ready to be driven right now, or null. The five states are
791
+ * the design's; `waiting_on_child` is repaired here rather than trusted,
792
+ * because a killed supervisor leaves it behind.
793
+ *
794
+ * Order matters: an interrupted turn outranks a queued message, because the
795
+ * recovery prompt carries the queued messages with it; a node that has never
796
+ * been driven takes its charter before anything else.
797
+ */
798
+ readyKind(node) {
799
+ if (this.journal.openAsks(node.node_id).length > 0) return null;
800
+ if (undeliverableReason(node, this.options.maxAttempts)) return null;
801
+ const turns = this.journal.turnsFor(node.node_id);
802
+ const last = turns.at(-1) ?? null;
803
+ if (last && last.state === "in_progress") return null;
804
+ if (turns.length === 0) return "charter";
805
+ if (last && last.state === "interrupted") return this.options.naive ? "naive" : "recovery";
806
+ if (this.journal.pendingMessages(node.node_id).length > 0) return "message";
807
+ return null;
808
+ }
809
+
810
+ /**
811
+ * Queued messages on a node that will never run again take a terminal state
812
+ * of their own.
813
+ *
814
+ * `recorded` is a promise that some turn will carry the message. When the node
815
+ * it was routed to is done, or failed at its attempt bound, no turn will ever
816
+ * do that, and leaving the row in `recorded` is the same silent lie as
817
+ * retiring a message that never reached a prompt. The sweep runs on every
818
+ * tick, so a message is stamped as soon as a supervisor can see that it cannot
819
+ * be delivered - including one that was queued while the node was still
820
+ * running and whose later turn ended the node instead of carrying it.
821
+ */
822
+ retireUndeliverableMessages() {
823
+ for (const node of this.journal.listNodes()) {
824
+ if (this.active.has(node.node_id)) continue;
825
+ const why = undeliverableReason(node, this.options.maxAttempts);
826
+ if (!why) continue;
827
+ const retired = this.journal.markMessagesUndeliverable(node.node_id);
828
+ if (retired.length > 0) {
829
+ this.log(
830
+ `fleet: ${why}; ${retired.length} queued message(s) for node ${node.node_id} can never be delivered: ${retired
831
+ .map((message) => message.msg_id)
832
+ .join(", ")}`,
833
+ );
834
+ }
835
+ }
836
+ }
837
+
838
+ // ── driving one node ─────────────────────────────────────────────────────
839
+
840
+ createDriver(node, turnId) {
841
+ const config = JSON.parse(node.driver_config || "{}");
842
+ const sessionId = node.session_id ?? `fleet-${node.node_id}-${randomBytes(3).toString("hex")}`;
843
+ if (node.driver === "scripted") {
844
+ return createScriptedDriver({
845
+ steps: config.steps ?? [],
846
+ worldFile: config.worldFile ?? join(this.root, "world.log"),
847
+ sessionId,
848
+ sessionFile: node.session_file ?? join(this.root, "sessions", `${sessionId}.jsonl`),
849
+ });
850
+ }
851
+ return createRpcDriver({
852
+ piBin: this.options.piBin,
853
+ cwd: node.project_root,
854
+ sessionId,
855
+ sessionDir: join(this.root, "sessions"),
856
+ model: this.options.model,
857
+ extensions: this.options.extensions,
858
+ guardExtension: GUARD_EXTENSION,
859
+ env: guardEnvironment({ root: this.root, nodeId: node.node_id, turnId }),
860
+ extraArgs: this.options.extraArgs,
861
+ fullDiscovery: this.options.fullDiscovery,
862
+ thinking: this.options.thinking,
863
+ });
864
+ }
865
+
866
+ async driveNode(node, kind) {
867
+ const lease = this.journal.acquireLease({
868
+ nodeId: node.node_id,
869
+ ownerPid: process.pid,
870
+ epoch: this.epoch,
871
+ ttlMs: this.options.leaseTtlMs,
872
+ });
873
+ if (!lease) return;
874
+ const ctx = {
875
+ nodeId: node.node_id,
876
+ turnId: null,
877
+ driver: null,
878
+ kind,
879
+ toolEnds: 0,
880
+ requestSeq: 0,
881
+ effectSeq: 0,
882
+ usage: { input: 0, output: 0 },
883
+ settled: false,
884
+ settleResolve: null,
885
+ finalText: "",
886
+ lastAssistantText: "",
887
+ error: null,
888
+ startedAt: Date.now(),
889
+ // What this turn's prompt carries, and what was steered into it. Both are
890
+ // decided by the turn itself; the settle gate retires exactly these.
891
+ carried: [],
892
+ steeredIds: [],
893
+ };
894
+ ctx.settlePromise = new Promise((resolveSettle) => {
895
+ ctx.settleResolve = resolveSettle;
896
+ });
897
+ this.active.set(node.node_id, ctx);
898
+ try {
899
+ const refusal = this.journal.budgetRefusal(node.project_root, node.node_id);
900
+ if (refusal) {
901
+ this.journal.setNodeState(node.node_id, "failed", { done_reason: `budget: ${refusal}` });
902
+ return;
903
+ }
904
+ const pending = this.journal.pendingMessages(node.node_id);
905
+ const turns = this.journal.turnsFor(node.node_id);
906
+ const interrupted = turns.at(-1)?.state === "interrupted" ? turns.at(-1) : null;
907
+ let prompt;
908
+ let promptKind = kind;
909
+ if (kind === "recovery" && interrupted) {
910
+ prompt = recoveryPrompt({
911
+ node,
912
+ interruptedTurn: interrupted,
913
+ completed: this.journal.settledEffects(interrupted.turn_id),
914
+ unknown: this.journal
915
+ .effects(interrupted.turn_id)
916
+ .filter((effect) => effect.state === "unknown" || effect.state === "attempted"),
917
+ pending,
918
+ // In-process only, and no command-line flag maps to it: the
919
+ // note-removal arm is the single caller that turns the note off.
920
+ note: this.options.recoveryNote !== false,
921
+ });
922
+ } else if (kind === "message") {
923
+ prompt = ["FLEET MESSAGE.", ...pending.map((message) => `- ${message.body}`), "", "Act on it, then end your turn."].join(
924
+ "\n",
925
+ );
926
+ } else {
927
+ prompt = charterPrompt(node, this._charter(node), pending);
928
+ if (kind === "naive") promptKind = "naive-replay";
929
+ }
930
+ // The messages this prompt carries, asked of the text that is about to be
931
+ // journalled rather than of the branch that built it. The settle gate
932
+ // retires this list and nothing else, so a turn kind that forgets the
933
+ // queue cannot inherit a delivery it never made: the message keeps
934
+ // waiting, and the next turn that does carry it retires it.
935
+ ctx.carried = pending.filter((message) => prompt.includes(message.body)).map((message) => message.msg_id);
936
+ this.journal.setNodeState(node.node_id, "waiting_on_child");
937
+ const turnId = this.journal.beginTurn(node.node_id, {
938
+ kind: promptKind,
939
+ promptText: prompt,
940
+ sessionId: node.session_id,
941
+ supervisorEpoch: this.epoch,
942
+ });
943
+ ctx.turnId = turnId;
944
+ for (const msgId of ctx.carried) {
945
+ this.journal.markMessageDelivered(msgId, {
946
+ tier: "B",
947
+ turnId,
948
+ afterEffectSeq: 0,
949
+ beforeRequestSeq: 0,
950
+ });
951
+ }
952
+ const driver = this.createDriver(node, turnId);
953
+ ctx.driver = driver;
954
+ driver.onEvent((event) => this._onDriverEvent(ctx, event));
955
+ const { sessionId, sessionFile } = await driver.start();
956
+ ctx.sessionId = sessionId;
957
+ this.journal.setNodeSession(node.node_id, { sessionId, sessionFile });
958
+ if (driver.pid) this.journal.setTurnChild(turnId, { pid: driver.pid, pgid: driver.pgid });
959
+ await driver.prompt(prompt);
960
+ await this._waitForSettle(ctx);
961
+ await this._settleTurn(node, ctx);
962
+ } catch (error) {
963
+ ctx.error = error.message;
964
+ if (ctx.turnId) {
965
+ this.journal.markUnknownEffects(ctx.turnId);
966
+ this.journal.endTurn(ctx.turnId, { state: "failed", ...this._usage(ctx) });
967
+ // The prompt went out and the turn did not finish it, so its messages
968
+ // are queued again rather than left in a `delivered` row nothing will
969
+ // ever read.
970
+ this.journal.releaseMessages([...ctx.carried, ...ctx.steeredIds]);
971
+ this.journal.bumpAttempt(node.node_id);
972
+ }
973
+ this.journal.setNodeState(node.node_id, "failed", { done_reason: `turn failed: ${error.message}` });
974
+ this.log(`fleet: node ${node.node_id} turn failed: ${error.message}`);
975
+ } finally {
976
+ if (ctx.driver) await ctx.driver.stop().catch(() => {});
977
+ // After the child is gone, so stderr written on the way out is included.
978
+ const report = childStderrReport(ctx.driver?.stderr);
979
+ if (report) {
980
+ this.log(
981
+ `fleet: node ${node.node_id} turn ${ctx.turnId ?? "-"} child stderr ${report.chars} chars: ${report.shown}`,
982
+ );
983
+ }
984
+ this.journal.releaseLease(node.node_id, this.epoch);
985
+ this.active.delete(node.node_id);
986
+ }
987
+ }
988
+
989
+ _charter(node) {
990
+ try {
991
+ return readFileSync(node.charter_path, "utf8");
992
+ } catch (error) {
993
+ return `(charter unreadable at ${node.charter_path}: ${error.code ?? error.message})`;
994
+ }
995
+ }
996
+
997
+ _usage(ctx) {
998
+ return {
999
+ tokensIn: ctx.usage.input,
1000
+ tokensOut: ctx.usage.output,
1001
+ spendUsd: 0,
1002
+ };
1003
+ }
1004
+
1005
+ _onDriverEvent(ctx, event) {
1006
+ if (!ctx.turnId) return;
1007
+ switch (event.type) {
1008
+ case "tool_start":
1009
+ ctx.effectSeq += 1;
1010
+ this.journal.effectAttempted({
1011
+ turnId: ctx.turnId,
1012
+ nodeId: ctx.nodeId,
1013
+ toolCallId: event.toolCallId,
1014
+ toolName: event.toolName,
1015
+ args: event.args,
1016
+ replay: replayPolicy(event.toolName),
1017
+ seq: ctx.effectSeq,
1018
+ });
1019
+ break;
1020
+ case "tool_end":
1021
+ ctx.toolEnds += 1;
1022
+ this.journal.effectSettled({
1023
+ turnId: ctx.turnId,
1024
+ toolCallId: event.toolCallId,
1025
+ result: event.result,
1026
+ isError: event.isError,
1027
+ });
1028
+ break;
1029
+ case "model_request":
1030
+ ctx.requestSeq = this.journal.recordModelRequest(ctx.turnId, {
1031
+ messageCount: event.messageCount,
1032
+ toolCount: event.toolCount,
1033
+ });
1034
+ break;
1035
+ case "message_end":
1036
+ if (event.role === "assistant") {
1037
+ ctx.lastAssistantText = assistantText(event.message);
1038
+ const usage = event.message?.usage;
1039
+ if (usage) {
1040
+ ctx.usage.input += Number(usage.input ?? 0);
1041
+ ctx.usage.output += Number(usage.output ?? 0);
1042
+ }
1043
+ }
1044
+ break;
1045
+ case "final_text":
1046
+ ctx.finalText = event.text;
1047
+ ctx.settled = true;
1048
+ ctx.settleResolve?.();
1049
+ break;
1050
+ case "steer_delivered": {
1051
+ // The only delivery this fleet can testify to: the child ran the text it
1052
+ // was given, in this turn. Rows are matched by the body the child
1053
+ // carried, so a delivery cannot be charged to a message that was never
1054
+ // run. Every message is recorded before the steer is requested, which
1055
+ // is what makes an event that arrives before the ack findable.
1056
+ const waiting = this.journal.pendingMessages(ctx.nodeId);
1057
+ if (event.text == null) {
1058
+ // A driver that cannot name the body leaves the oldest waiting row as
1059
+ // the only honest candidate; saying so is better than a silent guess.
1060
+ this.log(`fleet: node ${ctx.nodeId} reported a steer with no body; the oldest waiting message is charged with it`);
1061
+ }
1062
+ const matched = event.text == null ? waiting.slice(0, 1) : waiting.filter((message) => message.body === event.text);
1063
+ if (event.text != null && matched.length === 0) {
1064
+ this.log(`fleet: node ${ctx.nodeId} ran a steered message no queued row carried, so no row is marked delivered`);
1065
+ }
1066
+ for (const message of matched) {
1067
+ this.journal.markMessageDelivered(message.msg_id, {
1068
+ tier: "A",
1069
+ turnId: ctx.turnId,
1070
+ afterEffectSeq: event.afterEffectSeq,
1071
+ beforeRequestSeq: ctx.requestSeq,
1072
+ });
1073
+ if (!ctx.steeredIds.includes(message.msg_id)) ctx.steeredIds.push(message.msg_id);
1074
+ }
1075
+ break;
1076
+ }
1077
+ case "settled":
1078
+ ctx.finalText = ctx.finalText || ctx.lastAssistantText;
1079
+ ctx.settled = true;
1080
+ ctx.settleResolve?.();
1081
+ break;
1082
+ default:
1083
+ break;
1084
+ }
1085
+ }
1086
+
1087
+ async _waitForSettle(ctx) {
1088
+ const deadline = Date.now() + this.options.turnTimeoutMs;
1089
+ while (!ctx.settled) {
1090
+ if (Date.now() > deadline) throw new Error(`turn exceeded ${this.options.turnTimeoutMs} ms`);
1091
+ if (ctx.driver?.exit) {
1092
+ const exited = await Promise.race([ctx.driver.exit.then(() => true), new Promise((resolveWait) => setTimeout(() => resolveWait(false), 50))]);
1093
+ if (exited && !ctx.settled) throw new Error("driver exited before settling");
1094
+ }
1095
+ await new Promise((resolveWait) => setTimeout(resolveWait, 50));
1096
+ }
1097
+ }
1098
+
1099
+ async _settleTurn(node, ctx) {
1100
+ const text = ctx.finalText || ctx.lastAssistantText;
1101
+ const ask = parseAsk(text);
1102
+ let turnState = "settled";
1103
+ let nodeState = "settled";
1104
+ let doneReason = null;
1105
+ if (ask) {
1106
+ const token = `ask-${randomBytes(6).toString("hex")}`;
1107
+ this.journal.recordAsk({
1108
+ token,
1109
+ nodeId: node.node_id,
1110
+ turnId: ctx.turnId,
1111
+ priority: ask.priority,
1112
+ question: ask.question,
1113
+ defaultAnswer: ask.defaultAnswer,
1114
+ deadlineAt: Date.now() + this.options.askTtlMs,
1115
+ });
1116
+ turnState = "parked";
1117
+ nodeState = "waiting_on_human";
1118
+ doneReason = `parked on ${token}`;
1119
+ this.log(`fleet: node ${node.node_id} parked on ${token}: ${ask.question}`);
1120
+ } else if (isDone(text)) {
1121
+ turnState = "done";
1122
+ nodeState = "done";
1123
+ doneReason = "charter reported done";
1124
+ }
1125
+ this.journal.endTurn(ctx.turnId, { state: turnState, ...this._usage(ctx), sessionId: ctx.sessionId });
1126
+ this.journal.addNodeUsage(node.node_id, {
1127
+ tokensIn: this._usage(ctx).tokensIn,
1128
+ tokensOut: this._usage(ctx).tokensOut,
1129
+ projectRoot: node.project_root,
1130
+ });
1131
+ // Only what this turn's prompt carried is retired. A message whose body was
1132
+ // not in the prompt keeps the state that makes the next turn pick it up,
1133
+ // which is what makes `consumed` a record of what a model read.
1134
+ this.journal.consumeMessages([...ctx.carried, ...ctx.steeredIds]);
1135
+ this.journal.setNodeState(node.node_id, nodeState, doneReason ? { done_reason: doneReason } : {});
1136
+ this.log(
1137
+ `fleet: node ${node.node_id} turn ${ctx.turnId} ${turnState} effects=${this.journal.effects(ctx.turnId).length} requests=${ctx.requestSeq}`,
1138
+ );
1139
+ }
1140
+
1141
+ // ── control channel ──────────────────────────────────────────────────────
1142
+
1143
+ socketPath() {
1144
+ return controlSocketPath(this.root);
1145
+ }
1146
+
1147
+ /**
1148
+ * The live responder on `path`, or null when nothing answered: no file, no
1149
+ * listener behind the file, or a reply this process wrote itself.
1150
+ */
1151
+ async _responderOn(path) {
1152
+ if (!existsSync(path)) return null;
1153
+ const reply = await requestOnControlSocket(path, { op: "ping" }, { timeoutMs: CONTROL_PROBE_MS });
1154
+ const data = reply?.ok ? reply.data : null;
1155
+ if (!data || data.pid == null || data.pid === process.pid) return null;
1156
+ return data;
1157
+ }
1158
+
1159
+ async _listen() {
1160
+ const path = this.socketPath();
1161
+ const incumbent = await this._responderOn(path);
1162
+ if (incumbent) {
1163
+ // A live responder with another pid is the owner of this root's channel.
1164
+ // Binding over it would leave a supervisor that runs and works while
1165
+ // nothing can reach it, and nothing would say so for the rest of its
1166
+ // life, so this process refuses instead of clobbering. The journal row
1167
+ // follows: `live` for a process that never served would be the same lie.
1168
+ this.journal.setSupervisorState(process.pid, "refused");
1169
+ throw new Error(
1170
+ `control channel ${path} is served by supervisor pid ${incumbent.pid} (epoch ${incumbent.epoch}); refusing to bind over it`,
1171
+ );
1172
+ }
1173
+ if (existsSync(path)) {
1174
+ // A socket file outlives the process that bound it, so a path nobody
1175
+ // answers is a departed binder's leavings. Reclaiming it is what lets the
1176
+ // channel come back after a crash; refusing to would leave a root that no
1177
+ // supervisor can ever serve again.
1178
+ this.log(`fleet: no answer on ${path}; reclaiming the path`);
1179
+ rmSync(path, { force: true });
1180
+ }
1181
+ this.server = createServer((socket) => {
1182
+ let buffer = "";
1183
+ socket.on("data", (chunk) => {
1184
+ buffer += chunk.toString("utf8");
1185
+ let index;
1186
+ while ((index = buffer.indexOf("\n")) >= 0) {
1187
+ const line = buffer.slice(0, index);
1188
+ buffer = buffer.slice(index + 1);
1189
+ if (!line.trim()) continue;
1190
+ this._handleControl(line)
1191
+ .then((reply) => socket.write(JSON.stringify(reply) + "\n"))
1192
+ .catch((error) => socket.write(JSON.stringify({ ok: false, error: error.message }) + "\n"));
1193
+ }
1194
+ });
1195
+ socket.on("error", () => socket.destroy());
1196
+ });
1197
+ await new Promise((resolveListen, rejectListen) => {
1198
+ this.server.once("error", rejectListen);
1199
+ this.server.listen(path, () => {
1200
+ this.server.removeListener("error", rejectListen);
1201
+ chmodSync(path, 0o600);
1202
+ resolveListen();
1203
+ });
1204
+ });
1205
+ const identity = fileIdentity(path);
1206
+ this.boundSocket = identity ? { path, dev: identity.dev, ino: identity.ino } : null;
1207
+ this.journal.setMeta("control_socket", path);
1208
+ return path;
1209
+ }
1210
+
1211
+ async _handleControl(line) {
1212
+ const request = JSON.parse(line);
1213
+ switch (request.op) {
1214
+ case "ping":
1215
+ return { ok: true, data: { pid: process.pid, epoch: this.epoch, active: [...this.active.keys()] } };
1216
+ case "status":
1217
+ return { ok: true, data: buildStatus(this.journal, this) };
1218
+ case "inject": {
1219
+ const node = this.journal.getNode(request.node);
1220
+ if (!node) return { ok: false, error: `no such node: ${request.node}` };
1221
+ // A message for a node no future turn will drive cannot be delivered,
1222
+ // and recording it would be the command claiming it can. Refusing here
1223
+ // is what turns that into an answer the operator sees now.
1224
+ const retired = undeliverableReason(node, this.options.maxAttempts);
1225
+ if (retired) return { ok: false, error: `cannot queue a message for ${node.node_id}: ${retired}; no turn will ever carry it` };
1226
+ const msgId = request.msgId ?? `m-${randomBytes(6).toString("hex")}`;
1227
+ let kind = "user";
1228
+ if (request.token) {
1229
+ const ask = this.journal.getAsk(request.token);
1230
+ if (!ask || ask.state !== "open") return { ok: false, error: `no open ask: ${request.token}` };
1231
+ this.journal.resolveAsk(request.token, { answerText: request.text, answerSource: "fleet-inject" });
1232
+ kind = "answer";
1233
+ if (node.state === "waiting_on_human") this.journal.setNodeState(node.node_id, "settled");
1234
+ }
1235
+ const message = this.journal.addMessage({
1236
+ msgId,
1237
+ nodeId: node.node_id,
1238
+ kind,
1239
+ body: request.text,
1240
+ token: request.token,
1241
+ });
1242
+ const ctx = this.active.get(node.node_id);
1243
+ if (ctx?.driver && !ctx.settled && ctx.turnId) {
1244
+ try {
1245
+ const ack = await ctx.driver.steer(request.text);
1246
+ const disposition = ack?.data?.disposition ?? ack?.disposition ?? "queued";
1247
+ // The ack is the child's queue talking, not a delivery. pi answers
1248
+ // `steer` once the text is queued and hands it to the model later,
1249
+ // or never, so the row stays `recorded` until a `steer_delivered`
1250
+ // event matches its body. That is what keeps a steer the turn never
1251
+ // ran: the next turn carries it, instead of the row being retired
1252
+ // as read by a model that never saw it.
1253
+ return {
1254
+ ok: true,
1255
+ data: { msgId, steered: true, disposition, turnId: ctx.turnId, effectSeq: ctx.toolEnds, message },
1256
+ };
1257
+ } catch (error) {
1258
+ this.log(`fleet: steer for ${node.node_id} refused: ${error.message}`);
1259
+ return { ok: true, data: { msgId, tier: "B", reason: `steer refused: ${error.message}`, message } };
1260
+ }
1261
+ }
1262
+ // Tier B is journalled and prepended to the next drive's prompt. It is
1263
+ // deliberately not pi's `nextTurn` delivery: that waits for the next
1264
+ // user prompt and can land after the topic has moved, which is drift.
1265
+ return { ok: true, data: { msgId, tier: "B", reason: "no turn in flight", message } };
1266
+ }
1267
+ case "stop":
1268
+ setTimeout(() => {
1269
+ this.stop().catch((error) => this.log(`fleet: stop failed: ${error.message}`));
1270
+ }, 10);
1271
+ return { ok: true, data: { stopping: true } };
1272
+ default:
1273
+ return { ok: false, error: `unknown op: ${request.op}` };
1274
+ }
1275
+ }
1276
+
1277
+ /**
1278
+ * Whether the path still resolves to the file this process bound, and why
1279
+ * not when it does not. An unreadable path is not an ownership claim either:
1280
+ * a claim this process cannot check is a claim it does not have.
1281
+ */
1282
+ _socketPathState(path) {
1283
+ const mine = this.boundSocket;
1284
+ if (!mine || mine.path !== path || mine.ino == null) return { owned: false, why: "this process never bound it" };
1285
+ let current;
1286
+ try {
1287
+ current = fileIdentity(path);
1288
+ } catch (error) {
1289
+ return { owned: false, why: `it cannot be read (${error.code ?? error.message})` };
1290
+ }
1291
+ if (!current) return { owned: false, why: "the file is gone" };
1292
+ if (current.dev !== mine.dev || current.ino !== mine.ino) return { owned: false, why: "another process bound it" };
1293
+ return { owned: true, why: "this process bound it and nothing has replaced it" };
1294
+ }
1295
+
1296
+ /**
1297
+ * Give up the control path, if it is still this process's to give up.
1298
+ *
1299
+ * Closing a unix-socket server removes the path it bound, whatever is there
1300
+ * now: `net.Server.close()` unlinks by name, so a server closed after another
1301
+ * process rebound the path deletes that process's socket and leaves a live
1302
+ * supervisor unreachable. The close is therefore the deletion that has to be
1303
+ * gated, not only the unlink after it. When the path is no longer this
1304
+ * process's, the listener is abandoned instead: it is unreachable either way,
1305
+ * and it keeps the event loop referenced, so the only way out of this process
1306
+ * is an explicit `process.exit()` - which does not run the close that would
1307
+ * delete the file that replaced ours.
1308
+ */
1309
+ async _releaseSocketPath() {
1310
+ const path = this.socketPath();
1311
+ const state = this._socketPathState(path);
1312
+ if (!state.owned) {
1313
+ this.log(`fleet: leaving control socket ${path} alone: ${state.why}`);
1314
+ return;
1315
+ }
1316
+ await new Promise((resolveClose) => this.server.close(resolveClose));
1317
+ this.server = null;
1318
+ // The close above removed the path on this platform; the unlink is for a
1319
+ // runtime whose close leaves it behind, and it is safe only because the
1320
+ // identity was confirmed first.
1321
+ rmSync(path, { force: true });
1322
+ }
1323
+
1324
+ async stop() {
1325
+ if (this.stopping) return;
1326
+ this.stopping = true;
1327
+ if (this.timer) clearTimeout(this.timer);
1328
+ if (this.server) await this._releaseSocketPath();
1329
+ for (const [nodeId, ctx] of [...this.active]) {
1330
+ ctx.driver?.killGroup?.("SIGKILL");
1331
+ if (ctx.turnId) {
1332
+ this.journal.markUnknownEffects(ctx.turnId);
1333
+ this.journal.endTurn(ctx.turnId, { state: "interrupted", ...this._usage(ctx) });
1334
+ // The same rule as a failed turn: the prompt went out and the turn did
1335
+ // not settle, so its messages are queued for the next drive rather than
1336
+ // left in a `delivered` row nothing will read again.
1337
+ this.journal.releaseMessages([...ctx.carried, ...ctx.steeredIds]);
1338
+ }
1339
+ this.journal.setNodeState(nodeId, "settled");
1340
+ this.journal.releaseLease(nodeId, this.epoch);
1341
+ this.active.delete(nodeId);
1342
+ }
1343
+ this.journal.setSupervisorState(process.pid, "stopped");
1344
+ this.journal.setMeta("fences", JSON.stringify(this.fences));
1345
+ this.journal.close();
1346
+ // Every stop ends the process the entry point is waiting on: the idle exit,
1347
+ // the control channel's `stop`, and a signal all leave the same way. Stop
1348
+ // is the exit, so nothing has to guess whether the process should still be
1349
+ // up.
1350
+ this.doneResolve(true);
1351
+ }
1352
+ }
1353
+
1354
+ /**
1355
+ * One readable snapshot of the fleet, assembled from the journal plus `ps`.
1356
+ * Shared by `fleet status` and the control channel so both print the same
1357
+ * numbers instead of two nearly-equal views.
1358
+ */
1359
+ export function buildStatus(journal, supervisor = null) {
1360
+ const nodes = journal.listNodes().map((node) => {
1361
+ const turns = journal.turnsFor(node.node_id);
1362
+ const last = turns.at(-1) ?? null;
1363
+ const effects = last ? journal.effects(last.turn_id) : [];
1364
+ const counts = effects.reduce((acc, effect) => {
1365
+ acc[effect.state] = (acc[effect.state] ?? 0) + 1;
1366
+ return acc;
1367
+ }, {});
1368
+ return {
1369
+ nodeId: node.node_id,
1370
+ parentId: node.parent_id,
1371
+ state: node.state,
1372
+ attempts: node.attempts,
1373
+ turns: node.turn_count,
1374
+ sessionId: node.session_id,
1375
+ sessionFile: node.session_file,
1376
+ doneReason: node.done_reason,
1377
+ lastTurn: last ? { turnId: last.turn_id, kind: last.kind, state: last.state, effects: counts } : null,
1378
+ openAsks: journal.openAsks(node.node_id).map((ask) => ask.token),
1379
+ lease: journal.liveLease(node.node_id)?.epoch ?? null,
1380
+ pending: journal.pendingMessages(node.node_id).length,
1381
+ // Queued messages that no turn will ever carry: a count an operator can
1382
+ // read without opening SQLite.
1383
+ undeliverable: journal.messages(node.node_id).filter((message) => message.state === "undeliverable").length,
1384
+ };
1385
+ });
1386
+ const liveSupervisors = journal.supervisors().filter((row) => row.state === "live" && pidIsAlive(row.pid));
1387
+ const childPids = journal
1388
+ .allOpenTurns()
1389
+ .map((turn) => turn.child_pid)
1390
+ .filter(Boolean);
1391
+ const resident = residentMb([...liveSupervisors.map((row) => row.pid), ...childPids]);
1392
+ const meta = journal.meta();
1393
+ return {
1394
+ store: journal.path,
1395
+ schemaVersion: SCHEMA_VERSION,
1396
+ meta,
1397
+ epoch: supervisor?.epoch ?? null,
1398
+ cap: supervisor?.options.cap ?? null,
1399
+ active: supervisor ? [...supervisor.active.keys()] : [],
1400
+ nodes,
1401
+ leases: journal.liveLeases().map((lease) => ({ nodeId: lease.node_id, epoch: lease.epoch, pid: lease.owner_pid })),
1402
+ childPids,
1403
+ supervisors: journal.supervisors().map((row) => ({ pid: row.pid, epoch: row.epoch, state: row.state, driver: row.driver })),
1404
+ asks: journal.openAsks().map((ask) => ({
1405
+ token: ask.token,
1406
+ nodeId: ask.node_id,
1407
+ priority: ask.priority,
1408
+ question: ask.question,
1409
+ defaultAnswer: ask.default_answer,
1410
+ deadlineAt: ask.deadline_at,
1411
+ })),
1412
+ fences: JSON.parse(meta.fences ?? "[]"),
1413
+ // The drain's two facts: a pass that could not finish, and a pass that had
1414
+ // to wait for the store's lock. Both are journal rows, so an operator sees
1415
+ // a supervisor struggling with the store without reading its log, and a
1416
+ // deferral that a later drain completed says so rather than disappearing.
1417
+ drain: {
1418
+ deferred: jsonMeta(meta, "drain_deferred"),
1419
+ waited: jsonMeta(meta, "drain_contention"),
1420
+ },
1421
+ resident,
1422
+ };
1423
+ }
1424
+
1425
+ // ── entry point ──────────────────────────────────────────────────────────────
1426
+
1427
+ function parseArgs(argv) {
1428
+ const args = { _: [] };
1429
+ for (let i = 0; i < argv.length; i += 1) {
1430
+ const token = argv[i];
1431
+ if (!token.startsWith("--")) {
1432
+ args._.push(token);
1433
+ continue;
1434
+ }
1435
+ const [key, inline] = token.slice(2).split("=");
1436
+ if (inline != null) {
1437
+ args[key] = inline;
1438
+ continue;
1439
+ }
1440
+ const next = argv[i + 1];
1441
+ if (next == null || next.startsWith("--")) {
1442
+ args[key] = true;
1443
+ continue;
1444
+ }
1445
+ args[key] = next;
1446
+ i += 1;
1447
+ }
1448
+ return args;
1449
+ }
1450
+
1451
+ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
1452
+ const args = parseArgs(process.argv.slice(2));
1453
+ if (!args.root) {
1454
+ console.error(
1455
+ "usage: supervisor.mjs --root <fleet-root> [--cap N] [--driver rpc|scripted] [--naive] [--idle-exit-ms N]\n" +
1456
+ "\n" +
1457
+ "Each child loads the duplication guard. FLEET_GUARD=off in this process's environment disables it for every child it spawns.",
1458
+ );
1459
+ process.exit(2);
1460
+ }
1461
+ const supervisor = new Supervisor({
1462
+ root: String(args.root),
1463
+ cap: args.cap ? Number(args.cap) : undefined,
1464
+ driver: args.driver ? String(args.driver) : undefined,
1465
+ model: args.model ? String(args.model) : undefined,
1466
+ naive: Boolean(args.naive),
1467
+ turnTimeoutMs: args["turn-timeout-ms"] ? Number(args["turn-timeout-ms"]) : undefined,
1468
+ askTtlMs: args["ask-ttl-ms"] ? Number(args["ask-ttl-ms"]) : undefined,
1469
+ idleExitMs: args["idle-exit-ms"] ? Number(args["idle-exit-ms"]) : undefined,
1470
+ extensions: args.extension ? [].concat(args.extension) : [],
1471
+ fullDiscovery: Boolean(args["full-discovery"]),
1472
+ piVersion: args["pi-version"] ? String(args["pi-version"]) : undefined,
1473
+ });
1474
+ try {
1475
+ await supervisor.start();
1476
+ } catch (error) {
1477
+ // A start that cannot happen is an operator's answer, not a stack trace:
1478
+ // the loudest thing this process can do with a refusal is print it and go.
1479
+ console.error(`fleet: ${error.message}`);
1480
+ process.exit(1);
1481
+ }
1482
+ const shutdown = async (signal) => {
1483
+ console.log(`fleet: ${signal} received, stopping`);
1484
+ await supervisor.stop();
1485
+ process.exit(0);
1486
+ };
1487
+ process.on("SIGTERM", () => shutdown("SIGTERM"));
1488
+ process.on("SIGINT", () => shutdown("SIGINT"));
1489
+ // The tick timer is what held the process up; it stops rescheduling itself
1490
+ // when the fleet has nothing left to do. `done` resolves after that stop has
1491
+ // finished, so this is an exit and not an intention to exit.
1492
+ const cleanExit = await supervisor.done;
1493
+ process.exit(cleanExit ? 0 : 1);
1494
+ }